diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2017-09-18 14:34:04 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2017-10-04 11:15:27 +0000 |
commit | e6430e577f105ad8813c92e75c54660c4985026e (patch) | |
tree | 88115e5d1fb471fea807111924dcccbeadbf9e4f /chromium/components/ntp_snippets | |
parent | 53d399fe6415a96ea6986ec0d402a9c07da72453 (diff) | |
download | qtwebengine-chromium-e6430e577f105ad8813c92e75c54660c4985026e.tar.gz |
BASELINE: Update Chromium to 61.0.3163.99
Change-Id: I8452f34574d88ca2b27af9bd56fc9ff3f16b1367
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>
Diffstat (limited to 'chromium/components/ntp_snippets')
72 files changed, 6969 insertions, 2026 deletions
diff --git a/chromium/components/ntp_snippets/BUILD.gn b/chromium/components/ntp_snippets/BUILD.gn index 54a66cf3d0e..d783ae7facb 100644 --- a/chromium/components/ntp_snippets/BUILD.gn +++ b/chromium/components/ntp_snippets/BUILD.gn @@ -15,6 +15,17 @@ static_library("ntp_snippets") { "bookmarks/bookmark_last_visit_utils.h", "bookmarks/bookmark_suggestions_provider.cc", "bookmarks/bookmark_suggestions_provider.h", + "breaking_news/breaking_news_gcm_app_handler.cc", + "breaking_news/breaking_news_gcm_app_handler.h", + "breaking_news/breaking_news_listener.h", + "breaking_news/breaking_news_suggestions_provider.cc", + "breaking_news/breaking_news_suggestions_provider.h", + "breaking_news/subscription_json_request.cc", + "breaking_news/subscription_json_request.h", + "breaking_news/subscription_manager.cc", + "breaking_news/subscription_manager.h", + "breaking_news/subscription_manager_impl.cc", + "breaking_news/subscription_manager_impl.h", "callbacks.h", "category.cc", "category.h", @@ -49,15 +60,26 @@ static_library("ntp_snippets") { "pref_util.h", "reading_list/reading_list_suggestions_provider.cc", "reading_list/reading_list_suggestions_provider.h", + "remote/cached_image_fetcher.cc", + "remote/cached_image_fetcher.h", + "remote/contextual_json_request.cc", + "remote/contextual_json_request.h", "remote/json_request.cc", "remote/json_request.h", + "remote/json_to_categories.cc", + "remote/json_to_categories.h", "remote/persistent_scheduler.h", + "remote/prefetched_pages_tracker.h", + "remote/prefetched_pages_tracker_impl.cc", + "remote/prefetched_pages_tracker_impl.h", "remote/remote_suggestion.cc", "remote/remote_suggestion.h", "remote/remote_suggestions_database.cc", "remote/remote_suggestions_database.h", "remote/remote_suggestions_fetcher.cc", "remote/remote_suggestions_fetcher.h", + "remote/remote_suggestions_fetcher_impl.cc", + "remote/remote_suggestions_fetcher_impl.h", "remote/remote_suggestions_provider.cc", "remote/remote_suggestions_provider.h", "remote/remote_suggestions_provider_impl.cc", @@ -103,8 +125,10 @@ static_library("ntp_snippets") { "//components/data_use_measurement/core", "//components/favicon/core", "//components/favicon_base", + "//components/gcm_driver", "//components/history/core/browser", "//components/image_fetcher/core", + "//components/language/core/browser", "//components/metrics", "//components/ntp_snippets/remote/proto", "//components/offline_pages/core", @@ -115,7 +139,6 @@ static_library("ntp_snippets") { "//components/sessions", "//components/strings", "//components/sync_sessions", - "//components/translate/core/browser", "//components/url_formatter", "//components/variations", "//components/variations/net", @@ -141,6 +164,9 @@ source_set("unit_tests") { sources = [ "bookmarks/bookmark_last_visit_utils_unittest.cc", "bookmarks/bookmark_suggestions_provider_unittest.cc", + "breaking_news/breaking_news_suggestions_provider_unittest.cc", + "breaking_news/subscription_json_request_unittest.cc", + "breaking_news/subscription_manager_impl_unittest.cc", "category_rankers/click_based_category_ranker_unittest.cc", "category_rankers/constant_category_ranker_unittest.cc", "category_unittest.cc", @@ -149,10 +175,13 @@ source_set("unit_tests") { "offline_pages/recent_tab_suggestions_provider_unittest.cc", "physical_web_pages/physical_web_page_suggestions_provider_unittest.cc", "reading_list/reading_list_suggestions_provider_unittest.cc", + "remote/cached_image_fetcher_unittest.cc", + "remote/contextual_json_request_unittest.cc", "remote/json_request_unittest.cc", + "remote/prefetched_pages_tracker_impl_unittest.cc", "remote/remote_suggestion_unittest.cc", "remote/remote_suggestions_database_unittest.cc", - "remote/remote_suggestions_fetcher_unittest.cc", + "remote/remote_suggestions_fetcher_impl_unittest.cc", "remote/remote_suggestions_provider_impl_unittest.cc", "remote/remote_suggestions_scheduler_impl_unittest.cc", "remote/remote_suggestions_status_service_unittest.cc", @@ -187,6 +216,7 @@ source_set("unit_tests") { "//components/signin/core/common", "//components/strings", "//components/sync:test_support_driver", + "//components/sync_preferences:test_support", "//components/sync_sessions", "//components/variations:test_support", "//components/web_resource:web_resource", diff --git a/chromium/components/ntp_snippets/DEPS b/chromium/components/ntp_snippets/DEPS index 777f897e174..9e6b868768e 100644 --- a/chromium/components/ntp_snippets/DEPS +++ b/chromium/components/ntp_snippets/DEPS @@ -2,17 +2,19 @@ include_rules = [ "+components/data_use_measurement/core", "+components/favicon/core", "+components/favicon_base", + "+components/gcm_driver", "+components/history/core", "+components/image_fetcher", "+components/keyed_service/core", + "+components/language/core/browser", "+components/leveldb_proto", "+components/metrics", "+components/prefs", "+components/reading_list", "+components/signin", "+components/strings/grit/components_strings.h", + "+components/sync_preferences/testing_pref_service_syncable.h", "+components/sync/driver", - "+components/translate/core/browser", "+components/url_formatter", "+components/variations", "+components/version_info", diff --git a/chromium/components/ntp_snippets/OWNERS b/chromium/components/ntp_snippets/OWNERS index c8c8515dcc5..7d3309a4b5b 100644 --- a/chromium/components/ntp_snippets/OWNERS +++ b/chromium/components/ntp_snippets/OWNERS @@ -4,6 +4,7 @@ treib@chromium.org # Unsure who to ask? Choose from the above. markusheintz@chromium.org +sfiera@chromium.org vitaliii@chromium.org # For ios: diff --git a/chromium/components/ntp_snippets/breaking_news/breaking_news_gcm_app_handler.cc b/chromium/components/ntp_snippets/breaking_news/breaking_news_gcm_app_handler.cc new file mode 100644 index 00000000000..5a8584131c6 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/breaking_news_gcm_app_handler.cc @@ -0,0 +1,174 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/breaking_news/breaking_news_gcm_app_handler.h" + +#include "base/strings/string_util.h" +#include "components/gcm_driver/gcm_driver.h" +#include "components/gcm_driver/gcm_profile_service.h" +#include "components/gcm_driver/instance_id/instance_id.h" +#include "components/gcm_driver/instance_id/instance_id_driver.h" +#include "components/ntp_snippets/pref_names.h" + +using instance_id::InstanceID; + +namespace ntp_snippets { + +const char kBreakingNewsGCMAppID[] = "com.google.breakingnews.gcm"; + +// The sender ID is used in the registration process. +// See: https://developers.google.com/cloud-messaging/gcm#senderid +const char kBreakingNewsGCMSenderId[] = "667617379155"; + +// OAuth2 Scope passed to getToken to obtain GCM registration tokens. +// Must match Java GoogleCloudMessaging.INSTANCE_ID_SCOPE. +const char kGCMScope[] = "GCM"; + +// Key of the news json in the data in the pushed breaking news. +const char kPushedNewsKey[] = "payload"; + +BreakingNewsGCMAppHandler::BreakingNewsGCMAppHandler( + gcm::GCMDriver* gcm_driver, + instance_id::InstanceIDDriver* instance_id_driver, + PrefService* pref_service, + std::unique_ptr<SubscriptionManager> subscription_manager, + const ParseJSONCallback& parse_json_callback) + : gcm_driver_(gcm_driver), + instance_id_driver_(instance_id_driver), + pref_service_(pref_service), + subscription_manager_(std::move(subscription_manager)), + parse_json_callback_(parse_json_callback), + weak_ptr_factory_(this) {} + +BreakingNewsGCMAppHandler::~BreakingNewsGCMAppHandler() { + StopListening(); +} + +void BreakingNewsGCMAppHandler::StartListening( + OnNewContentCallback on_new_content_callback) { +#if !defined(OS_ANDROID) + NOTREACHED() + << "The BreakingNewsGCMAppHandler should only be used on Android."; +#endif + Subscribe(); + on_new_content_callback_ = std::move(on_new_content_callback); + gcm_driver_->AddAppHandler(kBreakingNewsGCMAppID, this); +} + +void BreakingNewsGCMAppHandler::StopListening() { + DCHECK_EQ(gcm_driver_->GetAppHandler(kBreakingNewsGCMAppID), this); + gcm_driver_->RemoveAppHandler(kBreakingNewsGCMAppID); + on_new_content_callback_ = OnNewContentCallback(); + subscription_manager_->Unsubscribe(); +} + +void BreakingNewsGCMAppHandler::Subscribe() { + // TODO(mamir): This logic should be moved to the SubscriptionManager. + std::string token = + pref_service_->GetString(prefs::kBreakingNewsGCMSubscriptionTokenCache); + // If a token has been already obtained, subscribe directly at the content + // suggestions server. Otherwise, obtain a GCM token first. + if (!token.empty()) { + if (!subscription_manager_->IsSubscribed() || + subscription_manager_->NeedsToResubscribe()) { + subscription_manager_->Subscribe(token); + } + return; + } + + instance_id_driver_->GetInstanceID(kBreakingNewsGCMAppID) + ->GetToken(kBreakingNewsGCMSenderId, kGCMScope, + /*options=*/std::map<std::string, std::string>(), + base::Bind(&BreakingNewsGCMAppHandler::DidSubscribe, + weak_ptr_factory_.GetWeakPtr())); +} + +void BreakingNewsGCMAppHandler::DidSubscribe( + const std::string& subscription_token, + InstanceID::Result result) { + switch (result) { + case InstanceID::SUCCESS: + pref_service_->SetString(prefs::kBreakingNewsGCMSubscriptionTokenCache, + subscription_token); + subscription_manager_->Subscribe(subscription_token); + return; + case InstanceID::INVALID_PARAMETER: + case InstanceID::DISABLED: + case InstanceID::ASYNC_OPERATION_PENDING: + case InstanceID::SERVER_ERROR: + case InstanceID::UNKNOWN_ERROR: + DLOG(WARNING) + << "Push messaging subscription failed; InstanceID::Result = " + << result; + break; + case InstanceID::NETWORK_ERROR: + break; + } +} + +void BreakingNewsGCMAppHandler::ShutdownHandler() {} + +void BreakingNewsGCMAppHandler::OnStoreReset() { + pref_service_->ClearPref(prefs::kBreakingNewsGCMSubscriptionTokenCache); +} + +void BreakingNewsGCMAppHandler::OnMessage(const std::string& app_id, + const gcm::IncomingMessage& message) { + DCHECK_EQ(app_id, kBreakingNewsGCMAppID); + + gcm::MessageData::const_iterator it = message.data.find(kPushedNewsKey); + if (it == message.data.end()) { + LOG(WARNING) + << "Receiving pushed content failure: Breaking News ID missing."; + return; + } + + std::string news = it->second; + + parse_json_callback_.Run(news, + base::Bind(&BreakingNewsGCMAppHandler::OnJsonSuccess, + weak_ptr_factory_.GetWeakPtr()), + base::Bind(&BreakingNewsGCMAppHandler::OnJsonError, + weak_ptr_factory_.GetWeakPtr(), news)); +} + +void BreakingNewsGCMAppHandler::OnMessagesDeleted(const std::string& app_id) { + // Messages don't get deleted. + NOTREACHED() << "BreakingNewsGCMAppHandler messages don't get deleted."; +} + +void BreakingNewsGCMAppHandler::OnSendError( + const std::string& app_id, + const gcm::GCMClient::SendErrorDetails& details) { + // Should never be called because we don't send GCM messages to + // the server. + NOTREACHED() << "BreakingNewsGCMAppHandler doesn't send GCM messages."; +} + +void BreakingNewsGCMAppHandler::OnSendAcknowledged( + const std::string& app_id, + const std::string& message_id) { + // Should never be called because we don't send GCM messages to + // the server. + NOTREACHED() << "BreakingNewsGCMAppHandler doesn't send GCM messages."; +} + +void BreakingNewsGCMAppHandler::RegisterProfilePrefs( + PrefRegistrySimple* registry) { + registry->RegisterStringPref(prefs::kBreakingNewsGCMSubscriptionTokenCache, + std::string()); +} + +void BreakingNewsGCMAppHandler::OnJsonSuccess( + std::unique_ptr<base::Value> content) { + on_new_content_callback_.Run(std::move(content)); +} + +void BreakingNewsGCMAppHandler::OnJsonError(const std::string& json_str, + const std::string& error) { + LOG(WARNING) << "Error parsing JSON:" << error + << " when parsing:" << json_str; +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/breaking_news/breaking_news_gcm_app_handler.h b/chromium/components/ntp_snippets/breaking_news/breaking_news_gcm_app_handler.h new file mode 100644 index 00000000000..59527c39d21 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/breaking_news_gcm_app_handler.h @@ -0,0 +1,106 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_BREAKING_NEWS_GCM_APP_HANDLER_H_ +#define COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_BREAKING_NEWS_GCM_APP_HANDLER_H_ + +#include "base/memory/weak_ptr.h" +#include "components/gcm_driver/gcm_app_handler.h" +#include "components/gcm_driver/instance_id/instance_id.h" +#include "components/ntp_snippets/breaking_news/breaking_news_listener.h" +#include "components/ntp_snippets/breaking_news/subscription_manager.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" + +class PrefRegistrySimple; + +namespace gcm { +class GCMDriver; +} + +namespace instance_id { +class InstanceIDDriver; +} + +namespace ntp_snippets { + +// Handler for pushed GCM breaking news. It retrieves a subscription token +// from the GCM server and registers/unregisters itself with the GCM service to +// be called upon received push breaking news. +class BreakingNewsGCMAppHandler : public BreakingNewsListener, + public gcm::GCMAppHandler { + public: + // Callbacks for JSON parsing to allow injecting platform-dependent code. + using SuccessCallback = + base::Callback<void(std::unique_ptr<base::Value> result)>; + using ErrorCallback = base::Callback<void(const std::string& error)>; + using ParseJSONCallback = + base::Callback<void(const std::string& raw_json_string, + const SuccessCallback& success_callback, + const ErrorCallback& error_callback)>; + + using OnNewContentCallback = + base::Callback<void(std::unique_ptr<base::Value> content)>; + + BreakingNewsGCMAppHandler( + gcm::GCMDriver* gcm_driver, + instance_id::InstanceIDDriver* instance_id_driver, + PrefService* pref_service_, + std::unique_ptr<SubscriptionManager> subscription_manager, + const ParseJSONCallback& parse_json_callback); + + // If still listening, calls StopListening() + ~BreakingNewsGCMAppHandler() override; + + // BreakingNewsListener overrides. + void StartListening(OnNewContentCallback on_new_content_callback) override; + void StopListening() override; + + // GCMAppHandler overrides. + void ShutdownHandler() override; + void OnStoreReset() override; + void OnMessage(const std::string& app_id, + const gcm::IncomingMessage& message) override; + void OnMessagesDeleted(const std::string& app_id) override; + void OnSendError(const std::string& app_id, + const gcm::GCMClient::SendErrorDetails& details) override; + void OnSendAcknowledged(const std::string& app_id, + const std::string& message_id) override; + + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + private: + // Retrieves a subscription token that allows the content suggestions server + // to push content via GCM messages. Calling this method multiple times is not + // necessary but does not harm since the same token is returned everytime. + void Subscribe(); + + // Called after the subscription is obtained from the GCM server. + void DidSubscribe(const std::string& subscription_token, + instance_id::InstanceID::Result result); + + // Called after successfully parsing the received suggestion JSON. + void OnJsonSuccess(std::unique_ptr<base::Value> content); + + // Called in case the received suggestion JSON inside the GCM has a parse + // error. + void OnJsonError(const std::string& json_str, const std::string& error); + + gcm::GCMDriver* const gcm_driver_; + instance_id::InstanceIDDriver* const instance_id_driver_; + PrefService* const pref_service_; + const std::unique_ptr<SubscriptionManager> subscription_manager_; + const ParseJSONCallback parse_json_callback_; + + // Called after every time a new message is received in OnMessage() to notify + // the content provider. + OnNewContentCallback on_new_content_callback_; + + base::WeakPtrFactory<BreakingNewsGCMAppHandler> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(BreakingNewsGCMAppHandler); +}; +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_BREAKING_NEWS_GCM_APP_HANDLER_H_ diff --git a/chromium/components/ntp_snippets/breaking_news/breaking_news_listener.h b/chromium/components/ntp_snippets/breaking_news/breaking_news_listener.h new file mode 100644 index 00000000000..e516322db1c --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/breaking_news_listener.h @@ -0,0 +1,32 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_BREAKING_NEWS_LISTENER_H_ +#define COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_BREAKING_NEWS_LISTENER_H_ + +#include <memory> + +#include "base/callback_forward.h" +#include "base/values.h" + +namespace ntp_snippets { + +class BreakingNewsListener { + public: + using OnNewContentCallback = + base::Callback<void(std::unique_ptr<base::Value> content)>; + + virtual ~BreakingNewsListener() = default; + + // Subscribe to the breaking news service and start listening for pushed + // breaking news. Must not be called if already listening. + virtual void StartListening(OnNewContentCallback on_new_content_callback) = 0; + + // Stop listening for incoming breaking news. Any further pushed breaking news + // will be ignored. Must be called while listening. + virtual void StopListening() = 0; +}; +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_BREAKING_NEWS_LISTENER_H_ diff --git a/chromium/components/ntp_snippets/breaking_news/breaking_news_suggestions_provider.cc b/chromium/components/ntp_snippets/breaking_news/breaking_news_suggestions_provider.cc new file mode 100644 index 00000000000..d4672ad8f76 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/breaking_news_suggestions_provider.cc @@ -0,0 +1,155 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/breaking_news/breaking_news_suggestions_provider.h" + +#include "base/bind.h" +#include "base/json/json_writer.h" +#include "base/time/clock.h" +#include "components/ntp_snippets/breaking_news/breaking_news_listener.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/ntp_snippets/remote/json_to_categories.h" +#include "components/strings/grit/components_strings.h" + +namespace ntp_snippets { + +BreakingNewsSuggestionsProvider::BreakingNewsSuggestionsProvider( + ContentSuggestionsProvider::Observer* observer, + std::unique_ptr<BreakingNewsListener> breaking_news_raw_data_provider, + std::unique_ptr<base::Clock> clock, + std::unique_ptr<RemoteSuggestionsDatabase> database) + : ContentSuggestionsProvider(observer), + breaking_news_raw_data_provider_( + std::move(breaking_news_raw_data_provider)), + clock_(std::move(clock)), + database_(std::move(database)), + provided_category_( + Category::FromKnownCategory(KnownCategories::BREAKING_NEWS)), + category_status_(CategoryStatus::INITIALIZING) { + database_->SetErrorCallback( + base::Bind(&BreakingNewsSuggestionsProvider::OnDatabaseError, + base::Unretained(this))); + database_->LoadSnippets( + base::Bind(&BreakingNewsSuggestionsProvider::OnDatabaseLoaded, + base::Unretained(this))); + // Unretained because |this| owns |breaking_news_listener_|. + breaking_news_raw_data_provider_->StartListening( + base::Bind(&BreakingNewsSuggestionsProvider::OnNewContentSuggestion, + base::Unretained(this))); +} + +BreakingNewsSuggestionsProvider::~BreakingNewsSuggestionsProvider() { + breaking_news_raw_data_provider_->StopListening(); +} + +void BreakingNewsSuggestionsProvider::OnNewContentSuggestion( + std::unique_ptr<base::Value> content) { + DCHECK(content); + const base::Time receive_time = clock_->Now(); + FetchedCategoriesVector categories; + if (!JsonToCategories(*content, &categories, receive_time)) { + std::string content_json; + base::JSONWriter::Write(*content, &content_json); + LOG(WARNING) << "Received invalid breaking news: " << content_json; + return; + } + DCHECK_EQ(categories.size(), static_cast<size_t>(1)); + auto& fetched_category = categories[0]; + Category category = fetched_category.category; + DCHECK(category.IsKnownCategory(KnownCategories::BREAKING_NEWS)); + if (database_->IsInitialized()) { + database_->SaveSnippets(fetched_category.suggestions); + } else { + // TODO(mamir): Check how often a breaking news is received before DB is + // initialized. + LOG(WARNING) << "Cannot store breaking news, database is not initialized."; + } + NotifyNewSuggestions(std::move(fetched_category.suggestions)); +} + +//////////////////////////////////////////////////////////////////////////////// +// Private methods + +CategoryStatus BreakingNewsSuggestionsProvider::GetCategoryStatus( + Category category) { + DCHECK_EQ(category, provided_category_); + return category_status_; +} + +CategoryInfo BreakingNewsSuggestionsProvider::GetCategoryInfo( + Category category) { + // TODO(mamir): needs to be corrected, just a placeholer + return CategoryInfo(base::string16(), + ContentSuggestionsCardLayout::MINIMAL_CARD, + ContentSuggestionsAdditionalAction::VIEW_ALL, + /*show_if_empty=*/false, base::string16()); +} + +void BreakingNewsSuggestionsProvider::DismissSuggestion( + const ContentSuggestion::ID& suggestion_id) { + // TODO(mamir): implement. +} + +void BreakingNewsSuggestionsProvider::FetchSuggestionImage( + const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) { + // TODO(mamir): implement. +} + +void BreakingNewsSuggestionsProvider::Fetch( + const Category& category, + const std::set<std::string>& known_suggestion_ids, + const FetchDoneCallback& callback) { + // TODO(jkrcal): Make Fetch method optional. +} + +void BreakingNewsSuggestionsProvider::ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) { + observer()->OnNewSuggestions(this, provided_category_, + std::vector<ContentSuggestion>()); +} + +void BreakingNewsSuggestionsProvider::ClearCachedSuggestions( + Category category) { + DCHECK_EQ(category, provided_category_); + // TODO(mamir): clear the cached suggestions. +} + +void BreakingNewsSuggestionsProvider::GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) { + // TODO(mamir): implement. +} + +void BreakingNewsSuggestionsProvider::ClearDismissedSuggestionsForDebugging( + Category category) { + // TODO(mamir): implement. +} + +void BreakingNewsSuggestionsProvider::OnDatabaseLoaded( + std::vector<std::unique_ptr<RemoteSuggestion>> suggestions) { + // TODO(mamir): check and update DB status. + NotifyNewSuggestions(std::move(suggestions)); +} + +void BreakingNewsSuggestionsProvider::OnDatabaseError() { + // TODO(mamir): implement. +} + +void BreakingNewsSuggestionsProvider::NotifyNewSuggestions( + std::vector<std::unique_ptr<RemoteSuggestion>> suggestions) { + std::vector<ContentSuggestion> result; + for (const std::unique_ptr<RemoteSuggestion>& suggestion : suggestions) { + result.emplace_back(suggestion->ToContentSuggestion(provided_category_)); + } + + DVLOG(1) << "NotifyNewSuggestions(): " << result.size() + << " items in category " << provided_category_; + observer()->OnNewSuggestions(this, provided_category_, std::move(result)); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/breaking_news/breaking_news_suggestions_provider.h b/chromium/components/ntp_snippets/breaking_news/breaking_news_suggestions_provider.h new file mode 100644 index 00000000000..eb62272dbb2 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/breaking_news_suggestions_provider.h @@ -0,0 +1,83 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_BREAKING_NEWS_SUGGESTIONS_PROVIDER_H_ +#define COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_BREAKING_NEWS_SUGGESTIONS_PROVIDER_H_ + +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "components/ntp_snippets/remote/remote_suggestions_database.h" +#include "components/prefs/pref_registry_simple.h" + +namespace ntp_snippets { +class BreakingNewsListener; +} + +namespace base { +class Clock; +} + +namespace ntp_snippets { + +// Receives breaking news suggestions asynchronously via BreakingNewsListener, +// stores them and provides them as content suggestions. +// This class is final because it does things in its constructor which make it +// unsafe to derive from it. +class BreakingNewsSuggestionsProvider final + : public ContentSuggestionsProvider { + public: + BreakingNewsSuggestionsProvider( + ContentSuggestionsProvider::Observer* observer, + std::unique_ptr<BreakingNewsListener> breaking_news_raw_data_provider, + std::unique_ptr<base::Clock> clock, + std::unique_ptr<RemoteSuggestionsDatabase> database); + ~BreakingNewsSuggestionsProvider() override; + + // ContentSuggestionsProvider implementation. + CategoryStatus GetCategoryStatus(Category category) override; + CategoryInfo GetCategoryInfo(Category category) override; + void DismissSuggestion(const ContentSuggestion::ID& suggestion_id) override; + void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) override; + void Fetch(const Category& category, + const std::set<std::string>& known_suggestion_ids, + const FetchDoneCallback& callback) override; + void ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) override; + void ClearCachedSuggestions(Category category) override; + void GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) override; + void ClearDismissedSuggestionsForDebugging(Category category) override; + + private: + // Callback called from the breaking news listener when new content has been + // pushed from the server. + void OnNewContentSuggestion(std::unique_ptr<base::Value> content); + + // Callbacks for the RemoteSuggestionsDatabase. + void OnDatabaseLoaded( + std::vector<std::unique_ptr<RemoteSuggestion>> suggestions); + void OnDatabaseError(); + + void NotifyNewSuggestions( + std::vector<std::unique_ptr<RemoteSuggestion>> suggestions); + + std::unique_ptr<BreakingNewsListener> breaking_news_raw_data_provider_; + std::unique_ptr<base::Clock> clock_; + + // The database for persisting suggestions. + std::unique_ptr<RemoteSuggestionsDatabase> database_; + + const Category provided_category_; + CategoryStatus category_status_; + + DISALLOW_COPY_AND_ASSIGN(BreakingNewsSuggestionsProvider); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_BREAKING_NEWS_SUGGESTIONS_PROVIDER_H_ diff --git a/chromium/components/ntp_snippets/breaking_news/breaking_news_suggestions_provider_unittest.cc b/chromium/components/ntp_snippets/breaking_news/breaking_news_suggestions_provider_unittest.cc new file mode 100644 index 00000000000..bdfbbddaa77 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/breaking_news_suggestions_provider_unittest.cc @@ -0,0 +1,152 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/breaking_news/breaking_news_suggestions_provider.h" + +#include "base/files/scoped_temp_dir.h" +#include "base/json/json_reader.h" +#include "base/memory/ptr_util.h" +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" +#include "base/single_thread_task_runner.h" +#include "base/strings/utf_string_conversions.h" +#include "base/test/simple_test_clock.h" +#include "base/threading/thread_task_runner_handle.h" +#include "components/ntp_snippets/breaking_news/breaking_news_listener.h" +#include "components/ntp_snippets/breaking_news/breaking_news_suggestions_provider.h" +#include "components/ntp_snippets/mock_content_suggestions_provider_observer.h" +#include "testing/gtest/include/gtest/gtest.h" + +using testing::_; +using testing::Eq; +using testing::Property; +using testing::SaveArg; +using testing::SizeIs; +using testing::StrictMock; + +namespace ntp_snippets { + +namespace { + +class MockBreakingNewsListener : public BreakingNewsListener { + public: + MOCK_METHOD1(StartListening, void(OnNewContentCallback callback)); + MOCK_METHOD0(StopListening, void()); +}; + +class BreakingNewsSuggestionsProviderTest : public testing::Test { + public: + BreakingNewsSuggestionsProviderTest() { + CHECK(database_dir_.CreateUniqueTempDir()); + } + + ~BreakingNewsSuggestionsProviderTest() { + EXPECT_CALL(*listener_, StopListening()).RetiresOnSaturation(); + } + + protected: + void InitializeBreakingNewsSuggestionsProvider() { + auto listener = base::MakeUnique<StrictMock<MockBreakingNewsListener>>(); + listener_ = listener.get(); + + scoped_refptr<base::SingleThreadTaskRunner> task_runner( + base::ThreadTaskRunnerHandle::Get()); + // TODO(mamir): Use a mock DB instead of a real one. A DB interface needs to + // be extracted for that. + // TODO(mamir): Add tests for database failure once the DB gets mockable. + auto database = base::MakeUnique<RemoteSuggestionsDatabase>( + database_dir_.GetPath(), task_runner); + + EXPECT_CALL(*listener_, StartListening(_)) + .WillOnce(SaveArg<0>(&on_new_content_callback_)) + .RetiresOnSaturation(); + // The observer will be updated with an empty list upon loading the + // database. + EXPECT_CALL(observer_, OnNewSuggestions(_, + Category::FromKnownCategory( + KnownCategories::BREAKING_NEWS), + SizeIs(0))) + .RetiresOnSaturation(); + provider_ = base::MakeUnique<BreakingNewsSuggestionsProvider>( + &observer_, std::move(listener), + base::MakeUnique<base::SimpleTestClock>(), std::move(database)); + + // TODO(mamir): Find a better way to wait for initialization to finish. + base::RunLoop().RunUntilIdle(); + } + + base::Time StringToTime(const std::string& time_string) { + base::Time out_time; + CHECK(base::Time::FromString(time_string.c_str(), &out_time)); + return out_time; + } + + BreakingNewsListener::OnNewContentCallback on_new_content_callback_; + base::MessageLoop message_loop_; + StrictMock<MockBreakingNewsListener>* listener_; + std::unique_ptr<BreakingNewsSuggestionsProvider> provider_; + StrictMock<MockContentSuggestionsProviderObserver> observer_; + base::ScopedTempDir database_dir_; +}; + +TEST_F(BreakingNewsSuggestionsProviderTest, + ShouldPropagatePushedNewsWithoutModifyingToObserver) { + InitializeBreakingNewsSuggestionsProvider(); + std::string json = + "{\"categories\" : [{" + " \"id\": 8," + " \"localizedTitle\": \"Breaking News\"," + " \"suggestions\" : [{" + " \"ids\" : [\"http://localhost/id\"]," + " \"title\" : \"Title string\"," + " \"snippet\" : \"Snippet string\"," + " \"fullPageUrl\" : \"http://localhost/fullUrl\"," + " \"creationTime\" : \"2016-06-30T11:01:37.000Z\"," + " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," + " \"attribution\" : \"Attribution string\"," + " \"imageUrl\" : \"http://localhost/foobar.jpg\" " + " }]" + "}]}"; + + // TODO(mamir): Test imageUrl and expirationTime. They aren't directly + // testable because they aren't part of ContentSuggestion. However, they could + // be checked indirectly via FetchImage or by providing an expired suggestion + // and checking that it is not propagated further. + EXPECT_CALL( + observer_, + OnNewSuggestions( + _, Eq(Category::FromKnownCategory(KnownCategories::BREAKING_NEWS)), + ElementsAre(AllOf( + Property(&ContentSuggestion::id, + ContentSuggestion::ID(Category::FromRemoteCategory(8), + "http://localhost/id")), + Property(&ContentSuggestion::title, + base::UTF8ToUTF16("Title string")), + Property(&ContentSuggestion::snippet_text, + base::UTF8ToUTF16("Snippet string")), + Property(&ContentSuggestion::url, + GURL("http://localhost/fullUrl")), + Property(&ContentSuggestion::publish_date, + StringToTime("2016-06-30T11:01:37.000Z")), + Property(&ContentSuggestion::publisher_name, + base::UTF8ToUTF16("Attribution string")) + + )))); + on_new_content_callback_.Run(base::JSONReader().ReadToValue(json)); +} + +TEST_F(BreakingNewsSuggestionsProviderTest, + ClearHistoryShouldNotifyObserverWithEmptySuggestionsList) { + InitializeBreakingNewsSuggestionsProvider(); + EXPECT_CALL(observer_, OnNewSuggestions(_, + Category::FromKnownCategory( + KnownCategories::BREAKING_NEWS), + SizeIs(0))); + provider_->ClearHistory(base::Time::UnixEpoch(), base::Time::Max(), + base::Callback<bool(const GURL& url)>()); +} + +} // namespace + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/breaking_news/subscription_json_request.cc b/chromium/components/ntp_snippets/breaking_news/subscription_json_request.cc new file mode 100644 index 00000000000..187e4638852 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/subscription_json_request.cc @@ -0,0 +1,182 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/breaking_news/subscription_json_request.h" + +#include "base/json/json_writer.h" +#include "base/memory/ptr_util.h" +#include "base/strings/stringprintf.h" +#include "base/values.h" +#include "components/data_use_measurement/core/data_use_user_data.h" +#include "components/variations/net/variations_http_headers.h" +#include "net/base/load_flags.h" +#include "net/http/http_status_code.h" +#include "net/url_request/url_fetcher.h" +#include "net/url_request/url_request_context_getter.h" + +using net::URLFetcher; +using net::URLRequestContextGetter; +using net::HttpRequestHeaders; +using net::URLRequestStatus; + +namespace ntp_snippets { + +namespace internal { + +SubscriptionJsonRequest::SubscriptionJsonRequest() = default; + +SubscriptionJsonRequest::~SubscriptionJsonRequest() = default; + +void SubscriptionJsonRequest::Start(CompletedCallback callback) { + DCHECK(request_completed_callback_.is_null()) << "Request already running!"; + request_completed_callback_ = std::move(callback); + url_fetcher_->Start(); +} + +//////////////////////////////////////////////////////////////////////////////// +// URLFetcherDelegate overrides +void SubscriptionJsonRequest::OnURLFetchComplete(const URLFetcher* source) { + DCHECK_EQ(url_fetcher_.get(), source); + const URLRequestStatus& status = url_fetcher_->GetStatus(); + int response = url_fetcher_->GetResponseCode(); + + if (!status.is_success()) { + std::move(request_completed_callback_) + .Run(Status(StatusCode::TEMPORARY_ERROR, + base::StringPrintf("Network Error: %d", status.error()))); + } else if (response != net::HTTP_OK) { + std::move(request_completed_callback_) + .Run(Status(StatusCode::PERMANENT_ERROR, + base::StringPrintf("HTTP Error: %d", response))); + } else { + std::move(request_completed_callback_) + .Run(Status(StatusCode::SUCCESS, std::string())); + } +} + +SubscriptionJsonRequest::Builder::Builder() = default; +SubscriptionJsonRequest::Builder::Builder(SubscriptionJsonRequest::Builder&&) = + default; +SubscriptionJsonRequest::Builder::~Builder() = default; + +std::unique_ptr<SubscriptionJsonRequest> +SubscriptionJsonRequest::Builder::Build() const { + DCHECK(!url_.is_empty()); + DCHECK(url_request_context_getter_); + auto request = base::WrapUnique(new SubscriptionJsonRequest()); + + std::string body = BuildBody(); + std::string headers = BuildHeaders(); + request->url_fetcher_ = BuildURLFetcher(request.get(), headers, body); + + // Log the request for debugging network issues. + DVLOG(1) << "Building a subscription request to " << url_ << ":\n" + << headers << "\n" + << body; + + return request; +} + +SubscriptionJsonRequest::Builder& SubscriptionJsonRequest::Builder::SetToken( + const std::string& token) { + token_ = token; + return *this; +} + +SubscriptionJsonRequest::Builder& SubscriptionJsonRequest::Builder::SetUrl( + const GURL& url) { + url_ = url; + return *this; +} + +SubscriptionJsonRequest::Builder& +SubscriptionJsonRequest::Builder::SetUrlRequestContextGetter( + const scoped_refptr<URLRequestContextGetter>& context_getter) { + url_request_context_getter_ = context_getter; + return *this; +} + +SubscriptionJsonRequest::Builder& +SubscriptionJsonRequest::Builder::SetAuthenticationHeader( + const std::string& auth_header) { + auth_header_ = auth_header; + return *this; +} + +std::string SubscriptionJsonRequest::Builder::BuildHeaders() const { + HttpRequestHeaders headers; + headers.SetHeader(HttpRequestHeaders::kContentType, + "application/json; charset=UTF-8"); + if (!auth_header_.empty()) { + headers.SetHeader(HttpRequestHeaders::kAuthorization, auth_header_); + } + // Add X-Client-Data header with experiment IDs from field trials. + // Note: It's OK to pass |is_signed_in| false if it's unknown, as it does + // not affect transmission of experiments coming from the variations server. + variations::AppendVariationHeaders(url_, + false, // incognito + false, // uma_enabled + false, // is_signed_in + &headers); + return headers.ToString(); +} + +std::string SubscriptionJsonRequest::Builder::BuildBody() const { + base::DictionaryValue request; + request.SetString("token", token_); + + std::string request_json; + bool success = base::JSONWriter::Write(request, &request_json); + DCHECK(success); + return request_json; +} + +std::unique_ptr<URLFetcher> SubscriptionJsonRequest::Builder::BuildURLFetcher( + URLFetcherDelegate* delegate, + const std::string& headers, + const std::string& body) const { + net::NetworkTrafficAnnotationTag traffic_annotation = + net::DefineNetworkTrafficAnnotation("gcm_subscription", R"( + semantics { + sender: "Subscribe for breaking news delivered via GCM push messages" + description: + "Chromium can receive breaking news via GCM push messages. " + "This request suscribes the client to receiving them." + trigger: + "Subscription takes place only once per profile lifetime. " + data: + "The subscription token that identifies this Chromium profile." + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: false + setting: + "This feature cannot be disabled by settings now" + chrome_policy { + NTPContentSuggestionsEnabled { + policy_options {mode: MANDATORY} + NTPContentSuggestionsEnabled: false + } + } + })"); + std::unique_ptr<URLFetcher> url_fetcher = + URLFetcher::Create(url_, URLFetcher::POST, delegate, traffic_annotation); + url_fetcher->SetRequestContext(url_request_context_getter_.get()); + url_fetcher->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES | + net::LOAD_DO_NOT_SAVE_COOKIES); + data_use_measurement::DataUseUserData::AttachToFetcher( + url_fetcher.get(), + data_use_measurement::DataUseUserData::NTP_SNIPPETS_SUGGESTIONS); + + url_fetcher->SetExtraRequestHeaders(headers); + url_fetcher->SetUploadData("application/json", body); + + // Fetchers are sometimes cancelled because a network change was detected. + url_fetcher->SetAutomaticallyRetryOnNetworkChanges(1); + return url_fetcher; +} + +} // namespace internal + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/breaking_news/subscription_json_request.h b/chromium/components/ntp_snippets/breaking_news/subscription_json_request.h new file mode 100644 index 00000000000..b0aa71c4bad --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/subscription_json_request.h @@ -0,0 +1,93 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_SUBSCRIPTION_JSON_REQUEST_H_ +#define COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_SUBSCRIPTION_JSON_REQUEST_H_ + +#include <memory> +#include <string> +#include <utility> + +#include "base/callback.h" +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "base/time/time.h" +#include "components/ntp_snippets/status.h" +#include "google_apis/gaia/oauth2_token_service.h" +#include "net/http/http_request_headers.h" + +namespace ntp_snippets { + +namespace internal { + +// A single request to subscribe for breaking news via GCM. The Request has to +// stay alive in order to be successfully completed. +class SubscriptionJsonRequest : public net::URLFetcherDelegate { + public: + // A client can expect a message in the status only, if there was any error + // during the subscription. In successful cases, it will be an empty string. + using CompletedCallback = base::OnceCallback<void(const Status& status)>; + + // Builds non-authenticated and authenticated SubscriptionJsonRequests. + class Builder { + public: + Builder(); + Builder(Builder&&); + ~Builder(); + + // Builds a Request object that contains all data to fetch new snippets. + std::unique_ptr<SubscriptionJsonRequest> Build() const; + + Builder& SetToken(const std::string& token); + Builder& SetUrl(const GURL& url); + Builder& SetUrlRequestContextGetter( + const scoped_refptr<net::URLRequestContextGetter>& context_getter); + Builder& SetAuthenticationHeader(const std::string& auth_header); + + private: + std::string BuildHeaders() const; + std::string BuildBody() const; + std::unique_ptr<net::URLFetcher> BuildURLFetcher( + net::URLFetcherDelegate* request, + const std::string& headers, + const std::string& body) const; + + // GCM subscription token obtained from GCM driver (instanceID::getToken()). + std::string token_; + // TODO(mamir): Additional fields to be added: country, language. + + GURL url_; + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_; + std::string auth_header_; + + DISALLOW_COPY_AND_ASSIGN(Builder); + }; + + ~SubscriptionJsonRequest() override; + + // Starts an async request. The callback is invoked when the request succeeds + // or fails. The callback is not called if the request is destroyed. + void Start(CompletedCallback callback); + + private: + friend class Builder; + SubscriptionJsonRequest(); + // URLFetcherDelegate implementation. + void OnURLFetchComplete(const net::URLFetcher* source) override; + + // The fetcher for subscribing. + std::unique_ptr<net::URLFetcher> url_fetcher_; + + // The callback to notify when URLFetcher finished and results are available. + // When the request is finished/aborted/destroyed, it's called in the dtor! + CompletedCallback request_completed_callback_; + + DISALLOW_COPY_AND_ASSIGN(SubscriptionJsonRequest); +}; + +} // namespace internal + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_SUBSCRIPTION_JSON_REQUEST_H_ diff --git a/chromium/components/ntp_snippets/breaking_news/subscription_json_request_unittest.cc b/chromium/components/ntp_snippets/breaking_news/subscription_json_request_unittest.cc new file mode 100644 index 00000000000..0f09e8bd204 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/subscription_json_request_unittest.cc @@ -0,0 +1,190 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/breaking_news/subscription_json_request.h" + +#include "base/json/json_reader.h" +#include "base/memory/ptr_util.h" +#include "base/message_loop/message_loop.h" +#include "base/test/gtest_util.h" +#include "base/test/mock_callback.h" +#include "base/values.h" +#include "net/url_request/test_url_fetcher_factory.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace ntp_snippets { + +namespace internal { + +namespace { + +using testing::_; +using testing::SaveArg; + +// TODO(mamir): Create a test_helper.cc file instead of duplicating all this +// code. +MATCHER_P(EqualsJSON, json, "equals JSON") { + std::unique_ptr<base::Value> expected = base::JSONReader::Read(json); + if (!expected) { + *result_listener << "INTERNAL ERROR: couldn't parse expected JSON"; + return false; + } + + std::string err_msg; + int err_line, err_col; + std::unique_ptr<base::Value> actual = base::JSONReader::ReadAndReturnError( + arg, base::JSON_PARSE_RFC, nullptr, &err_msg, &err_line, &err_col); + if (!actual) { + *result_listener << "input:" << err_line << ":" << err_col << ": " + << "parse error: " << err_msg; + return false; + } + return base::Value::Equals(actual.get(), expected.get()); +} + +} // namespace + +class SubscriptionJsonRequestTest : public testing::Test { + public: + SubscriptionJsonRequestTest() + : request_context_getter_( + new net::TestURLRequestContextGetter(message_loop_.task_runner())) { + } + + scoped_refptr<net::URLRequestContextGetter> GetRequestContext() { + return request_context_getter_.get(); + } + + net::TestURLFetcher* GetRunningFetcher() { + // All created TestURLFetchers have ID 0 by default. + net::TestURLFetcher* url_fetcher = url_fetcher_factory_.GetFetcherByID(0); + DCHECK(url_fetcher); + return url_fetcher; + } + + void RespondWithData(const std::string& data) { + net::TestURLFetcher* url_fetcher = GetRunningFetcher(); + url_fetcher->set_status(net::URLRequestStatus()); + url_fetcher->set_response_code(net::HTTP_OK); + url_fetcher->SetResponseString(data); + // Call the URLFetcher delegate to continue the test. + url_fetcher->delegate()->OnURLFetchComplete(url_fetcher); + } + + void RespondWithError(int error_code) { + net::TestURLFetcher* url_fetcher = GetRunningFetcher(); + url_fetcher->set_status(net::URLRequestStatus::FromError(error_code)); + url_fetcher->SetResponseString(std::string()); + // Call the URLFetcher delegate to continue the test. + url_fetcher->delegate()->OnURLFetchComplete(url_fetcher); + } + + private: + base::MessageLoop message_loop_; + scoped_refptr<net::TestURLRequestContextGetter> request_context_getter_; + net::TestURLFetcherFactory url_fetcher_factory_; + + DISALLOW_COPY_AND_ASSIGN(SubscriptionJsonRequestTest); +}; + +TEST_F(SubscriptionJsonRequestTest, BuildRequest) { + std::string token = "1234567890"; + GURL url("http://valid-url.test"); + + base::MockCallback<SubscriptionJsonRequest::CompletedCallback> callback; + + SubscriptionJsonRequest::Builder builder; + std::unique_ptr<SubscriptionJsonRequest> request = + builder.SetToken(token) + .SetUrl(url) + .SetUrlRequestContextGetter(GetRequestContext()) + .Build(); + request->Start(callback.Get()); + + net::TestURLFetcher* url_fetcher = GetRunningFetcher(); + + EXPECT_EQ(url, url_fetcher->GetOriginalURL()); + + net::HttpRequestHeaders headers; + url_fetcher->GetExtraRequestHeaders(&headers); + + std::string header; + EXPECT_FALSE(headers.GetHeader("Authorization", &header)); + EXPECT_TRUE(headers.GetHeader("Content-Type", &header)); + EXPECT_EQ(header, "application/json; charset=UTF-8"); + + std::string expected_body = + "{" + " \"token\": " + " \"1234567890\"" + "}"; + EXPECT_THAT(url_fetcher->upload_data(), EqualsJSON(expected_body)); +} + +TEST_F(SubscriptionJsonRequestTest, ShouldNotInvokeCallbackWhenCancelled) { + std::string token = "1234567890"; + GURL url("http://valid-url.test"); + + base::MockCallback<SubscriptionJsonRequest::CompletedCallback> callback; + EXPECT_CALL(callback, Run(_)).Times(0); + + SubscriptionJsonRequest::Builder builder; + std::unique_ptr<SubscriptionJsonRequest> request = + builder.SetToken(token) + .SetUrl(url) + .SetUrlRequestContextGetter(GetRequestContext()) + .Build(); + request->Start(callback.Get()); + + // Destroy the request before getting any response. + request.reset(); +} + +TEST_F(SubscriptionJsonRequestTest, SubscribeWithoutErrors) { + std::string token = "1234567890"; + GURL url("http://valid-url.test"); + + base::MockCallback<SubscriptionJsonRequest::CompletedCallback> callback; + ntp_snippets::Status status(StatusCode::PERMANENT_ERROR, "initial"); + EXPECT_CALL(callback, Run(_)).WillOnce(SaveArg<0>(&status)); + + SubscriptionJsonRequest::Builder builder; + std::unique_ptr<SubscriptionJsonRequest> request = + builder.SetToken(token) + .SetUrl(url) + .SetUrlRequestContextGetter(GetRequestContext()) + .Build(); + request->Start(callback.Get()); + + RespondWithData("{}"); + + EXPECT_EQ(status.code, StatusCode::SUCCESS); +} + +TEST_F(SubscriptionJsonRequestTest, SubscribeWithErrors) { + std::string token = "1234567890"; + GURL url("http://valid-url.test"); + + base::MockCallback<SubscriptionJsonRequest::CompletedCallback> callback; + ntp_snippets::Status status(StatusCode::SUCCESS, "initial"); + EXPECT_CALL(callback, Run(_)).WillOnce(SaveArg<0>(&status)); + + SubscriptionJsonRequest::Builder builder; + std::unique_ptr<SubscriptionJsonRequest> request = + builder.SetToken(token) + .SetUrl(url) + .SetUrlRequestContextGetter(GetRequestContext()) + .Build(); + request->Start(callback.Get()); + + RespondWithError(net::ERR_TIMED_OUT); + + EXPECT_EQ(status.code, StatusCode::TEMPORARY_ERROR); +} + +} // namespace internal + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/breaking_news/subscription_manager.cc b/chromium/components/ntp_snippets/breaking_news/subscription_manager.cc new file mode 100644 index 00000000000..1dbd535dbe8 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/subscription_manager.cc @@ -0,0 +1,65 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/breaking_news/subscription_manager.h" + +#include "base/metrics/field_trial_params.h" +#include "components/ntp_snippets/features.h" +#include "components/ntp_snippets/ntp_snippets_constants.h" + +namespace ntp_snippets { + +namespace { + +// Variation parameter for chrome-push-subscription backend. +const char kPushSubscriptionBackendParam[] = "push_subscription_backend"; + +// Variation parameter for chrome-push-unsubscription backend. +const char kPushUnsubscriptionBackendParam[] = "push_unsubscription_backend"; + +} // namespace + +GURL GetPushUpdatesSubscriptionEndpoint(version_info::Channel channel) { + std::string endpoint = base::GetFieldTrialParamValueByFeature( + kBreakingNewsPushFeature, kPushSubscriptionBackendParam); + if (!endpoint.empty()) { + return GURL{endpoint}; + } + + switch (channel) { + case version_info::Channel::STABLE: + case version_info::Channel::BETA: + return GURL{kPushUpdatesSubscriptionServer}; + + case version_info::Channel::DEV: + case version_info::Channel::CANARY: + case version_info::Channel::UNKNOWN: + return GURL{kPushUpdatesSubscriptionStagingServer}; + } + NOTREACHED(); + return GURL{kPushUpdatesSubscriptionStagingServer}; +} + +GURL GetPushUpdatesUnsubscriptionEndpoint(version_info::Channel channel) { + std::string endpoint = base::GetFieldTrialParamValueByFeature( + kBreakingNewsPushFeature, kPushUnsubscriptionBackendParam); + if (!endpoint.empty()) { + return GURL{endpoint}; + } + + switch (channel) { + case version_info::Channel::STABLE: + case version_info::Channel::BETA: + return GURL{kPushUpdatesUnsubscriptionServer}; + + case version_info::Channel::DEV: + case version_info::Channel::CANARY: + case version_info::Channel::UNKNOWN: + return GURL{kPushUpdatesUnsubscriptionStagingServer}; + } + NOTREACHED(); + return GURL{kPushUpdatesUnsubscriptionStagingServer}; +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/breaking_news/subscription_manager.h b/chromium/components/ntp_snippets/breaking_news/subscription_manager.h new file mode 100644 index 00000000000..fc2343e04eb --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/subscription_manager.h @@ -0,0 +1,42 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_SUBSCRIPTION_MANAGER_H_ +#define COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_SUBSCRIPTION_MANAGER_H_ + +#include <string> + +#include "components/version_info/version_info.h" +#include "url/gurl.h" + +namespace ntp_snippets { + +// Returns the appropriate API endpoint for subscribing for push updates, in +// consideration of the channel and field trial parameters. +GURL GetPushUpdatesSubscriptionEndpoint(version_info::Channel channel); + +// Returns the appropriate API endpoint for unsubscribing for push updates, in +// consideration of the channel and field trial parameters. +GURL GetPushUpdatesUnsubscriptionEndpoint(version_info::Channel channel); + +// Handles subscription to content suggestions server for push updates (e.g. via +// GCM). +class SubscriptionManager { + public: + virtual ~SubscriptionManager() = default; + + virtual void Subscribe(const std::string& token) = 0; + virtual void Unsubscribe() = 0; + virtual bool IsSubscribed() = 0; + + virtual void Resubscribe(const std::string& new_token) = 0; + + // Checks if some data that has been used when subscribing has changed. For + // example, the user has signed in. + virtual bool NeedsToResubscribe() = 0; +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_SUBSCRIPTION_MANAGER_H_ diff --git a/chromium/components/ntp_snippets/breaking_news/subscription_manager_impl.cc b/chromium/components/ntp_snippets/breaking_news/subscription_manager_impl.cc new file mode 100644 index 00000000000..22fd71ae6c9 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/subscription_manager_impl.cc @@ -0,0 +1,270 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/breaking_news/subscription_manager_impl.h" + +#include "base/bind.h" +#include "base/memory/ptr_util.h" +#include "base/metrics/field_trial_params.h" +#include "base/strings/stringprintf.h" +#include "components/ntp_snippets/breaking_news/subscription_json_request.h" +#include "components/ntp_snippets/features.h" +#include "components/ntp_snippets/ntp_snippets_constants.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" +#include "components/signin/core/browser/access_token_fetcher.h" +#include "components/signin/core/browser/signin_manager_base.h" +#include "net/base/url_util.h" + +namespace ntp_snippets { + +using internal::SubscriptionJsonRequest; + +namespace { + +const char kApiKeyParamName[] = "key"; +const char kAuthorizationRequestHeaderFormat[] = "Bearer %s"; + +} // namespace + +class SubscriptionManagerImpl::SigninObserver + : public SigninManagerBase::Observer { + public: + SigninObserver(SigninManagerBase* signin_manager, + const base::Closure& signin_status_changed_callback) + : signin_manager_(signin_manager), + signin_status_changed_callback_(signin_status_changed_callback) { + signin_manager_->AddObserver(this); + } + + ~SigninObserver() override { signin_manager_->RemoveObserver(this); } + + private: + // SigninManagerBase::Observer implementation. + void GoogleSigninSucceeded(const std::string& account_id, + const std::string& username) override { + signin_status_changed_callback_.Run(); + } + + void GoogleSignedOut(const std::string& account_id, + const std::string& username) override { + signin_status_changed_callback_.Run(); + } + + SigninManagerBase* const signin_manager_; + base::Closure signin_status_changed_callback_; +}; + +SubscriptionManagerImpl::SubscriptionManagerImpl( + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter, + PrefService* pref_service, + SigninManagerBase* signin_manager, + OAuth2TokenService* access_token_service, + const std::string& api_key, + const GURL& subscribe_url, + const GURL& unsubscribe_url) + : url_request_context_getter_(std::move(url_request_context_getter)), + pref_service_(pref_service), + signin_manager_(signin_manager), + signin_observer_(base::MakeUnique<SigninObserver>( + signin_manager, + base::Bind(&SubscriptionManagerImpl::SigninStatusChanged, + base::Unretained(this)))), + access_token_service_(access_token_service), + api_key_(api_key), + subscribe_url_(subscribe_url), + unsubscribe_url_(unsubscribe_url) {} + +SubscriptionManagerImpl::~SubscriptionManagerImpl() = default; + +void SubscriptionManagerImpl::Subscribe(const std::string& subscription_token) { + // If there is a request in flight, cancel it. + if (request_) { + request_ = nullptr; + } + if (signin_manager_->IsAuthenticated()) { + StartAccessTokenRequest(subscription_token); + } else { + SubscribeInternal(subscription_token, /*access_token=*/std::string()); + } +} + +void SubscriptionManagerImpl::SubscribeInternal( + const std::string& subscription_token, + const std::string& access_token) { + SubscriptionJsonRequest::Builder builder; + builder.SetToken(subscription_token) + .SetUrlRequestContextGetter(url_request_context_getter_); + + if (!access_token.empty()) { + builder.SetUrl(subscribe_url_); + builder.SetAuthenticationHeader(base::StringPrintf( + kAuthorizationRequestHeaderFormat, access_token.c_str())); + } else { + // When not providing OAuth token, we need to pass the Google API key. + builder.SetUrl( + net::AppendQueryParameter(subscribe_url_, kApiKeyParamName, api_key_)); + } + + request_ = builder.Build(); + request_->Start(base::BindOnce(&SubscriptionManagerImpl::DidSubscribe, + base::Unretained(this), subscription_token, + /*is_authenticated=*/!access_token.empty())); +} + +void SubscriptionManagerImpl::StartAccessTokenRequest( + const std::string& subscription_token) { + // If there is already an ongoing token request, destroy it. + if (access_token_fetcher_) { + access_token_fetcher_ = nullptr; + } + + OAuth2TokenService::ScopeSet scopes = {kContentSuggestionsApiScope}; + access_token_fetcher_ = base::MakeUnique<AccessTokenFetcher>( + "ntp_snippets", signin_manager_, access_token_service_, scopes, + base::BindOnce(&SubscriptionManagerImpl::AccessTokenFetchFinished, + base::Unretained(this), subscription_token)); +} + +void SubscriptionManagerImpl::AccessTokenFetchFinished( + const std::string& subscription_token, + const GoogleServiceAuthError& error, + const std::string& access_token) { + // Delete the fetcher only after we leave this method (which is called from + // the fetcher itself). + std::unique_ptr<AccessTokenFetcher> access_token_fetcher_deleter( + std::move(access_token_fetcher_)); + + if (error.state() != GoogleServiceAuthError::NONE) { + // In case of error, we will retry on next Chrome restart. + return; + } + DCHECK(!access_token.empty()); + SubscribeInternal(subscription_token, access_token); +} + +void SubscriptionManagerImpl::DidSubscribe( + const std::string& subscription_token, + bool is_authenticated, + const Status& status) { + // Delete the request only after we leave this method (which is called from + // the request itself). + std::unique_ptr<internal::SubscriptionJsonRequest> request_deleter( + std::move(request_)); + + switch (status.code) { + case StatusCode::SUCCESS: + // In case of successful subscription, store the same data used for + // subscription in order to be able to resubscribe in case of data + // change. + // TODO(mamir): Store region and language. + pref_service_->SetString(prefs::kBreakingNewsSubscriptionDataToken, + subscription_token); + pref_service_->SetBoolean( + prefs::kBreakingNewsSubscriptionDataIsAuthenticated, + is_authenticated); + break; + default: + // TODO(mamir): Handle failure. + break; + } +} + +void SubscriptionManagerImpl::Unsubscribe() { + std::string token = + pref_service_->GetString(prefs::kBreakingNewsSubscriptionDataToken); + ResubscribeInternal(/*old_token=*/token, /*new_token=*/std::string()); +} + +void SubscriptionManagerImpl::ResubscribeInternal( + const std::string& old_token, + const std::string& new_token) { + // If there is an request in flight, cancel it. + if (request_) { + request_ = nullptr; + } + + SubscriptionJsonRequest::Builder builder; + builder.SetToken(old_token).SetUrlRequestContextGetter( + url_request_context_getter_); + builder.SetUrl( + net::AppendQueryParameter(unsubscribe_url_, kApiKeyParamName, api_key_)); + + request_ = builder.Build(); + request_->Start(base::BindOnce(&SubscriptionManagerImpl::DidUnsubscribe, + base::Unretained(this), new_token)); +} + +bool SubscriptionManagerImpl::IsSubscribed() { + std::string subscription_token = + pref_service_->GetString(prefs::kBreakingNewsSubscriptionDataToken); + return !subscription_token.empty(); +} + +bool SubscriptionManagerImpl::NeedsToResubscribe() { + // Check if authentication state changed after subscription. + bool is_auth_subscribe = pref_service_->GetBoolean( + prefs::kBreakingNewsSubscriptionDataIsAuthenticated); + bool is_authenticated = signin_manager_->IsAuthenticated(); + return is_auth_subscribe != is_authenticated; +} + +void SubscriptionManagerImpl::Resubscribe(const std::string& new_token) { + std::string old_token = + pref_service_->GetString(prefs::kBreakingNewsSubscriptionDataToken); + if (old_token == new_token) { + // If the token didn't change, subscribe directly. The server handles the + // unsubscription if previous subscriptions exists. + Subscribe(old_token); + } else { + ResubscribeInternal(old_token, new_token); + } +} + +void SubscriptionManagerImpl::DidUnsubscribe(const std::string& new_token, + const Status& status) { + // Delete the request only after we leave this method (which is called from + // the request itself). + std::unique_ptr<internal::SubscriptionJsonRequest> request_deleter( + std::move(request_)); + + switch (status.code) { + case StatusCode::SUCCESS: + // In case of successful unsubscription, clear the previously stored data. + // TODO(mamir): Clear stored region and language. + pref_service_->ClearPref(prefs::kBreakingNewsSubscriptionDataToken); + pref_service_->ClearPref( + prefs::kBreakingNewsSubscriptionDataIsAuthenticated); + if (!new_token.empty()) { + Subscribe(new_token); + } + break; + default: + // TODO(mamir): Handle failure. + break; + } +} + +void SubscriptionManagerImpl::SigninStatusChanged() { + // If subscribed already, resubscribe. + if (IsSubscribed()) { + if (request_) { + request_ = nullptr; + } + std::string token = + pref_service_->GetString(prefs::kBreakingNewsSubscriptionDataToken); + Subscribe(token); + } +} + +void SubscriptionManagerImpl::RegisterProfilePrefs( + PrefRegistrySimple* registry) { + registry->RegisterStringPref(prefs::kBreakingNewsSubscriptionDataToken, + std::string()); + registry->RegisterBooleanPref( + prefs::kBreakingNewsSubscriptionDataIsAuthenticated, false); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/breaking_news/subscription_manager_impl.h b/chromium/components/ntp_snippets/breaking_news/subscription_manager_impl.h new file mode 100644 index 00000000000..b1fd05bda06 --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/subscription_manager_impl.h @@ -0,0 +1,105 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_SUBSCRIPTION_MANAGER_IMPL_H_ +#define COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_SUBSCRIPTION_MANAGER_IMPL_H_ + +#include <memory> +#include <string> + +#include "base/memory/ref_counted.h" +#include "components/ntp_snippets/breaking_news/subscription_json_request.h" +#include "components/ntp_snippets/breaking_news/subscription_manager.h" +#include "components/signin/core/browser/signin_manager_base.h" +#include "net/url_request/url_request_context_getter.h" +#include "url/gurl.h" + +class AccessTokenFetcher; +class OAuth2TokenService; +class PrefRegistrySimple; +class PrefService; + +namespace ntp_snippets { + +// Class that wraps around the functionality of SubscriptionJsonRequest. It uses +// the SubscriptionJsonRequest to send subscription and unsubscription requests +// to the content suggestions server and does the bookkeeping for the data used +// for subscription. Bookkeeping is required to detect any change (e.g. the +// token render invalid), and resubscribe accordingly. +class SubscriptionManagerImpl : public SubscriptionManager { + public: + SubscriptionManagerImpl( + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter, + PrefService* pref_service, + SigninManagerBase* signin_manager, + OAuth2TokenService* access_token_service, + const std::string& api_key, + const GURL& subscribe_url, + const GURL& unsubscribe_url); + + ~SubscriptionManagerImpl() override; + + // SubscriptionManager implementation. + void Subscribe(const std::string& token) override; + void Unsubscribe() override; + bool IsSubscribed() override; + + void Resubscribe(const std::string& new_token) override; + + // Checks if some data that has been used when subscribing has changed. For + // example, the user has signed in. + bool NeedsToResubscribe() override; + + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + private: + class SigninObserver; + + void SigninStatusChanged(); + + void DidSubscribe(const std::string& subscription_token, + bool is_authenticated, + const Status& status); + void DidUnsubscribe(const std::string& new_token, const Status& status); + void SubscribeInternal(const std::string& subscription_token, + const std::string& access_token); + + // If |new_token| is empty, this will just unsubscribe. If |new_token| is + // non-empty, a subscription request with the |new_token| will be started upon + // successful unsubscription. + void ResubscribeInternal(const std::string& old_token, + const std::string& new_token); + + // |subscription_token| is the token when subscribing after obtaining the + // access token. + void StartAccessTokenRequest(const std::string& subscription_token); + void AccessTokenFetchFinished(const std::string& subscription_token, + const GoogleServiceAuthError& error, + const std::string& access_token); + + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_; + + std::unique_ptr<internal::SubscriptionJsonRequest> request_; + std::unique_ptr<AccessTokenFetcher> access_token_fetcher_; + + PrefService* pref_service_; + + // Authentication for signed-in users. + SigninManagerBase* signin_manager_; + std::unique_ptr<SigninObserver> signin_observer_; + OAuth2TokenService* access_token_service_; + + // API key to use for non-authenticated requests. + const std::string api_key_; + + // API endpoint for subscribing and unsubscribing. + const GURL subscribe_url_; + const GURL unsubscribe_url_; + + DISALLOW_COPY_AND_ASSIGN(SubscriptionManagerImpl); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_BREAKING_NEWS_SUBSCRIPTION_MANAGER_IMPL_H_ diff --git a/chromium/components/ntp_snippets/breaking_news/subscription_manager_impl_unittest.cc b/chromium/components/ntp_snippets/breaking_news/subscription_manager_impl_unittest.cc new file mode 100644 index 00000000000..2c8cebdb57f --- /dev/null +++ b/chromium/components/ntp_snippets/breaking_news/subscription_manager_impl_unittest.cc @@ -0,0 +1,334 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/breaking_news/subscription_manager_impl.h" + +#include "base/message_loop/message_loop.h" +#include "build/build_config.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/ntp_snippets/remote/test_utils.h" +#include "components/prefs/testing_pref_service.h" +#include "components/signin/core/browser/fake_profile_oauth2_token_service.h" +#include "components/signin/core/browser/fake_signin_manager.h" +#include "components/signin/core/browser/test_signin_client.h" +#include "google_apis/gaia/fake_oauth2_token_service_delegate.h" +#include "net/url_request/test_url_fetcher_factory.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace ntp_snippets { + +const char kTestEmail[] = "test@email.com"; +const char kAPIKey[] = "fakeAPIkey"; +const char kSubscriptionUrl[] = "http://valid-url.test/subscribe"; +const char kSubscriptionUrlSignedIn[] = "http://valid-url.test/subscribe"; +; +const char kSubscriptionUrlSignedOut[] = + "http://valid-url.test/subscribe?key=fakeAPIkey"; +const char kUnsubscriptionUrl[] = "http://valid-url.test/unsubscribe"; +const char kUnsubscriptionUrlSignedIn[] = "http://valid-url.test/unsubscribe"; +const char kUnsubscriptionUrlSignedOut[] = + "http://valid-url.test/unsubscribe?key=fakeAPIkey"; + +class SubscriptionManagerImplTest : public testing::Test { + public: + SubscriptionManagerImplTest() + : request_context_getter_( + new net::TestURLRequestContextGetter(message_loop_.task_runner())) { + } + + void SetUp() override { + SubscriptionManagerImpl::RegisterProfilePrefs( + utils_.pref_service()->registry()); + } + + scoped_refptr<net::URLRequestContextGetter> GetRequestContext() { + return request_context_getter_.get(); + } + + PrefService* GetPrefService() { return utils_.pref_service(); } + + FakeProfileOAuth2TokenService* GetOAuth2TokenService() { + return utils_.token_service(); + } + + SigninManagerBase* GetSigninManager() { return utils_.fake_signin_manager(); } + + net::TestURLFetcher* GetRunningFetcher() { + // All created TestURLFetchers have ID 0 by default. + net::TestURLFetcher* url_fetcher = url_fetcher_factory_.GetFetcherByID(0); + DCHECK(url_fetcher); + return url_fetcher; + } + + void RespondToSubscriptionRequestSuccessfully(bool is_signed_in) { + net::TestURLFetcher* url_fetcher = GetRunningFetcher(); + if (is_signed_in) { + ASSERT_EQ(GURL(kSubscriptionUrlSignedIn), url_fetcher->GetOriginalURL()); + } else { + ASSERT_EQ(GURL(kSubscriptionUrlSignedOut), url_fetcher->GetOriginalURL()); + } + RespondSuccessfully(); + } + + void RespondToUnsubscriptionRequestSuccessfully(bool is_signed_in) { + net::TestURLFetcher* url_fetcher = GetRunningFetcher(); + if (is_signed_in) { + ASSERT_EQ(GURL(kUnsubscriptionUrlSignedIn), + url_fetcher->GetOriginalURL()); + } else { + ASSERT_EQ(GURL(kUnsubscriptionUrlSignedOut), + url_fetcher->GetOriginalURL()); + } + RespondSuccessfully(); + } + + void RespondToSubscriptionWithError(bool is_signed_in, int error_code) { + net::TestURLFetcher* url_fetcher = GetRunningFetcher(); + if (is_signed_in) { + ASSERT_EQ(GURL(kSubscriptionUrlSignedIn), url_fetcher->GetOriginalURL()); + } else { + ASSERT_EQ(GURL(kSubscriptionUrlSignedOut), url_fetcher->GetOriginalURL()); + } + RespondWithError(error_code); + } + + void RespondToUnsubscriptionWithError(bool is_signed_in, int error_code) { + net::TestURLFetcher* url_fetcher = GetRunningFetcher(); + if (is_signed_in) { + ASSERT_EQ(GURL(kUnsubscriptionUrlSignedIn), + url_fetcher->GetOriginalURL()); + } else { + ASSERT_EQ(GURL(kUnsubscriptionUrlSignedOut), + url_fetcher->GetOriginalURL()); + } + RespondWithError(error_code); + } + +#if !defined(OS_CHROMEOS) + void SignIn() { + utils_.fake_signin_manager()->SignIn(kTestEmail, "user", "pass"); + } + + void SignOut() { utils_.fake_signin_manager()->ForceSignOut(); } +#endif // !defined(OS_CHROMEOS) + + void IssueRefreshToken(FakeProfileOAuth2TokenService* auth_token_service) { + auth_token_service->GetDelegate()->UpdateCredentials(kTestEmail, "token"); + } + + void IssueAccessToken(FakeProfileOAuth2TokenService* auth_token_service) { + auth_token_service->IssueAllTokensForAccount(kTestEmail, "access_token", + base::Time::Max()); + } + + private: + void RespondSuccessfully() { + net::TestURLFetcher* url_fetcher = GetRunningFetcher(); + url_fetcher->set_status(net::URLRequestStatus()); + url_fetcher->set_response_code(net::HTTP_OK); + url_fetcher->SetResponseString(std::string()); + // Call the URLFetcher delegate to continue the test. + url_fetcher->delegate()->OnURLFetchComplete(url_fetcher); + } + + void RespondWithError(int error_code) { + net::TestURLFetcher* url_fetcher = GetRunningFetcher(); + url_fetcher->set_status(net::URLRequestStatus::FromError(error_code)); + url_fetcher->SetResponseString(std::string()); + // Call the URLFetcher delegate to continue the test. + url_fetcher->delegate()->OnURLFetchComplete(url_fetcher); + } + + base::MessageLoop message_loop_; + test::RemoteSuggestionsTestUtils utils_; + scoped_refptr<net::TestURLRequestContextGetter> request_context_getter_; + net::TestURLFetcherFactory url_fetcher_factory_; +}; + +TEST_F(SubscriptionManagerImplTest, SubscribeSuccessfully) { + std::string subscription_token = "1234567890"; + SubscriptionManagerImpl manager(GetRequestContext(), GetPrefService(), + GetSigninManager(), GetOAuth2TokenService(), + kAPIKey, GURL(kSubscriptionUrl), + GURL(kUnsubscriptionUrl)); + manager.Subscribe(subscription_token); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/false); + ASSERT_TRUE(manager.IsSubscribed()); + EXPECT_EQ(subscription_token, GetPrefService()->GetString( + prefs::kBreakingNewsSubscriptionDataToken)); + EXPECT_FALSE(GetPrefService()->GetBoolean( + prefs::kBreakingNewsSubscriptionDataIsAuthenticated)); +} + +// This test is relevant only on non-ChromeOS platforms, as the flow being +// tested here is not possible on ChromeOS. +#if !defined(OS_CHROMEOS) +TEST_F(SubscriptionManagerImplTest, + ShouldSubscribeWithAuthenticationWhenAuthenticated) { + // Sign in. + FakeProfileOAuth2TokenService* auth_token_service = GetOAuth2TokenService(); + SignIn(); + IssueRefreshToken(auth_token_service); + + // Create manager and subscribe. + std::string subscription_token = "1234567890"; + SubscriptionManagerImpl manager(GetRequestContext(), GetPrefService(), + GetSigninManager(), auth_token_service, + kAPIKey, GURL(kSubscriptionUrl), + GURL(kUnsubscriptionUrl)); + manager.Subscribe(subscription_token); + + // Make sure that subscription is pending an access token. + ASSERT_FALSE(manager.IsSubscribed()); + ASSERT_EQ(1u, auth_token_service->GetPendingRequests().size()); + + // Issue the access token and respond to the subscription request. + IssueAccessToken(auth_token_service); + ASSERT_FALSE(manager.IsSubscribed()); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/true); + ASSERT_TRUE(manager.IsSubscribed()); + + // Check that we are now subscribed correctly with authentication. + EXPECT_EQ(subscription_token, GetPrefService()->GetString( + prefs::kBreakingNewsSubscriptionDataToken)); + EXPECT_TRUE(GetPrefService()->GetBoolean( + prefs::kBreakingNewsSubscriptionDataIsAuthenticated)); +} +#endif + +TEST_F(SubscriptionManagerImplTest, ShouldNotSubscribeIfError) { + std::string subscription_token = "1234567890"; + SubscriptionManagerImpl manager(GetRequestContext(), GetPrefService(), + GetSigninManager(), GetOAuth2TokenService(), + kAPIKey, GURL(kSubscriptionUrl), + GURL(kUnsubscriptionUrl)); + + manager.Subscribe(subscription_token); + RespondToSubscriptionWithError(/*is_signed_in=*/false, net::ERR_TIMED_OUT); + EXPECT_FALSE(manager.IsSubscribed()); +} + +TEST_F(SubscriptionManagerImplTest, UnsubscribeSuccessfully) { + std::string subscription_token = "1234567890"; + SubscriptionManagerImpl manager(GetRequestContext(), GetPrefService(), + GetSigninManager(), GetOAuth2TokenService(), + kAPIKey, GURL(kSubscriptionUrl), + GURL(kUnsubscriptionUrl)); + manager.Subscribe(subscription_token); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/false); + ASSERT_TRUE(manager.IsSubscribed()); + manager.Unsubscribe(); + RespondToUnsubscriptionRequestSuccessfully(/*is_signed_in=*/false); + EXPECT_FALSE(manager.IsSubscribed()); + EXPECT_FALSE( + GetPrefService()->HasPrefPath(prefs::kBreakingNewsSubscriptionDataToken)); +} + +TEST_F(SubscriptionManagerImplTest, + ShouldRemainSubscribedIfErrorDuringUnsubscribe) { + std::string subscription_token = "1234567890"; + SubscriptionManagerImpl manager(GetRequestContext(), GetPrefService(), + GetSigninManager(), GetOAuth2TokenService(), + kAPIKey, GURL(kSubscriptionUrl), + GURL(kUnsubscriptionUrl)); + manager.Subscribe(subscription_token); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/false); + ASSERT_TRUE(manager.IsSubscribed()); + manager.Unsubscribe(); + RespondToUnsubscriptionWithError(/*is_signed_in=*/false, net::ERR_TIMED_OUT); + ASSERT_TRUE(manager.IsSubscribed()); + EXPECT_EQ(subscription_token, GetPrefService()->GetString( + prefs::kBreakingNewsSubscriptionDataToken)); +} + +// This test is relevant only on non-ChromeOS platforms, as the flow being +// tested here is not possible on ChromeOS. +#if !defined(OS_CHROMEOS) +TEST_F(SubscriptionManagerImplTest, + ShouldResubscribeIfSignInAfterSubscription) { + // Create manager and subscribe. + FakeProfileOAuth2TokenService* auth_token_service = GetOAuth2TokenService(); + std::string subscription_token = "1234567890"; + SubscriptionManagerImpl manager(GetRequestContext(), GetPrefService(), + GetSigninManager(), auth_token_service, + kAPIKey, GURL(kSubscriptionUrl), + GURL(kUnsubscriptionUrl)); + manager.Subscribe(subscription_token); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/false); + ASSERT_FALSE(manager.NeedsToResubscribe()); + + // Sign in. This should trigger a resubscribe. + SignIn(); + IssueRefreshToken(auth_token_service); + ASSERT_TRUE(manager.NeedsToResubscribe()); + ASSERT_EQ(1u, auth_token_service->GetPendingRequests().size()); + IssueAccessToken(auth_token_service); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/true); + + // Check that we are now subscribed with authentication. + EXPECT_TRUE(GetPrefService()->GetBoolean( + prefs::kBreakingNewsSubscriptionDataIsAuthenticated)); +} +#endif + +// This test is relevant only on non-ChromeOS platforms, as the flow being +// tested here is not possible on ChromeOS. +#if !defined(OS_CHROMEOS) +TEST_F(SubscriptionManagerImplTest, + ShouldResubscribeIfSignOutAfterSubscription) { + // Signin and subscribe. + FakeProfileOAuth2TokenService* auth_token_service = GetOAuth2TokenService(); + SignIn(); + IssueRefreshToken(auth_token_service); + std::string subscription_token = "1234567890"; + SubscriptionManagerImpl manager(GetRequestContext(), GetPrefService(), + GetSigninManager(), auth_token_service, + kAPIKey, GURL(kSubscriptionUrl), + GURL(kUnsubscriptionUrl)); + manager.Subscribe(subscription_token); + ASSERT_EQ(1u, auth_token_service->GetPendingRequests().size()); + IssueAccessToken(auth_token_service); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/true); + + // Signout, this should trigger a resubscribe. + SignOut(); + EXPECT_TRUE(manager.NeedsToResubscribe()); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/false); + + // Check that we are now subscribed without authentication. + EXPECT_FALSE(GetPrefService()->GetBoolean( + prefs::kBreakingNewsSubscriptionDataIsAuthenticated)); +} +#endif + +TEST_F(SubscriptionManagerImplTest, + ShouldUpdateTokenInPrefWhenResubscribeWithChangeInToken) { + // Create manager and subscribe. + std::string old_subscription_token = "1234567890"; + SubscriptionManagerImpl manager(GetRequestContext(), GetPrefService(), + GetSigninManager(), GetOAuth2TokenService(), + kAPIKey, GURL(kSubscriptionUrl), + GURL(kUnsubscriptionUrl)); + manager.Subscribe(old_subscription_token); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/false); + EXPECT_EQ( + old_subscription_token, + GetPrefService()->GetString(prefs::kBreakingNewsSubscriptionDataToken)); + + // Resubscribe with a new token. + std::string new_subscription_token = "0987654321"; + manager.Resubscribe(new_subscription_token); + // Resubscribe with a new token should issue an unsubscribe request before + // subscribing. + RespondToUnsubscriptionRequestSuccessfully(/*is_signed_in=*/false); + RespondToSubscriptionRequestSuccessfully(/*is_signed_in=*/false); + + // Check we are now subscribed with the new token. + EXPECT_EQ( + new_subscription_token, + GetPrefService()->GetString(prefs::kBreakingNewsSubscriptionDataToken)); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/category.h b/chromium/components/ntp_snippets/category.h index 97abf753c9b..b63c6af935c 100644 --- a/chromium/components/ntp_snippets/category.h +++ b/chromium/components/ntp_snippets/category.h @@ -51,10 +51,12 @@ enum class KnownCategories { // Articles for you. ARTICLES, + // Breaking News + BREAKING_NEWS = 10008, // ****************** INSERT NEW REMOTE CATEGORIES HERE! ****************** // Tracks the last known remote category - LAST_KNOWN_REMOTE_CATEGORY = ARTICLES, + LAST_KNOWN_REMOTE_CATEGORY = BREAKING_NEWS, }; // A category groups ContentSuggestions which belong together. Use the diff --git a/chromium/components/ntp_snippets/content_suggestion.cc b/chromium/components/ntp_snippets/content_suggestion.cc index a26bfa1e255..c3919f7c740 100644 --- a/chromium/components/ntp_snippets/content_suggestion.cc +++ b/chromium/components/ntp_snippets/content_suggestion.cc @@ -25,12 +25,15 @@ bool ContentSuggestion::ID::operator!=(const ID& rhs) const { } ContentSuggestion::ContentSuggestion(const ID& id, const GURL& url) - : id_(id), url_(url), score_(0) {} + : id_(id), url_(url), score_(0), is_video_suggestion_(false) {} ContentSuggestion::ContentSuggestion(Category category, const std::string& id_within_category, const GURL& url) - : id_(category, id_within_category), url_(url), score_(0) {} + : id_(category, id_within_category), + url_(url), + score_(0), + is_video_suggestion_(false) {} ContentSuggestion::ContentSuggestion(ContentSuggestion&&) = default; diff --git a/chromium/components/ntp_snippets/content_suggestion.h b/chromium/components/ntp_snippets/content_suggestion.h index 1b70c38fbd1..312c710f062 100644 --- a/chromium/components/ntp_snippets/content_suggestion.h +++ b/chromium/components/ntp_snippets/content_suggestion.h @@ -141,6 +141,11 @@ class ContentSuggestion { publisher_name_ = publisher_name; } + bool is_video_suggestion() const { return is_video_suggestion_; } + void set_is_video_suggestion(bool is_video_suggestion) { + is_video_suggestion_ = is_video_suggestion; + } + // TODO(pke): Remove the score from the ContentSuggestion class. The UI only // uses it to track user clicks (histogram data). Instead, the providers // should be informed about clicks and do appropriate logging themselves. @@ -209,6 +214,8 @@ class ContentSuggestion { // RemoteSuggestion. base::Time fetch_date_; + bool is_video_suggestion_; + DISALLOW_COPY_AND_ASSIGN(ContentSuggestion); }; diff --git a/chromium/components/ntp_snippets/content_suggestions_metrics.cc b/chromium/components/ntp_snippets/content_suggestions_metrics.cc index 348a555694e..3be66f2a0eb 100644 --- a/chromium/components/ntp_snippets/content_suggestions_metrics.cc +++ b/chromium/components/ntp_snippets/content_suggestions_metrics.cc @@ -23,8 +23,8 @@ const int kMaxSuggestionsPerCategory = 10; const int kMaxSuggestionsTotal = 50; const int kMaxCategories = 10; -const char kHistogramCountOnNtpOpened[] = - "NewTabPage.ContentSuggestions.CountOnNtpOpened"; +const char kHistogramCountOnNtpOpenedIfVisible[] = + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible"; const char kHistogramSectionCountOnNtpOpened[] = "NewTabPage.ContentSuggestions.SectionCountOnNtpOpened"; const char kHistogramShown[] = "NewTabPage.ContentSuggestions.Shown"; @@ -78,6 +78,7 @@ enum HistogramCategories { FOREIGN_TABS, ARTICLES, READING_LIST, + BREAKING_NEWS, // Insert new values here! COUNT }; @@ -107,6 +108,8 @@ HistogramCategories GetHistogramCategory(Category category) { return HistogramCategories::ARTICLES; case KnownCategories::READING_LIST: return HistogramCategories::READING_LIST; + case KnownCategories::BREAKING_NEWS: + return HistogramCategories::BREAKING_NEWS; case KnownCategories::LOCAL_CATEGORIES_COUNT: case KnownCategories::REMOTE_CATEGORIES_OFFSET: NOTREACHED(); @@ -137,6 +140,8 @@ std::string GetCategorySuffix(Category category) { return "Experimental"; case HistogramCategories::READING_LIST: return "ReadingList"; + case HistogramCategories::BREAKING_NEWS: + return "BreakingNews"; case HistogramCategories::COUNT: NOTREACHED(); break; @@ -213,17 +218,24 @@ void RecordContentSuggestionsUsage() { } // namespace -void OnPageShown( - const std::vector<std::pair<Category, int>>& suggestions_per_category, - int visible_categories_count) { +void OnPageShown(const std::vector<Category>& categories, + const std::vector<int>& suggestions_per_category, + const std::vector<bool>& is_category_visible) { + DCHECK_EQ(categories.size(), suggestions_per_category.size()); + DCHECK_EQ(categories.size(), is_category_visible.size()); int suggestions_total = 0; - for (const std::pair<Category, int>& item : suggestions_per_category) { - LogCategoryHistogramPosition(kHistogramCountOnNtpOpened, item.first, - item.second, kMaxSuggestionsPerCategory); - suggestions_total += item.second; + int visible_categories_count = 0; + for (size_t i = 0; i < categories.size(); ++i) { + if (is_category_visible[i]) { + LogCategoryHistogramPosition(kHistogramCountOnNtpOpenedIfVisible, + categories[i], suggestions_per_category[i], + kMaxSuggestionsPerCategory); + suggestions_total += suggestions_per_category[i]; + ++visible_categories_count; + } } - UMA_HISTOGRAM_EXACT_LINEAR(kHistogramCountOnNtpOpened, suggestions_total, - kMaxSuggestionsTotal); + UMA_HISTOGRAM_EXACT_LINEAR(kHistogramCountOnNtpOpenedIfVisible, + suggestions_total, kMaxSuggestionsTotal); UMA_HISTOGRAM_EXACT_LINEAR(kHistogramSectionCountOnNtpOpened, visible_categories_count, kMaxCategories); } diff --git a/chromium/components/ntp_snippets/content_suggestions_metrics.h b/chromium/components/ntp_snippets/content_suggestions_metrics.h index e31b7713633..160242aec10 100644 --- a/chromium/components/ntp_snippets/content_suggestions_metrics.h +++ b/chromium/components/ntp_snippets/content_suggestions_metrics.h @@ -15,9 +15,12 @@ namespace ntp_snippets { namespace metrics { -void OnPageShown( - const std::vector<std::pair<Category, int>>& suggestions_per_category, - int visible_categories_count); +// |is_category_visible| contains true iff the corresponding category can be +// seen by the user on this page (even if it is empty). It does not depend on +// whether the user actually saw the category. +void OnPageShown(const std::vector<Category>& categories, + const std::vector<int>& suggestions_per_category, + const std::vector<bool>& is_category_visible); // Should only be called once per NTP for each suggestion. void OnSuggestionShown(int global_position, diff --git a/chromium/components/ntp_snippets/content_suggestions_metrics_unittest.cc b/chromium/components/ntp_snippets/content_suggestions_metrics_unittest.cc index fe8193e300f..fd636e4b967 100644 --- a/chromium/components/ntp_snippets/content_suggestions_metrics_unittest.cc +++ b/chromium/components/ntp_snippets/content_suggestions_metrics_unittest.cc @@ -15,6 +15,7 @@ namespace metrics { namespace { using testing::ElementsAre; +using testing::IsEmpty; TEST(ContentSuggestionsMetricsTest, ShouldLogOnSuggestionsShown) { base::HistogramTester histogram_tester; @@ -45,6 +46,87 @@ TEST(ContentSuggestionsMetricsTest, ShouldLogOnSuggestionsShown) { base::Bucket(/*min=*/11, /*count=*/1))); } +TEST(ContentSuggestionsMetricsTest, + ShouldNotLogNotShownCategoriesWhenPageShown) { + base::HistogramTester histogram_tester; + OnPageShown(std::vector<Category>( + {Category::FromKnownCategory(KnownCategories::ARTICLES)}), + /*suggestions_per_category=*/{0}, + /*is_category_visible=*/{false}); + EXPECT_THAT( + histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible.Articles"), + IsEmpty()); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.SectionCountOnNtpOpened"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); +} + +TEST(ContentSuggestionsMetricsTest, + ShouldLogEmptyShownCategoriesWhenPageShown) { + base::HistogramTester histogram_tester; + OnPageShown(std::vector<Category>( + {Category::FromKnownCategory(KnownCategories::ARTICLES)}), + /*suggestions_per_category=*/{0}, + /*is_category_visible=*/{true}); + EXPECT_THAT( + histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible.Articles"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.SectionCountOnNtpOpened"), + ElementsAre(base::Bucket(/*min=*/1, /*count=*/1))); +} + +TEST(ContentSuggestionsMetricsTest, + ShouldLogNonEmptyShownCategoryWhenPageShown) { + base::HistogramTester histogram_tester; + OnPageShown(std::vector<Category>( + {Category::FromKnownCategory(KnownCategories::ARTICLES)}), + /*suggestions_per_category=*/{10}, + /*is_category_visible=*/{true}); + EXPECT_THAT( + histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible.Articles"), + ElementsAre(base::Bucket(/*min=*/10, /*count=*/1))); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible"), + ElementsAre(base::Bucket(/*min=*/10, /*count=*/1))); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.SectionCountOnNtpOpened"), + ElementsAre(base::Bucket(/*min=*/1, /*count=*/1))); +} + +TEST(ContentSuggestionsMetricsTest, + ShouldLogMultipleNonEmptyShownCategoriesWhenPageShown) { + base::HistogramTester histogram_tester; + OnPageShown(std::vector<Category>( + {Category::FromKnownCategory(KnownCategories::ARTICLES), + Category::FromKnownCategory(KnownCategories::BOOKMARKS)}), + /*suggestions_per_category=*/{10, 5}, + /*is_category_visible=*/{true, true}); + EXPECT_THAT( + histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible.Articles"), + ElementsAre(base::Bucket(/*min=*/10, /*count=*/1))); + EXPECT_THAT( + histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible.Bookmarks"), + ElementsAre(base::Bucket(/*min=*/5, /*count=*/1))); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.CountOnNtpOpenedIfVisible"), + ElementsAre(base::Bucket(/*min=*/15, /*count=*/1))); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.SectionCountOnNtpOpened"), + ElementsAre(base::Bucket(/*min=*/2, /*count=*/1))); +} + } // namespace } // namespace metrics } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/content_suggestions_service.cc b/chromium/components/ntp_snippets/content_suggestions_service.cc index 71f323594d2..d06e2170c45 100644 --- a/chromium/components/ntp_snippets/content_suggestions_service.cc +++ b/chromium/components/ntp_snippets/content_suggestions_service.cc @@ -25,6 +25,7 @@ #include "components/ntp_snippets/remote/remote_suggestions_provider.h" #include "components/prefs/pref_registry_simple.h" #include "components/prefs/pref_service.h" +#include "net/traffic_annotation/network_traffic_annotation.h" #include "ui/gfx/image/image.h" namespace ntp_snippets { @@ -232,6 +233,9 @@ void ContentSuggestionsService::OnGetFaviconFromCacheFinished( RecordFaviconFetchResult(continue_to_google_server ? FaviconFetchResult::SUCCESS_CACHED : FaviconFetchResult::SUCCESS_FETCHED); + // Update the time when the icon was last requested - postpone thus the + // automatic eviction of the favicon from the favicon database. + large_icon_service_->TouchIconFromGoogleServer(result.icon_url); return; } @@ -250,10 +254,29 @@ void ContentSuggestionsService::OnGetFaviconFromCacheFinished( // TODO(jkrcal): Currently used only for Articles for you which have public // URLs. Let the provider decide whether |publisher_url| may be private or // not. + net::NetworkTrafficAnnotationTag traffic_annotation = + net::DefineNetworkTrafficAnnotation("content_suggestion_get_favicon", R"( + semantics { + sender: "Content Suggestion" + description: + "Sends a request to a Google server to retrieve the favicon bitmap " + "for an article suggestion on the new tab page (URLs are public " + "and provided by Google)." + trigger: + "A request can be sent if Chrome does not have a favicon for a " + "particular page." + data: "Page URL and desired icon size." + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: false + setting: "This feature cannot be disabled by settings." + policy_exception_justification: "Not implemented." + })"); large_icon_service_ ->GetLargeIconOrFallbackStyleFromGoogleServerSkippingLocalCache( publisher_url, minimum_size_in_pixel, desired_size_in_pixel, - /*may_page_url_be_private=*/false, + /*may_page_url_be_private=*/false, traffic_annotation, base::Bind( &ContentSuggestionsService::OnGetFaviconFromGoogleServerFinished, base::Unretained(this), publisher_url, minimum_size_in_pixel, @@ -265,8 +288,8 @@ void ContentSuggestionsService::OnGetFaviconFromGoogleServerFinished( int minimum_size_in_pixel, int desired_size_in_pixel, const ImageFetchedCallback& callback, - bool success) { - if (!success) { + favicon_base::GoogleFaviconServerRequestStatus status) { + if (status != favicon_base::GoogleFaviconServerRequestStatus::SUCCESS) { callback.Run(gfx::Image()); RecordFaviconFetchResult(FaviconFetchResult::FAILURE); return; @@ -412,19 +435,28 @@ void ContentSuggestionsService::ReloadSuggestions() { } void ContentSuggestionsService::SetRemoteSuggestionsEnabled(bool enabled) { - pref_service_->SetBoolean(prefs::kEnableSnippets, enabled); + // TODO(dgn): Rewire if we decide to implement a dedicated prefs page. If not + // remove by M62. + NOTREACHED(); } bool ContentSuggestionsService::AreRemoteSuggestionsEnabled() const { - return pref_service_->GetBoolean(prefs::kEnableSnippets); + return remote_suggestions_provider_ && + !remote_suggestions_provider_->IsDisabled(); } bool ContentSuggestionsService::AreRemoteSuggestionsManaged() const { - return pref_service_->IsManagedPreference(prefs::kEnableSnippets); + // TODO(dgn): Rewire if we decide to implement a dedicated prefs page. If not + // remove by M62. + NOTREACHED(); + return false; } bool ContentSuggestionsService::AreRemoteSuggestionsManagedByCustodian() const { - return pref_service_->IsPreferenceManagedByCustodian(prefs::kEnableSnippets); + // TODO(dgn): Rewire if we decide to implement a dedicated prefs page. If not + // remove by M62. + NOTREACHED(); + return false; } //////////////////////////////////////////////////////////////////////////////// @@ -497,8 +529,7 @@ void ContentSuggestionsService::OnSuggestionInvalidated( // SigninManagerBase::Observer implementation void ContentSuggestionsService::GoogleSigninSucceeded( const std::string& account_id, - const std::string& username, - const std::string& password) { + const std::string& username) { OnSignInStateChanged(); } diff --git a/chromium/components/ntp_snippets/content_suggestions_service.h b/chromium/components/ntp_snippets/content_suggestions_service.h index 40c442bc6bb..de03d6b5cb0 100644 --- a/chromium/components/ntp_snippets/content_suggestions_service.h +++ b/chromium/components/ntp_snippets/content_suggestions_service.h @@ -15,7 +15,6 @@ #include "base/observer_list.h" #include "base/optional.h" #include "base/scoped_observer.h" -#include "base/supports_user_data.h" #include "base/task/cancelable_task_tracker.h" #include "base/time/time.h" #include "components/history/core/browser/history_service.h" @@ -48,7 +47,6 @@ class RemoteSuggestionsProvider; // Retrieves suggestions from a number of ContentSuggestionsProviders and serves // them grouped into categories. There can be at most one provider per category. class ContentSuggestionsService : public KeyedService, - public base::SupportsUserData, public ContentSuggestionsProvider::Observer, public SigninManagerBase::Observer, public history::HistoryServiceObserver { @@ -291,8 +289,7 @@ class ContentSuggestionsService : public KeyedService, // SigninManagerBase::Observer implementation void GoogleSigninSucceeded(const std::string& account_id, - const std::string& username, - const std::string& password) override; + const std::string& username) override; void GoogleSignedOut(const std::string& account_id, const std::string& username) override; @@ -353,7 +350,7 @@ class ContentSuggestionsService : public KeyedService, int minimum_size_in_pixel, int desired_size_in_pixel, const ImageFetchedCallback& callback, - bool success); + favicon_base::GoogleFaviconServerRequestStatus status); // Whether the content suggestions feature is enabled. State state_; diff --git a/chromium/components/ntp_snippets/features.cc b/chromium/components/ntp_snippets/features.cc index 6910f402873..f32adff9f4b 100644 --- a/chromium/components/ntp_snippets/features.cc +++ b/chromium/components/ntp_snippets/features.cc @@ -18,11 +18,12 @@ const base::Feature*(kAllFeatures[]) = {&kArticleSuggestionsFeature, &kBookmarkSuggestionsFeature, &kCategoryOrder, &kCategoryRanker, + &kBreakingNewsPushFeature, &kForeignSessionsSuggestionsFeature, &kIncreasedVisibility, + &kKeepPrefetchedContentSuggestions, &kNotificationsFeature, &kPhysicalWebPageSuggestionsFeature, - &kPreferAmpUrlsFeature, &kPublisherFaviconsFromNewServerFeature, &kRecentOfflineTabSuggestionsFeature, nullptr}; @@ -45,8 +46,8 @@ const base::Feature kPhysicalWebPageSuggestionsFeature{ const base::Feature kForeignSessionsSuggestionsFeature{ "NTPForeignSessionsSuggestions", base::FEATURE_DISABLED_BY_DEFAULT}; -const base::Feature kPreferAmpUrlsFeature{"NTPPreferAmpUrls", - base::FEATURE_ENABLED_BY_DEFAULT}; +const base::Feature kBreakingNewsPushFeature{"BreakingNewsPush", + base::FEATURE_DISABLED_BY_DEFAULT}; const base::Feature kCategoryRanker{"ContentSuggestionsCategoryRanker", base::FEATURE_ENABLED_BY_DEFAULT}; @@ -55,6 +56,10 @@ const base::Feature kPublisherFaviconsFromNewServerFeature{ "ContentSuggestionsFaviconsFromNewServer", base::FEATURE_DISABLED_BY_DEFAULT}; +const base::Feature kRemoteSuggestionsEmulateM58FetchingSchedule{ + "RemoteSuggestionsEmulateM58FetchingSchedule", + base::FEATURE_DISABLED_BY_DEFAULT}; + const char kCategoryRankerParameter[] = "category_ranker"; const char kCategoryRankerConstantRanker[] = "constant"; const char kCategoryRankerClickBasedRanker[] = "click_based"; @@ -65,6 +70,9 @@ CategoryRankerChoice GetSelectedCategoryRanker() { kCategoryRankerParameter); if (category_ranker_value.empty()) { + // TODO(crbug.com/735066): Remove the experiment configurations from + // fieldtrial_testing_config.json when enabling ClickBasedRanker by default. + // Default, Enabled or Disabled. return CategoryRankerChoice::CONSTANT; } @@ -141,4 +149,7 @@ const char kNotificationsOpenToNTPParam[] = "open_to_ntp"; const char kNotificationsDailyLimit[] = "daily_limit"; const char kNotificationsIgnoredLimitParam[] = "ignored_limit"; +const base::Feature kKeepPrefetchedContentSuggestions{ + "KeepPrefetchedContentSuggestions", base::FEATURE_DISABLED_BY_DEFAULT}; + } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/features.h b/chromium/components/ntp_snippets/features.h index 9946e7c2d0a..83800fa157c 100644 --- a/chromium/components/ntp_snippets/features.h +++ b/chromium/components/ntp_snippets/features.h @@ -37,8 +37,8 @@ extern const base::Feature kForeignSessionsSuggestionsFeature; // Feature to allow UI as specified here: https://crbug.com/660837. extern const base::Feature kIncreasedVisibility; -// Feature to prefer AMP URLs over regular URLs when available. -extern const base::Feature kPreferAmpUrlsFeature; +// Feature to listen for GCM push updates from the server. +extern const base::Feature kBreakingNewsPushFeature; // Feature to choose a category ranker. extern const base::Feature kCategoryRanker; @@ -46,6 +46,12 @@ extern const base::Feature kCategoryRanker; // Feature to allow the new Google favicon server for fetching publisher icons. extern const base::Feature kPublisherFaviconsFromNewServerFeature; +// Feature for simple experimental comparision and validation of changes since +// M58: enabling this brings back the M58 Stable fetching schedule (which is +// suitable for Holdback groups). +// TODO(jkrcal): Remove when the comparision is done (probably after M62). +extern const base::Feature kRemoteSuggestionsEmulateM58FetchingSchedule; + // Parameter and its values for the kCategoryRanker feature flag. extern const char kCategoryRankerParameter[]; extern const char kCategoryRankerConstantRanker[]; @@ -113,6 +119,10 @@ constexpr int kNotificationsDefaultDailyLimit = 1; extern const char kNotificationsIgnoredLimitParam[]; constexpr int kNotificationsIgnoredDefaultLimit = 3; +// Whether to keep some prefetched content suggestions even when new suggestions +// have been fetched. +extern const base::Feature kKeepPrefetchedContentSuggestions; + } // namespace ntp_snippets #endif // COMPONENTS_NTP_SNIPPETS_FEATURES_H_ diff --git a/chromium/components/ntp_snippets/ntp_snippets_constants.cc b/chromium/components/ntp_snippets/ntp_snippets_constants.cc index 82304939470..52abe08ab69 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_constants.cc +++ b/chromium/components/ntp_snippets/ntp_snippets_constants.cc @@ -9,6 +9,12 @@ namespace ntp_snippets { const base::FilePath::CharType kDatabaseFolder[] = FILE_PATH_LITERAL("NTPSnippets"); +const base::FilePath::CharType kBreakingNewsDatabaseFolder[] = + FILE_PATH_LITERAL("NTPBreakingNews"); + +const char kContentSuggestionsApiScope[] = + "https://www.googleapis.com/auth/chrome-content-suggestions"; + const char kContentSuggestionsServer[] = "https://chromecontentsuggestions-pa.googleapis.com/v1/suggestions/fetch"; const char kContentSuggestionsStagingServer[] = @@ -18,4 +24,24 @@ const char kContentSuggestionsAlphaServer[] = "https://alpha-chromecontentsuggestions-pa.sandbox.googleapis.com/v1/" "suggestions/fetch"; +const char kPushUpdatesSubscriptionServer[] = + "https://chromecontentsuggestions-pa.googleapis.com/v1/suggestions/" + "subscribe"; +const char kPushUpdatesSubscriptionStagingServer[] = + "https://staging-chromecontentsuggestions-pa.googleapis.com/v1/suggestions/" + "subscribe"; +const char kPushUpdatesSubscriptionAlphaServer[] = + "https://alpha-chromecontentsuggestions-pa.sandbox.googleapis.com/v1/" + "suggestions/subscribe"; + +const char kPushUpdatesUnsubscriptionServer[] = + "https://chromecontentsuggestions-pa.googleapis.com/v1/suggestions/" + "unsubscribe"; +const char kPushUpdatesUnsubscriptionStagingServer[] = + "https://staging-chromecontentsuggestions-pa.googleapis.com/v1/suggestions/" + "unsubscribe"; +const char kPushUpdatesUnsubscriptionAlphaServer[] = + "https://alpha-chromecontentsuggestions-pa.sandbox.googleapis.com/v1/" + "suggestions/unsubscribe"; + } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_constants.h b/chromium/components/ntp_snippets/ntp_snippets_constants.h index aff5c3720d4..f7ddd58acb3 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_constants.h +++ b/chromium/components/ntp_snippets/ntp_snippets_constants.h @@ -13,12 +13,29 @@ namespace ntp_snippets { // the name of the folder, not a full path - it must be appended to e.g. the // profile path. extern const base::FilePath::CharType kDatabaseFolder[]; +// TODO(mamir): Check if the same DB can be used. +extern const base::FilePath::CharType kBreakingNewsDatabaseFolder[]; + +// OAuth access token scope. +extern const char kContentSuggestionsApiScope[]; // Server endpoints for fetching snippets. extern const char kContentSuggestionsServer[]; // used on stable/beta extern const char kContentSuggestionsStagingServer[]; // used on dev/canary extern const char kContentSuggestionsAlphaServer[]; // for testing +// Server endpoints for push updates subscription. +extern const char kPushUpdatesSubscriptionServer[]; // used on stable/beta +extern const char + kPushUpdatesSubscriptionStagingServer[]; // used on dev/canary +extern const char kPushUpdatesSubscriptionAlphaServer[]; // for testing + +// Server endpoints for push updates unsubscription. +extern const char kPushUpdatesUnsubscriptionServer[]; // used on stable/beta +extern const char + kPushUpdatesUnsubscriptionStagingServer[]; // used on dev/canary +extern const char kPushUpdatesUnsubscriptionAlphaServer[]; // for testing + } // namespace ntp_snippets #endif diff --git a/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider_unittest.cc b/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider_unittest.cc index 7d8489ca0e0..c43ad141d2b 100644 --- a/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider_unittest.cc +++ b/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider_unittest.cc @@ -116,7 +116,9 @@ class RecentTabSuggestionsProviderTestNoLoad : public testing::Test { int tab_id = offline_pages::RecentTabsUIAdapterDelegate::TabIdFromClientId( item.client_id); RemoveTab(tab_id); - ui_adapter_->OfflinePageDeleted(item.offline_id, item.client_id); + ui_adapter_->OfflinePageDeleted( + offline_pages::OfflinePageModel::DeletedPageInfo( + item.offline_id, item.client_id, "" /* request_origin */)); } std::set<std::string> ReadDismissedIDsFromPrefs() { diff --git a/chromium/components/ntp_snippets/pref_names.cc b/chromium/components/ntp_snippets/pref_names.cc index bea67c36392..72851b04e21 100644 --- a/chromium/components/ntp_snippets/pref_names.cc +++ b/chromium/components/ntp_snippets/pref_names.cc @@ -81,5 +81,14 @@ const char kClickBasedCategoryRankerOrderWithClicks[] = const char kClickBasedCategoryRankerLastDecayTime[] = "ntp_suggestions.click_based_category_ranker.last_decay_time"; +const char kBreakingNewsSubscriptionDataToken[] = + "ntp_suggestions.breaking_news_subscription_data.token"; + +const char kBreakingNewsSubscriptionDataIsAuthenticated[] = + "ntp_suggestions.breaking_news_subscription_data.is_authenticated"; + +const char kBreakingNewsGCMSubscriptionTokenCache[] = + "ntp_suggestions.breaking_news_gcm_subscription_token_cache"; + } // namespace prefs } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/pref_names.h b/chromium/components/ntp_snippets/pref_names.h index fe3e198c83e..110aafee4e2 100644 --- a/chromium/components/ntp_snippets/pref_names.h +++ b/chromium/components/ntp_snippets/pref_names.h @@ -90,6 +90,26 @@ extern const char kClickBasedCategoryRankerOrderWithClicks[]; // The pref name for the time when last click decay has happened. extern const char kClickBasedCategoryRankerLastDecayTime[]; +// The folllowing prefs hold the data used when subscribing for content +// suggestions via GCM push updates. They are stored in pref such that in case +// of change (e.g. the token renders invalid), re-subscription is required. +// They are stored in prefs for persisting them across Chrome restarts. +/////////////////////////////////////////////////////////////////////////////// +// The pref name for the subscription token used when subscription for +// breaking news push updates. +extern const char kBreakingNewsSubscriptionDataToken[]; +// The pref name for whether the subscription is authenticated or not. +extern const char kBreakingNewsSubscriptionDataIsAuthenticated[]; +//////////////////////// End of breaking news subscription-related prefs. + +// The pref name for the subscription token received from the gcm server. As +// recommended by the GCM team, it is cached in pref for faster bookkeeping to +// see if subscription exists. This is pref holds the valid token even if +// different from the one used for subscription. When they are different, Chrome +// unsubscribes the old token from the content suggestions server, subscribe +// with the new one and update kBreakingNewsSubscriptionDataToken. +extern const char kBreakingNewsGCMSubscriptionTokenCache[]; + } // namespace prefs } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/DEPS b/chromium/components/ntp_snippets/remote/DEPS new file mode 100644 index 00000000000..984d8cbcbb5 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+components/offline_pages" +] diff --git a/chromium/components/ntp_snippets/remote/cached_image_fetcher.cc b/chromium/components/ntp_snippets/remote/cached_image_fetcher.cc new file mode 100644 index 00000000000..5faf2830cc1 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/cached_image_fetcher.cc @@ -0,0 +1,142 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/cached_image_fetcher.h" + +#include "base/bind.h" +#include "base/location.h" +#include "components/image_fetcher/core/image_decoder.h" +#include "components/image_fetcher/core/image_fetcher.h" +#include "components/ntp_snippets/remote/remote_suggestions_database.h" +#include "net/traffic_annotation/network_traffic_annotation.h" +#include "ui/gfx/geometry/size.h" +#include "ui/gfx/image/image.h" + +namespace ntp_snippets { + +CachedImageFetcher::CachedImageFetcher( + std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, + PrefService* pref_service, + RemoteSuggestionsDatabase* database) + : image_fetcher_(std::move(image_fetcher)), + database_(database), + thumbnail_requests_throttler_( + pref_service, + RequestThrottler::RequestType::CONTENT_SUGGESTION_THUMBNAIL) { + // |image_fetcher_| can be null in tests. + if (image_fetcher_) { + image_fetcher_->SetImageFetcherDelegate(this); + image_fetcher_->SetDataUseServiceName( + data_use_measurement::DataUseUserData::NTP_SNIPPETS_THUMBNAILS); + } +} + +CachedImageFetcher::~CachedImageFetcher() {} + +void CachedImageFetcher::FetchSuggestionImage( + const ContentSuggestion::ID& suggestion_id, + const GURL& url, + const ImageFetchedCallback& callback) { + database_->LoadImage( + suggestion_id.id_within_category(), + base::Bind(&CachedImageFetcher::OnImageFetchedFromDatabase, + base::Unretained(this), callback, suggestion_id, url)); +} + +// This function gets only called for caching the image data received from the +// network. The actual decoding is done in OnImageDecodedFromDatabase(). +void CachedImageFetcher::OnImageDataFetched( + const std::string& id_within_category, + const std::string& image_data) { + if (image_data.empty()) { + return; + } + database_->SaveImage(id_within_category, image_data); +} + +void CachedImageFetcher::OnImageDecodingDone( + const ImageFetchedCallback& callback, + const std::string& id_within_category, + const gfx::Image& image, + const image_fetcher::RequestMetadata& metadata) { + callback.Run(image); +} + +void CachedImageFetcher::OnImageFetchedFromDatabase( + const ImageFetchedCallback& callback, + const ContentSuggestion::ID& suggestion_id, + const GURL& url, + std::string data) { // SnippetImageCallback requires by-value. + // The image decoder is null in tests. + if (image_fetcher_->GetImageDecoder() && !data.empty()) { + image_fetcher_->GetImageDecoder()->DecodeImage( + data, + // We're not dealing with multi-frame images. + /*desired_image_frame_size=*/gfx::Size(), + base::Bind(&CachedImageFetcher::OnImageDecodedFromDatabase, + base::Unretained(this), callback, suggestion_id, url)); + return; + } + // Fetching from the DB failed; start a network fetch. + FetchImageFromNetwork(suggestion_id, url, callback); +} + +void CachedImageFetcher::OnImageDecodedFromDatabase( + const ImageFetchedCallback& callback, + const ContentSuggestion::ID& suggestion_id, + const GURL& url, + const gfx::Image& image) { + if (!image.IsEmpty()) { + callback.Run(image); + return; + } + // If decoding the image failed, delete the DB entry. + database_->DeleteImage(suggestion_id.id_within_category()); + FetchImageFromNetwork(suggestion_id, url, callback); +} + +void CachedImageFetcher::FetchImageFromNetwork( + const ContentSuggestion::ID& suggestion_id, + const GURL& url, + const ImageFetchedCallback& callback) { + if (url.is_empty() || !thumbnail_requests_throttler_.DemandQuotaForRequest( + /*interactive_request=*/true)) { + // Return an empty image. Directly, this is never synchronous with the + // original FetchSuggestionImage() call - an asynchronous database query has + // happened in the meantime. + callback.Run(gfx::Image()); + return; + } + + net::NetworkTrafficAnnotationTag traffic_annotation = + net::DefineNetworkTrafficAnnotation("remote_suggestions_provider", R"( + semantics { + sender: "Content Suggestion Thumbnail Fetch" + description: + "Retrieves thumbnails for content suggestions, for display on the " + "New Tab page or Chrome Home." + trigger: + "Triggered when the user looks at a content suggestion (and its " + "thumbnail isn't cached yet)." + data: "None." + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: false + setting: "Currently not available, but in progress: crbug.com/703684" + chrome_policy { + NTPContentSuggestionsEnabled { + policy_options {mode: MANDATORY} + NTPContentSuggestionsEnabled: false + } + } + })"); + image_fetcher_->StartOrQueueNetworkRequest( + suggestion_id.id_within_category(), url, + base::Bind(&CachedImageFetcher::OnImageDecodingDone, + base::Unretained(this), callback), + traffic_annotation); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/cached_image_fetcher.h b/chromium/components/ntp_snippets/remote/cached_image_fetcher.h new file mode 100644 index 00000000000..1bb290e2758 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/cached_image_fetcher.h @@ -0,0 +1,86 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_CACHED_IMAGE_FETCHER_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_CACHED_IMAGE_FETCHER_H_ + +#include <cstddef> +#include <memory> +#include <string> +#include <vector> + +#include "base/callback_forward.h" +#include "base/gtest_prod_util.h" +#include "base/macros.h" +#include "components/image_fetcher/core/image_fetcher_delegate.h" +#include "components/ntp_snippets/callbacks.h" +#include "components/ntp_snippets/content_suggestion.h" +#include "components/ntp_snippets/remote/request_throttler.h" + +class PrefService; + +namespace gfx { +class Image; +} // namespace gfx + +namespace image_fetcher { +class ImageFetcher; +struct RequestMetadata; +} // namespace image_fetcher + +namespace ntp_snippets { + +class RemoteSuggestionsDatabase; + +// CachedImageFetcher takes care of fetching images from the network and caching +// them in the database. +class CachedImageFetcher : public image_fetcher::ImageFetcherDelegate { + public: + // |pref_service| and |database| need to outlive the created image fetcher + // instance. + CachedImageFetcher(std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, + PrefService* pref_service, + RemoteSuggestionsDatabase* database); + ~CachedImageFetcher() override; + + // Fetches the image for a suggestion. The fetcher will first issue a lookup + // to the underlying cache with a fallback to the network. + void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, + const GURL& image_url, + const ImageFetchedCallback& callback); + + private: + // image_fetcher::ImageFetcherDelegate implementation. + void OnImageDataFetched(const std::string& id_within_category, + const std::string& image_data) override; + + void OnImageDecodingDone(const ImageFetchedCallback& callback, + const std::string& id_within_category, + const gfx::Image& image, + const image_fetcher::RequestMetadata& metadata); + void OnImageFetchedFromDatabase( + const ImageFetchedCallback& callback, + const ContentSuggestion::ID& suggestion_id, + const GURL& image_url, + // SnippetImageCallback requires by-value (not const ref). + std::string data); + void OnImageDecodedFromDatabase(const ImageFetchedCallback& callback, + const ContentSuggestion::ID& suggestion_id, + const GURL& url, + const gfx::Image& image); + void FetchImageFromNetwork(const ContentSuggestion::ID& suggestion_id, + const GURL& url, + const ImageFetchedCallback& callback); + + std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher_; + RemoteSuggestionsDatabase* database_; + // Request throttler for limiting requests to thumbnail images. + RequestThrottler thumbnail_requests_throttler_; + + DISALLOW_COPY_AND_ASSIGN(CachedImageFetcher); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_CACHED_IMAGE_FETCHER_H_ diff --git a/chromium/components/ntp_snippets/remote/cached_image_fetcher_unittest.cc b/chromium/components/ntp_snippets/remote/cached_image_fetcher_unittest.cc new file mode 100644 index 00000000000..b176acaa8ac --- /dev/null +++ b/chromium/components/ntp_snippets/remote/cached_image_fetcher_unittest.cc @@ -0,0 +1,153 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/cached_image_fetcher.h" + +#include <memory> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/files/file_path.h" +#include "base/files/scoped_temp_dir.h" +#include "base/memory/ptr_util.h" +#include "base/test/mock_callback.h" +#include "base/test/test_simple_task_runner.h" +#include "base/threading/thread_task_runner_handle.h" +#include "components/image_fetcher/core/image_decoder.h" +#include "components/image_fetcher/core/image_fetcher.h" +#include "components/image_fetcher/core/image_fetcher_impl.h" +#include "components/ntp_snippets/remote/proto/ntp_snippets.pb.h" +#include "components/ntp_snippets/remote/remote_suggestions_database.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/testing_pref_service.h" +#include "net/url_request/test_url_fetcher_factory.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "ui/gfx/image/image.h" +#include "ui/gfx/image/image_unittest_util.h" + +using testing::_; +using testing::Eq; +using testing::Property; + +namespace ntp_snippets { + +namespace { + +const char kImageData[] = "data"; +const char kImageURL[] = "http://image.test/test.png"; +const char kSnippetID[] = "http://localhost"; + +// Always decodes a valid image for all non-empty input. +class FakeImageDecoder : public image_fetcher::ImageDecoder { + public: + void DecodeImage( + const std::string& image_data, + const gfx::Size& desired_image_frame_size, + const image_fetcher::ImageDecodedCallback& callback) override { + gfx::Image image; + if (!image_data.empty()) { + image = gfx::test::CreateImage(); + } + callback.Run(image); + } +}; + +} // namespace + +class CachedImageFetcherTest : public testing::Test { + public: + CachedImageFetcherTest() + : fake_url_fetcher_factory_(nullptr), + mock_task_runner_(new base::TestSimpleTaskRunner()), + mock_task_runner_handle_(mock_task_runner_) { + EXPECT_TRUE(database_dir_.CreateUniqueTempDir()); + + RequestThrottler::RegisterProfilePrefs(pref_service_.registry()); + database_ = base::MakeUnique<RemoteSuggestionsDatabase>( + database_dir_.GetPath(), mock_task_runner_); + request_context_getter_ = scoped_refptr<net::TestURLRequestContextGetter>( + new net::TestURLRequestContextGetter(mock_task_runner_.get())); + + auto decoder = base::MakeUnique<FakeImageDecoder>(); + fake_image_decoder_ = decoder.get(); + cached_image_fetcher_ = base::MakeUnique<ntp_snippets::CachedImageFetcher>( + base::MakeUnique<image_fetcher::ImageFetcherImpl>( + std::move(decoder), request_context_getter_.get()), + &pref_service_, database_.get()); + RunUntilIdle(); + EXPECT_TRUE(database_->IsInitialized()); + } + + void FetchImage(const ImageFetchedCallback& callback) { + ContentSuggestion::ID content_suggestion_id( + Category::FromKnownCategory(KnownCategories::ARTICLES), kSnippetID); + cached_image_fetcher_->FetchSuggestionImage(content_suggestion_id, + GURL(kImageURL), callback); + } + + void RunUntilIdle() { mock_task_runner_->RunUntilIdle(); } + + RemoteSuggestionsDatabase* database() { return database_.get(); } + FakeImageDecoder* fake_image_decoder() { return fake_image_decoder_; } + net::FakeURLFetcherFactory* fake_url_fetcher_factory() { + return &fake_url_fetcher_factory_; + } + + private: + std::unique_ptr<CachedImageFetcher> cached_image_fetcher_; + std::unique_ptr<RemoteSuggestionsDatabase> database_; + base::ScopedTempDir database_dir_; + FakeImageDecoder* fake_image_decoder_; + net::FakeURLFetcherFactory fake_url_fetcher_factory_; + scoped_refptr<base::TestSimpleTaskRunner> mock_task_runner_; + base::ThreadTaskRunnerHandle mock_task_runner_handle_; + TestingPrefServiceSimple pref_service_; + scoped_refptr<net::TestURLRequestContextGetter> request_context_getter_; + + DISALLOW_COPY_AND_ASSIGN(CachedImageFetcherTest); +}; + +TEST_F(CachedImageFetcherTest, FetchImageFromCache) { + // Save the image in the database. + database()->SaveImage(kSnippetID, kImageData); + RunUntilIdle(); + + // Do not provide any URL responses and expect that the image is fetched (from + // cache). + base::MockCallback<ImageFetchedCallback> mock_image_fetched_callback; + EXPECT_CALL(mock_image_fetched_callback, + Run(Property(&gfx::Image::IsEmpty, Eq(false)))); + FetchImage(mock_image_fetched_callback.Get()); + RunUntilIdle(); +} + +TEST_F(CachedImageFetcherTest, FetchImageNotInCache) { + // Expect the image to be fetched by URL. + fake_url_fetcher_factory()->SetFakeResponse(GURL(kImageURL), kImageData, + net::HTTP_OK, + net::URLRequestStatus::SUCCESS); + base::MockCallback<ImageFetchedCallback> mock_image_fetched_callback; + EXPECT_CALL(mock_image_fetched_callback, + Run(Property(&gfx::Image::IsEmpty, Eq(false)))); + FetchImage(mock_image_fetched_callback.Get()); + RunUntilIdle(); +} + +TEST_F(CachedImageFetcherTest, FetchNonExistingImage) { + const std::string kErrorResponse = "error-response"; + fake_url_fetcher_factory()->SetFakeResponse(GURL(kImageURL), kErrorResponse, + net::HTTP_NOT_FOUND, + net::URLRequestStatus::FAILED); + // Expect an empty image is fetched if the URL cannot be requested. + const std::string kEmptyImageData; + base::MockCallback<ImageFetchedCallback> mock_image_fetched_callback; + EXPECT_CALL(mock_image_fetched_callback, + Run(Property(&gfx::Image::IsEmpty, Eq(true)))); + FetchImage(mock_image_fetched_callback.Get()); + RunUntilIdle(); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/contextual_json_request.cc b/chromium/components/ntp_snippets/remote/contextual_json_request.cc new file mode 100644 index 00000000000..fa1d6a66e44 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/contextual_json_request.cc @@ -0,0 +1,251 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/contextual_json_request.h" + +#include <algorithm> +#include <utility> +#include <vector> + +#include "base/json/json_writer.h" +#include "base/memory/ptr_util.h" +#include "base/strings/stringprintf.h" +#include "base/values.h" +#include "components/data_use_measurement/core/data_use_user_data.h" +#include "components/ntp_snippets/category_info.h" +#include "components/ntp_snippets/features.h" +#include "components/strings/grit/components_strings.h" +#include "components/variations/net/variations_http_headers.h" +#include "components/variations/variations_associated_data.h" +#include "net/base/load_flags.h" +#include "net/http/http_response_headers.h" +#include "net/http/http_status_code.h" +#include "net/traffic_annotation/network_traffic_annotation.h" +#include "net/url_request/url_fetcher.h" +#include "net/url_request/url_request_context_getter.h" +#include "third_party/icu/source/common/unicode/uloc.h" +#include "third_party/icu/source/common/unicode/utypes.h" + +using net::URLFetcher; +using net::URLRequestContextGetter; +using net::HttpRequestHeaders; +using net::URLRequestStatus; + +namespace ntp_snippets { + +namespace internal { + +namespace { + +const int k5xxRetries = 2; + +} // namespace + +ContextualJsonRequest::ContextualJsonRequest(const ParseJSONCallback& callback) + : parse_json_callback_(callback), weak_ptr_factory_(this) {} + +ContextualJsonRequest::~ContextualJsonRequest() { + DLOG_IF(ERROR, !request_completed_callback_.is_null()) + << "The CompletionCallback was never called!"; +} + +void ContextualJsonRequest::Start(CompletedCallback callback) { + request_completed_callback_ = std::move(callback); + url_fetcher_->Start(); +} + +std::string ContextualJsonRequest::GetResponseString() const { + std::string response; + url_fetcher_->GetResponseAsString(&response); + return response; +} + +// URLFetcherDelegate overrides +void ContextualJsonRequest::OnURLFetchComplete(const net::URLFetcher* source) { + DCHECK_EQ(url_fetcher_.get(), source); + const URLRequestStatus& status = url_fetcher_->GetStatus(); + int response = url_fetcher_->GetResponseCode(); + // TODO(gaschler): Add UMA metrics for response status code + + if (!status.is_success()) { + std::move(request_completed_callback_) + .Run(/*result=*/nullptr, FetchResult::URL_REQUEST_STATUS_ERROR, + /*error_details=*/base::StringPrintf(" %d", status.error())); + } else if (response != net::HTTP_OK) { + // TODO(jkrcal): https://crbug.com/609084 + // We need to deal with the edge case again where the auth + // token expires just before we send the request (in which case we need to + // fetch a new auth token). We should extract that into a common class + // instead of adding it to every single class that uses auth tokens. + std::move(request_completed_callback_) + .Run(/*result=*/nullptr, FetchResult::HTTP_ERROR, + /*error_details=*/base::StringPrintf(" %d", response)); + } else { + ParseJsonResponse(); + } +} + +void ContextualJsonRequest::ParseJsonResponse() { + std::string json_string; + bool stores_result_to_string = + url_fetcher_->GetResponseAsString(&json_string); + DCHECK(stores_result_to_string); + + parse_json_callback_.Run(json_string, + base::Bind(&ContextualJsonRequest::OnJsonParsed, + weak_ptr_factory_.GetWeakPtr()), + base::Bind(&ContextualJsonRequest::OnJsonError, + weak_ptr_factory_.GetWeakPtr())); +} + +void ContextualJsonRequest::OnJsonParsed(std::unique_ptr<base::Value> result) { + std::move(request_completed_callback_) + .Run(std::move(result), FetchResult::SUCCESS, + /*error_details=*/std::string()); +} + +void ContextualJsonRequest::OnJsonError(const std::string& error) { + std::string json_string; + url_fetcher_->GetResponseAsString(&json_string); + LOG(WARNING) << "Received invalid JSON (" << error << "): " << json_string; + std::move(request_completed_callback_) + .Run(/*result=*/nullptr, FetchResult::JSON_PARSE_ERROR, + /*error_details=*/base::StringPrintf(" (error %s)", error.c_str())); +} + +ContextualJsonRequest::Builder::Builder() = default; +ContextualJsonRequest::Builder::Builder(ContextualJsonRequest::Builder&&) = + default; +ContextualJsonRequest::Builder::~Builder() = default; + +std::unique_ptr<ContextualJsonRequest> ContextualJsonRequest::Builder::Build() + const { + DCHECK(url_request_context_getter_); + auto request = base::MakeUnique<ContextualJsonRequest>(parse_json_callback_); + std::string body = BuildBody(); + std::string headers = BuildHeaders(); + request->url_fetcher_ = BuildURLFetcher(request.get(), headers, body); + + // Log the request for debugging network issues. + VLOG(1) << "Sending a NTP snippets request to " << url_ << ":\n" + << headers << "\n" + << body; + + return request; +} + +ContextualJsonRequest::Builder& +ContextualJsonRequest::Builder::SetAuthentication( + const std::string& account_id, + const std::string& auth_header) { + auth_header_ = auth_header; + return *this; +} + +ContextualJsonRequest::Builder& +ContextualJsonRequest::Builder::SetParseJsonCallback( + ParseJSONCallback callback) { + parse_json_callback_ = callback; + return *this; +} + +ContextualJsonRequest::Builder& ContextualJsonRequest::Builder::SetUrl( + const GURL& url) { + url_ = url; + return *this; +} + +ContextualJsonRequest::Builder& +ContextualJsonRequest::Builder::SetUrlRequestContextGetter( + const scoped_refptr<net::URLRequestContextGetter>& context_getter) { + url_request_context_getter_ = context_getter; + return *this; +} + +ContextualJsonRequest::Builder& ContextualJsonRequest::Builder::SetContentUrl( + const GURL& url) { + content_url_ = url; + return *this; +} + +std::string ContextualJsonRequest::Builder::BuildHeaders() const { + net::HttpRequestHeaders headers; + headers.SetHeader("Content-Type", "application/json; charset=UTF-8"); + if (!auth_header_.empty()) { + headers.SetHeader("Authorization", auth_header_); + } + // Add X-Client-Data header with experiment IDs from field trials. + // Note: It's OK to pass |is_signed_in| false if it's unknown, as it does + // not affect transmission of experiments coming from the variations server. + bool is_signed_in = false; + variations::AppendVariationHeaders(url_, + false, // incognito + false, // uma_enabled + is_signed_in, &headers); + return headers.ToString(); +} + +std::string ContextualJsonRequest::Builder::BuildBody() const { + auto request = base::MakeUnique<base::DictionaryValue>(); + + request->SetString("url", content_url_.spec()); + auto categories = base::MakeUnique<base::ListValue>(); + categories->AppendString("RELATED_ARTICLES"); + categories->AppendString("PUBLIC_DEBATE"); + request->Set("categories", std::move(categories)); + + std::string request_json; + bool success = base::JSONWriter::WriteWithOptions( + *request, base::JSONWriter::OPTIONS_PRETTY_PRINT, &request_json); + DCHECK(success); + return request_json; +} + +std::unique_ptr<net::URLFetcher> +ContextualJsonRequest::Builder::BuildURLFetcher( + net::URLFetcherDelegate* delegate, + const std::string& headers, + const std::string& body) const { + net::NetworkTrafficAnnotationTag traffic_annotation = + net::DefineNetworkTrafficAnnotation("ntp_snippets_fetch", R"( + semantics { + sender: "New Tab Page Contextual Suggestions Fetch" + description: + "Chromium can show contextual suggestions that are related to the " + "currently visited page on the New Tab page. " + trigger: + "Triggered when Home sheet is pulled up." + data: + "The Chromium UI language, as well as a second language the user " + "understands, based on translate::LanguageModel. For signed-in " + "users, the requests is authenticated." + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: false + setting: + "This feature can be disabled by the flag " + "contextual-suggestions-carousel." + })"); + std::unique_ptr<net::URLFetcher> url_fetcher = net::URLFetcher::Create( + url_, net::URLFetcher::POST, delegate, traffic_annotation); + url_fetcher->SetRequestContext(url_request_context_getter_.get()); + url_fetcher->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES | + net::LOAD_DO_NOT_SAVE_COOKIES); + data_use_measurement::DataUseUserData::AttachToFetcher( + url_fetcher.get(), + data_use_measurement::DataUseUserData::NTP_SNIPPETS_SUGGESTIONS); + + url_fetcher->SetExtraRequestHeaders(headers); + url_fetcher->SetUploadData("application/json", body); + + // Fetchers are sometimes cancelled because a network change was detected. + url_fetcher->SetAutomaticallyRetryOnNetworkChanges(3); + url_fetcher->SetMaxRetriesOn5xx(k5xxRetries); + return url_fetcher; +} + +} // namespace internal + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/contextual_json_request.h b/chromium/components/ntp_snippets/remote/contextual_json_request.h new file mode 100644 index 00000000000..cf379da180c --- /dev/null +++ b/chromium/components/ntp_snippets/remote/contextual_json_request.h @@ -0,0 +1,115 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_CONTEXTUAL_JSON_REQUEST_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_CONTEXTUAL_JSON_REQUEST_H_ + +#include <memory> +#include <string> +#include <utility> + +#include "base/callback.h" +#include "base/memory/weak_ptr.h" +#include "components/ntp_snippets/remote/json_request.h" +#include "components/ntp_snippets/status.h" +#include "google_apis/gaia/oauth2_token_service.h" +#include "net/http/http_request_headers.h" + +namespace base { +class Value; +} // namespace base + +namespace ntp_snippets { + +namespace internal { + +// A request to query contextual suggestions. +class ContextualJsonRequest : public net::URLFetcherDelegate { + public: + // A client can expect error_details only, if there was any error during the + // fetching or parsing. In successful cases, it will be an empty string. + using CompletedCallback = + base::OnceCallback<void(std::unique_ptr<base::Value> result, + FetchResult result_code, + const std::string& error_details)>; + + // Builds authenticated and non-authenticated ContextualJsonRequests. + class Builder { + public: + Builder(); + Builder(Builder&&); + ~Builder(); + + // Builds a Request object that contains all data to fetch new snippets. + std::unique_ptr<ContextualJsonRequest> Build() const; + + Builder& SetAuthentication(const std::string& account_id, + const std::string& auth_header); + Builder& SetParseJsonCallback(ParseJSONCallback callback); + Builder& SetUrl(const GURL& url); + Builder& SetUrlRequestContextGetter( + const scoped_refptr<net::URLRequestContextGetter>& context_getter); + Builder& SetContentUrl(const GURL& url); + + // These preview methods allow to inspect the Request without exposing it + // publicly. + std::string PreviewRequestBodyForTesting() { return BuildBody(); } + std::string PreviewRequestHeadersForTesting() { return BuildHeaders(); } + + private: + std::string BuildHeaders() const; + std::string BuildBody() const; + std::unique_ptr<net::URLFetcher> BuildURLFetcher( + net::URLFetcherDelegate* request, + const std::string& headers, + const std::string& body) const; + + std::string auth_header_; + ParseJSONCallback parse_json_callback_; + GURL url_; + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_; + + // The URL for which to fetch contextual suggestions for. + GURL content_url_; + + DISALLOW_COPY_AND_ASSIGN(Builder); + }; + + ContextualJsonRequest(const ParseJSONCallback& callback); + ContextualJsonRequest(ContextualJsonRequest&&); + ~ContextualJsonRequest() override; + + void Start(CompletedCallback callback); + + std::string GetResponseString() const; + + private: + // URLFetcherDelegate implementation. + void OnURLFetchComplete(const net::URLFetcher* source) override; + + void ParseJsonResponse(); + void OnJsonParsed(std::unique_ptr<base::Value> result); + void OnJsonError(const std::string& error); + + // The fetcher for downloading the snippets. Only non-null if a fetch is + // currently ongoing. + std::unique_ptr<net::URLFetcher> url_fetcher_; + + // This callback is called to parse a json string. It contains callbacks for + // error and success cases. + ParseJSONCallback parse_json_callback_; + + // The callback to notify when URLFetcher finished and results are available. + CompletedCallback request_completed_callback_; + + base::WeakPtrFactory<ContextualJsonRequest> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(ContextualJsonRequest); +}; + +} // namespace internal + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_CONTEXTUAL_JSON_REQUEST_H_ diff --git a/chromium/components/ntp_snippets/remote/contextual_json_request_unittest.cc b/chromium/components/ntp_snippets/remote/contextual_json_request_unittest.cc new file mode 100644 index 00000000000..9e8f1ddf11c --- /dev/null +++ b/chromium/components/ntp_snippets/remote/contextual_json_request_unittest.cc @@ -0,0 +1,90 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/contextual_json_request.h" + +#include <utility> + +#include "base/json/json_reader.h" +#include "base/memory/ptr_util.h" +#include "base/message_loop/message_loop.h" +#include "base/test/test_mock_time_task_runner.h" +#include "base/values.h" +#include "components/ntp_snippets/features.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace ntp_snippets { + +namespace internal { + +namespace { + +using testing::_; +using testing::Eq; +using testing::StrEq; + +MATCHER_P(EqualsJSON, json, "equals JSON") { + std::unique_ptr<base::Value> expected = base::JSONReader::Read(json); + if (!expected) { + *result_listener << "INTERNAL ERROR: couldn't parse expected JSON"; + return false; + } + + std::string err_msg; + int err_line, err_col; + std::unique_ptr<base::Value> actual = base::JSONReader::ReadAndReturnError( + arg, base::JSON_PARSE_RFC, nullptr, &err_msg, &err_line, &err_col); + if (!actual) { + *result_listener << "input:" << err_line << ":" << err_col << ": " + << "parse error: " << err_msg; + return false; + } + return base::Value::Equals(actual.get(), expected.get()); +} + +} // namespace + +class ContextualJsonRequestTest : public testing::Test { + public: + ContextualJsonRequestTest() + : request_context_getter_( + new net::TestURLRequestContextGetter(loop_.task_runner())) {} + + ContextualJsonRequest::Builder CreateDefaultBuilder() { + ContextualJsonRequest::Builder builder; + builder.SetUrl(GURL("http://valid-url.test")) + .SetUrlRequestContextGetter(request_context_getter_.get()); + return builder; + } + + private: + base::MessageLoop loop_; + scoped_refptr<net::TestURLRequestContextGetter> request_context_getter_; + + DISALLOW_COPY_AND_ASSIGN(ContextualJsonRequestTest); +}; + +TEST_F(ContextualJsonRequestTest, AuthenticatedRequest) { + ContextualJsonRequest::Builder builder = CreateDefaultBuilder(); + builder.SetAuthentication("0BFUSGAIA", "headerstuff") + .SetContentUrl(GURL("http://my-url.test")) + .Build(); + + EXPECT_THAT(builder.PreviewRequestHeadersForTesting(), + StrEq("Content-Type: application/json; charset=UTF-8\r\n" + "Authorization: headerstuff\r\n" + "\r\n")); + EXPECT_THAT(builder.PreviewRequestBodyForTesting(), + EqualsJSON("{" + " \"categories\": [ \"RELATED_ARTICLES\"," + " \"PUBLIC_DEBATE\" ]," + " \"url\": \"http://my-url.test/\"" + "}")); +} + +} // namespace internal + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/fetch.py b/chromium/components/ntp_snippets/remote/fetch.py index aa753f3e7e4..d5624c05060 100755 --- a/chromium/components/ntp_snippets/remote/fetch.py +++ b/chromium/components/ntp_snippets/remote/fetch.py @@ -43,6 +43,8 @@ API_PATH = "/v1/suggestions/fetch" def main(): + default_lang = os.environ.get("LANG", "en_US").split(".")[0] + parser = argparse.ArgumentParser( description="fetch articles from server", parents=[oauth2client.tools.argparser]) @@ -51,6 +53,9 @@ def main(): help="component to fetch from (default: prod)") parser.add_argument("-x", "--experiment", action="append", type=int, help="include an experiment ID") + parser.add_argument("-l", "--ui-language", default=default_lang, + help="language code (default: %s)" % default_lang) + parser.add_argument("--ip", help="fake IP address") parser.add_argument("--api-key", type=str, help="API key to use for unauthenticated requests" " (default: use official key)") @@ -167,6 +172,9 @@ def PostRequest(args): if args.experiment: headers["X-Client-Data"] = EncodeExperiments(args.experiment) + if args.ip is not None: + headers["X-User-IP"] = args.ip + if args.signed_in: if args.client: client_id, client_secret = args.client.split(",") @@ -180,7 +188,11 @@ def PostRequest(args): api_key = GetAPIKey() url += "?key=" + api_key - return requests.post(url, headers=headers) + data = { + "uiLanguage": args.ui_language, + } + + return requests.post(url, headers=headers, data=data) def Authenticate(args, headers, client_id, client_secret): @@ -199,7 +211,7 @@ def PrintShortResponse(j): now = datetime.datetime.now() for category in j["categories"]: print("%s: " % category["localizedTitle"]) - for suggestion in category["suggestions"]: + for suggestion in category.get("suggestions", []): attribution = suggestion["attribution"] title = suggestion["title"] full_url = suggestion["fullPageUrl"] diff --git a/chromium/components/ntp_snippets/remote/json_request.cc b/chromium/components/ntp_snippets/remote/json_request.cc index 2ea9f4f7d57..2be9a6b5d27 100644 --- a/chromium/components/ntp_snippets/remote/json_request.cc +++ b/chromium/components/ntp_snippets/remote/json_request.cc @@ -37,11 +37,11 @@ #include "third_party/icu/source/common/unicode/utypes.h" #include "ui/base/l10n/l10n_util.h" +using language::UrlLanguageHistogram; using net::URLFetcher; using net::URLRequestContextGetter; using net::HttpRequestHeaders; using net::URLRequestStatus; -using translate::LanguageModel; namespace ntp_snippets { @@ -52,7 +52,7 @@ namespace { // Variation parameter for disabling the retry. const char kBackground5xxRetriesName[] = "background_5xx_retries_count"; -// Variation parameter for sending LanguageModel info to the server. +// Variation parameter for sending UrlLanguageHistogram info to the server. const char kSendTopLanguagesName[] = "send_top_languages"; // Variation parameter for sending UserClassifier info to the server. @@ -108,7 +108,7 @@ std::string ISO639FromPosixLocale(const std::string& locale) { } void AppendLanguageInfoToList(base::ListValue* list, - const LanguageModel::LanguageInfo& info) { + const UrlLanguageHistogram::LanguageInfo& info) { auto lang = base::MakeUnique<base::DictionaryValue>(); lang->SetString("language", info.language_code); lang->SetDouble("frequency", info.frequency); @@ -216,7 +216,7 @@ void JsonRequest::OnJsonError(const std::string& error) { /*error_details=*/base::StringPrintf(" (error %s)", error.c_str())); } -JsonRequest::Builder::Builder() : language_model_(nullptr) {} +JsonRequest::Builder::Builder() : language_histogram_(nullptr) {} JsonRequest::Builder::Builder(JsonRequest::Builder&&) = default; JsonRequest::Builder::~Builder() = default; @@ -246,9 +246,9 @@ JsonRequest::Builder& JsonRequest::Builder::SetAuthentication( return *this; } -JsonRequest::Builder& JsonRequest::Builder::SetLanguageModel( - const translate::LanguageModel* language_model) { - language_model_ = language_model; +JsonRequest::Builder& JsonRequest::Builder::SetLanguageHistogram( + const language::UrlLanguageHistogram* language_histogram) { + language_histogram_ = language_histogram; return *this; } @@ -326,8 +326,8 @@ std::string JsonRequest::Builder::BuildBody() const { request->SetString("userActivenessClass", user_class_); } - translate::LanguageModel::LanguageInfo ui_language; - translate::LanguageModel::LanguageInfo other_top_language; + language::UrlLanguageHistogram::LanguageInfo ui_language; + language::UrlLanguageHistogram::LanguageInfo other_top_language; PrepareLanguages(&ui_language, &other_top_language); if (ui_language.frequency != 0 || other_top_language.frequency != 0) { auto language_list = base::MakeUnique<base::ListValue>(); @@ -366,8 +366,8 @@ std::unique_ptr<net::URLFetcher> JsonRequest::Builder::BuildURLFetcher( "request." data: "The Chromium UI language, as well as a second language the user " - "understands, based on translate::LanguageModel. For signed-in " - "users, the requests is authenticated." + "understands, based on language::UrlLanguageHistogram. For " + "signed-in users, the requests is authenticated." destination: GOOGLE_OWNED_SERVICE } policy { @@ -402,12 +402,12 @@ std::unique_ptr<net::URLFetcher> JsonRequest::Builder::BuildURLFetcher( } void JsonRequest::Builder::PrepareLanguages( - translate::LanguageModel::LanguageInfo* ui_language, - translate::LanguageModel::LanguageInfo* other_top_language) const { + language::UrlLanguageHistogram::LanguageInfo* ui_language, + language::UrlLanguageHistogram::LanguageInfo* other_top_language) const { // TODO(jkrcal): Add language model factory for iOS and add fakes to tests so - // that |language_model| is never nullptr. Remove this check and add a DCHECK - // into the constructor. - if (!language_model_ || !IsSendingTopLanguagesEnabled()) { + // that |language_histogram| is never nullptr. Remove this check and add a + // DCHECK into the constructor. + if (!language_histogram_ || !IsSendingTopLanguagesEnabled()) { return; } @@ -415,11 +415,11 @@ void JsonRequest::Builder::PrepareLanguages( ui_language->language_code = ISO639FromPosixLocale( PosixLocaleFromBCP47Language(params_.language_code)); ui_language->frequency = - language_model_->GetLanguageFrequency(ui_language->language_code); + language_histogram_->GetLanguageFrequency(ui_language->language_code); - std::vector<LanguageModel::LanguageInfo> top_languages = - language_model_->GetTopLanguages(); - for (const LanguageModel::LanguageInfo& info : top_languages) { + std::vector<UrlLanguageHistogram::LanguageInfo> top_languages = + language_histogram_->GetTopLanguages(); + for (const UrlLanguageHistogram::LanguageInfo& info : top_languages) { if (info.language_code != ui_language->language_code) { *other_top_language = info; diff --git a/chromium/components/ntp_snippets/remote/json_request.h b/chromium/components/ntp_snippets/remote/json_request.h index 849df611b6b..e7770e1376a 100644 --- a/chromium/components/ntp_snippets/remote/json_request.h +++ b/chromium/components/ntp_snippets/remote/json_request.h @@ -13,9 +13,9 @@ #include "base/memory/weak_ptr.h" #include "base/optional.h" #include "base/time/time.h" +#include "components/language/core/browser/url_language_histogram.h" #include "components/ntp_snippets/remote/request_params.h" #include "components/ntp_snippets/status.h" -#include "components/translate/core/browser/language_model.h" #include "google_apis/gaia/oauth2_token_service.h" #include "net/http/http_request_headers.h" @@ -70,9 +70,10 @@ class JsonRequest : public net::URLFetcherDelegate { Builder& SetAuthentication(const std::string& account_id, const std::string& auth_header); Builder& SetCreationTime(base::TimeTicks creation_time); - // The language_model borrowed from the fetcher needs to stay alive until - // the request body is built. - Builder& SetLanguageModel(const translate::LanguageModel* language_model); + // The language_histogram borrowed from the fetcher needs to stay alive + // until the request body is built. + Builder& SetLanguageHistogram( + const language::UrlLanguageHistogram* language_histogram); Builder& SetParams(const RequestParams& params); Builder& SetParseJsonCallback(ParseJSONCallback callback); // The clock borrowed from the fetcher will be injected into the @@ -104,8 +105,8 @@ class JsonRequest : public net::URLFetcherDelegate { const std::string& body) const; void PrepareLanguages( - translate::LanguageModel::LanguageInfo* ui_language, - translate::LanguageModel::LanguageInfo* other_top_language) const; + language::UrlLanguageHistogram::LanguageInfo* ui_language, + language::UrlLanguageHistogram::LanguageInfo* other_top_language) const; // Only required, if the request needs to be sent. std::string auth_header_; @@ -118,7 +119,7 @@ class JsonRequest : public net::URLFetcherDelegate { // Optional properties. std::string obfuscated_gaia_id_; std::string user_class_; - const translate::LanguageModel* language_model_; + const language::UrlLanguageHistogram* language_histogram_; DISALLOW_COPY_AND_ASSIGN(Builder); }; diff --git a/chromium/components/ntp_snippets/remote/json_request_unittest.cc b/chromium/components/ntp_snippets/remote/json_request_unittest.cc index 3791ccd5059..0e52824c3b0 100644 --- a/chromium/components/ntp_snippets/remote/json_request_unittest.cc +++ b/chromium/components/ntp_snippets/remote/json_request_unittest.cc @@ -69,20 +69,21 @@ class JsonRequestTest : public testing::Test { clock_(mock_task_runner_->GetMockClock()), request_context_getter_( new net::TestURLRequestContextGetter(mock_task_runner_.get())) { - translate::LanguageModel::RegisterProfilePrefs(pref_service_->registry()); + language::UrlLanguageHistogram::RegisterProfilePrefs( + pref_service_->registry()); } - std::unique_ptr<translate::LanguageModel> MakeLanguageModel( + std::unique_ptr<language::UrlLanguageHistogram> MakeLanguageHistogram( const std::set<std::string>& codes) { - std::unique_ptr<translate::LanguageModel> language_model = - base::MakeUnique<translate::LanguageModel>(pref_service_.get()); + std::unique_ptr<language::UrlLanguageHistogram> language_histogram = + base::MakeUnique<language::UrlLanguageHistogram>(pref_service_.get()); // There must be at least 10 visits before the top languages are defined. for (int i = 0; i < 10; i++) { for (const std::string& code : codes) { - language_model->OnPageVisited(code); + language_histogram->OnPageVisited(code); } } - return language_model; + return language_histogram; } JsonRequest::Builder CreateMinimalBuilder() { @@ -221,12 +222,12 @@ TEST_F(JsonRequestTest, BuildRequestNoUserClass) { TEST_F(JsonRequestTest, BuildRequestWithTwoLanguages) { JsonRequest::Builder builder; - std::unique_ptr<translate::LanguageModel> language_model = - MakeLanguageModel({"de", "en"}); + std::unique_ptr<language::UrlLanguageHistogram> language_histogram = + MakeLanguageHistogram({"de", "en"}); RequestParams params; params.interactive_request = true; params.language_code = "en"; - builder.SetParams(params).SetLanguageModel(language_model.get()); + builder.SetParams(params).SetLanguageHistogram(language_histogram.get()); EXPECT_THAT(builder.PreviewRequestBodyForTesting(), EqualsJSON("{" @@ -248,12 +249,12 @@ TEST_F(JsonRequestTest, BuildRequestWithTwoLanguages) { TEST_F(JsonRequestTest, BuildRequestWithUILanguageOnly) { JsonRequest::Builder builder; - std::unique_ptr<translate::LanguageModel> language_model = - MakeLanguageModel({"en"}); + std::unique_ptr<language::UrlLanguageHistogram> language_histogram = + MakeLanguageHistogram({"en"}); RequestParams params; params.interactive_request = true; params.language_code = "en"; - builder.SetParams(params).SetLanguageModel(language_model.get()); + builder.SetParams(params).SetLanguageHistogram(language_histogram.get()); EXPECT_THAT(builder.PreviewRequestBodyForTesting(), EqualsJSON("{" diff --git a/chromium/components/ntp_snippets/remote/json_to_categories.cc b/chromium/components/ntp_snippets/remote/json_to_categories.cc new file mode 100644 index 00000000000..17c0c59f006 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/json_to_categories.cc @@ -0,0 +1,135 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/json_to_categories.h" + +#include "base/optional.h" +#include "base/strings/utf_string_conversions.h" +#include "components/strings/grit/components_strings.h" +#include "ui/base/l10n/l10n_util.h" + +namespace ntp_snippets { + +namespace { + +// Creates suggestions from dictionary values in |list| and adds them to +// |suggestions|. Returns true on success, false if anything went wrong. +bool AddSuggestionsFromListValue(int remote_category_id, + const base::ListValue& list, + RemoteSuggestion::PtrVector* suggestions, + const base::Time& fetch_time) { + for (const auto& value : list) { + const base::DictionaryValue* dict = nullptr; + if (!value.GetAsDictionary(&dict)) { + return false; + } + + std::unique_ptr<RemoteSuggestion> suggestion; + + suggestion = RemoteSuggestion::CreateFromContentSuggestionsDictionary( + *dict, remote_category_id, fetch_time); + + if (!suggestion) { + return false; + } + + suggestions->push_back(std::move(suggestion)); + } + return true; +} + +} // namespace + +FetchedCategory::FetchedCategory(Category c, CategoryInfo&& info) + : category(c), info(info) {} + +FetchedCategory::FetchedCategory(FetchedCategory&&) = default; + +FetchedCategory::~FetchedCategory() = default; + +FetchedCategory& FetchedCategory::operator=(FetchedCategory&&) = default; + +CategoryInfo BuildArticleCategoryInfo( + const base::Optional<base::string16>& title) { + return CategoryInfo( + title.has_value() ? title.value() + : l10n_util::GetStringUTF16( + IDS_NTP_ARTICLE_SUGGESTIONS_SECTION_HEADER), + ContentSuggestionsCardLayout::FULL_CARD, + ContentSuggestionsAdditionalAction::FETCH, + /*show_if_empty=*/true, + l10n_util::GetStringUTF16(IDS_NTP_ARTICLE_SUGGESTIONS_SECTION_EMPTY)); +} + +CategoryInfo BuildRemoteCategoryInfo(const base::string16& title, + bool allow_fetching_more_results) { + ContentSuggestionsAdditionalAction action = + ContentSuggestionsAdditionalAction::NONE; + if (allow_fetching_more_results) { + action = ContentSuggestionsAdditionalAction::FETCH; + } + return CategoryInfo( + title, ContentSuggestionsCardLayout::FULL_CARD, action, + /*show_if_empty=*/false, + // TODO(tschumann): The message for no-articles is likely wrong + // and needs to be added to the stubby protocol if we want to + // support it. + l10n_util::GetStringUTF16(IDS_NTP_ARTICLE_SUGGESTIONS_SECTION_EMPTY)); +} + +bool JsonToCategories(const base::Value& parsed, + FetchedCategoriesVector* categories, + const base::Time& fetch_time) { + const base::DictionaryValue* top_dict = nullptr; + if (!parsed.GetAsDictionary(&top_dict)) { + return false; + } + + const base::ListValue* categories_value = nullptr; + if (!top_dict->GetList("categories", &categories_value)) { + return false; + } + + for (const auto& v : *categories_value) { + std::string utf8_title; + int remote_category_id = -1; + const base::DictionaryValue* category_value = nullptr; + if (!(v.GetAsDictionary(&category_value) && + category_value->GetString("localizedTitle", &utf8_title) && + category_value->GetInteger("id", &remote_category_id) && + (remote_category_id > 0))) { + return false; + } + + RemoteSuggestion::PtrVector suggestions; + const base::ListValue* suggestions_list = nullptr; + // Absence of a list of suggestions is treated as an empty list, which + // is permissible. + if (category_value->GetList("suggestions", &suggestions_list)) { + if (!AddSuggestionsFromListValue(remote_category_id, *suggestions_list, + &suggestions, fetch_time)) { + return false; + } + } + Category category = Category::FromRemoteCategory(remote_category_id); + if (category.IsKnownCategory(KnownCategories::ARTICLES)) { + categories->push_back(FetchedCategory( + category, BuildArticleCategoryInfo(base::UTF8ToUTF16(utf8_title)))); + } else { + // TODO(tschumann): Right now, the backend does not yet populate this + // field. Make it mandatory once the backends provide it. + bool allow_fetching_more_results = false; + category_value->GetBoolean("allowFetchingMoreResults", + &allow_fetching_more_results); + categories->push_back(FetchedCategory( + category, BuildRemoteCategoryInfo(base::UTF8ToUTF16(utf8_title), + allow_fetching_more_results))); + } + categories->back().suggestions = std::move(suggestions); + } + + return true; +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/json_to_categories.h b/chromium/components/ntp_snippets/remote/json_to_categories.h new file mode 100644 index 00000000000..66eb80958fc --- /dev/null +++ b/chromium/components/ntp_snippets/remote/json_to_categories.h @@ -0,0 +1,45 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_HELPER_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_HELPER_H_ + +#include "base/optional.h" +#include "base/time/time.h" +#include "base/values.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_info.h" +#include "components/ntp_snippets/remote/remote_suggestion.h" + +namespace ntp_snippets { + +struct FetchedCategory { + Category category; + CategoryInfo info; + RemoteSuggestion::PtrVector suggestions; + + FetchedCategory(Category c, CategoryInfo&& info); + FetchedCategory(FetchedCategory&&); + ~FetchedCategory(); + FetchedCategory& operator=(FetchedCategory&&); +}; + +using FetchedCategoriesVector = std::vector<FetchedCategory>; + +// Provides the CategoryInfo data for article suggestions. If |title| is +// nullopt, then the default, hard-coded title will be used. +CategoryInfo BuildArticleCategoryInfo( + const base::Optional<base::string16>& title); + +// Provides the CategoryInfo data for other remote suggestions. +CategoryInfo BuildRemoteCategoryInfo(const base::string16& title, + bool allow_fetching_more_results); + +bool JsonToCategories(const base::Value& parsed, + FetchedCategoriesVector* categories, + const base::Time& fetch_time); + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_HELPER_H_ diff --git a/chromium/components/ntp_snippets/remote/persistent_scheduler.h b/chromium/components/ntp_snippets/remote/persistent_scheduler.h index 080ceb0b601..727c2adb2fb 100644 --- a/chromium/components/ntp_snippets/remote/persistent_scheduler.h +++ b/chromium/components/ntp_snippets/remote/persistent_scheduler.h @@ -15,13 +15,8 @@ namespace ntp_snippets { // schedule independent of whether Chrome is running at that moment. // // Once per period, the concrete implementation should call -// RemoteSuggestionsScheduler::OnFetchDue() where the scheduler object is -// obtained from ContentSuggestionsService. -// -// The implementation may also call -// RemoteSuggestionsScheduler::RescheduleFetching() when its own current -// schedule got corrupted for whatever reason and needs to be applied again -// (in turn, this will result in calling Schedule() on the implementation). +// RemoteSuggestionsScheduler::OnPersistentSchedulerWakeUp() where the scheduler +// object is obtained from ContentSuggestionsService. class PersistentScheduler { public: // Schedule periodic fetching of remote suggestions, with different periods diff --git a/chromium/components/ntp_snippets/remote/prefetched_pages_tracker.h b/chromium/components/ntp_snippets/remote/prefetched_pages_tracker.h new file mode 100644 index 00000000000..b81e78c1677 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/prefetched_pages_tracker.h @@ -0,0 +1,35 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_PREFETCHED_PAGES_TRACKER_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_PREFETCHED_PAGES_TRACKER_H_ + +#include <string> + +#include "base/callback.h" +#include "url/gurl.h" + +namespace ntp_snippets { + +// Synchronously answers whether there is a prefetched offline page for a given +// URL. +class PrefetchedPagesTracker { + public: + virtual ~PrefetchedPagesTracker() = default; + + // Whether the tracker has finished initialization. + virtual bool IsInitialized() const = 0; + + // Add a callback, which will be called when the initialization is completed. + // If the tracker has been initialized already, the callback is called + // immediately. + virtual void AddInitializationCompletedCallback( + base::OnceCallback<void()> callback) = 0; + + virtual bool PrefetchedOfflinePageExists(const GURL& url) const = 0; +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_PREFETCHED_PAGES_TRACKER_H_ diff --git a/chromium/components/ntp_snippets/remote/prefetched_pages_tracker_impl.cc b/chromium/components/ntp_snippets/remote/prefetched_pages_tracker_impl.cc new file mode 100644 index 00000000000..a595715a321 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/prefetched_pages_tracker_impl.cc @@ -0,0 +1,121 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/prefetched_pages_tracker_impl.h" + +#include "base/bind.h" +#include "components/offline_pages/core/client_namespace_constants.h" + +using offline_pages::OfflinePageItem; +using offline_pages::OfflinePageModel; +using offline_pages::OfflinePageModelQuery; +using offline_pages::OfflinePageModelQueryBuilder; + +namespace ntp_snippets { + +namespace { + +std::unique_ptr<OfflinePageModelQuery> BuildPrefetchedPagesQuery( + OfflinePageModel* model) { + OfflinePageModelQueryBuilder builder; + builder.RequireNamespace(offline_pages::kSuggestedArticlesNamespace); + return builder.Build(model->GetPolicyController()); +} + +bool IsOfflineItemPrefetchedPage(const OfflinePageItem& offline_page_item) { + return offline_page_item.client_id.name_space == + offline_pages::kSuggestedArticlesNamespace; +} + +const GURL& GetOfflinePageUrl(const OfflinePageItem& offline_page_item) { + return offline_page_item.original_url != GURL() + ? offline_page_item.original_url + : offline_page_item.url; +} + +} // namespace + +PrefetchedPagesTrackerImpl::PrefetchedPagesTrackerImpl( + OfflinePageModel* offline_page_model) + : initialized_(false), + offline_page_model_(offline_page_model), + weak_ptr_factory_(this) { + DCHECK(offline_page_model_); + // If Offline Page model is not loaded yet, it will process our query + // once it has finished loading. + offline_page_model_->GetPagesMatchingQuery( + BuildPrefetchedPagesQuery(offline_page_model), + base::Bind(&PrefetchedPagesTrackerImpl::Initialize, + weak_ptr_factory_.GetWeakPtr())); +} + +PrefetchedPagesTrackerImpl::~PrefetchedPagesTrackerImpl() { + offline_page_model_->RemoveObserver(this); +} + +bool PrefetchedPagesTrackerImpl::IsInitialized() const { + return initialized_; +} + +void PrefetchedPagesTrackerImpl::AddInitializationCompletedCallback( + base::OnceCallback<void()> callback) { + if (IsInitialized()) { + std::move(callback).Run(); + } + initialization_completed_callbacks_.push_back(std::move(callback)); +} + +bool PrefetchedPagesTrackerImpl::PrefetchedOfflinePageExists( + const GURL& url) const { + DCHECK(initialized_); + return prefetched_urls_.count(url) == 1; +} + +void PrefetchedPagesTrackerImpl::OfflinePageModelLoaded( + OfflinePageModel* model) { + // Ignored. Offline Page model delayes our requests until it is loaded. +} + +void PrefetchedPagesTrackerImpl::OfflinePageAdded( + OfflinePageModel* model, + const OfflinePageItem& added_page) { + if (IsOfflineItemPrefetchedPage(added_page)) { + AddOfflinePage(added_page); + } +} + +void PrefetchedPagesTrackerImpl::OfflinePageDeleted( + const offline_pages::OfflinePageModel::DeletedPageInfo& page_info) { + std::map<int64_t, GURL>::iterator it = + offline_id_to_url_mapping_.find(page_info.offline_id); + if (it != offline_id_to_url_mapping_.end()) { + DCHECK(prefetched_urls_.count(it->second)); + prefetched_urls_.erase(it->second); + offline_id_to_url_mapping_.erase(it); + } +} + +void PrefetchedPagesTrackerImpl::Initialize( + const std::vector<OfflinePageItem>& all_prefetched_offline_pages) { + for (const OfflinePageItem& item : all_prefetched_offline_pages) { + DCHECK(IsOfflineItemPrefetchedPage(item)); + AddOfflinePage(item); + } + + initialized_ = true; + offline_page_model_->AddObserver(this); + for (auto& callback : initialization_completed_callbacks_) { + std::move(callback).Run(); + } +} + +void PrefetchedPagesTrackerImpl::AddOfflinePage( + const OfflinePageItem& offline_page_item) { + const GURL& url = GetOfflinePageUrl(offline_page_item); + DCHECK(!prefetched_urls_.count(url)); + prefetched_urls_.insert(url); + offline_id_to_url_mapping_[offline_page_item.offline_id] = url; +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/prefetched_pages_tracker_impl.h b/chromium/components/ntp_snippets/remote/prefetched_pages_tracker_impl.h new file mode 100644 index 00000000000..32be1db182e --- /dev/null +++ b/chromium/components/ntp_snippets/remote/prefetched_pages_tracker_impl.h @@ -0,0 +1,68 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_PREFETCHED_PAGES_TRACKER_IMPL_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_PREFETCHED_PAGES_TRACKER_IMPL_H_ + +#include <map> +#include <set> +#include <string> +#include <vector> + +#include "base/memory/weak_ptr.h" +#include "components/ntp_snippets/remote/prefetched_pages_tracker.h" +#include "components/offline_pages/core/offline_page_model.h" +#include "url/gurl.h" + +namespace offline_pages { +struct OfflinePageItem; +} + +namespace ntp_snippets { + +// OfflinePageModel must outlive this class. +class PrefetchedPagesTrackerImpl + : public PrefetchedPagesTracker, + public offline_pages::OfflinePageModel::Observer { + public: + PrefetchedPagesTrackerImpl( + offline_pages::OfflinePageModel* offline_page_model); + ~PrefetchedPagesTrackerImpl() override; + + // PrefetchedPagesTracker implementation + bool IsInitialized() const override; + void AddInitializationCompletedCallback( + base::OnceCallback<void()> callback) override; + bool PrefetchedOfflinePageExists(const GURL& url) const override; + + // OfflinePageModel::Observer implementation. + void OfflinePageModelLoaded(offline_pages::OfflinePageModel* model) override; + void OfflinePageAdded( + offline_pages::OfflinePageModel* model, + const offline_pages::OfflinePageItem& added_page) override; + void OfflinePageDeleted( + const offline_pages::OfflinePageModel::DeletedPageInfo& page_info) + override; + + private: + void Initialize(const std::vector<offline_pages::OfflinePageItem>& + all_prefetched_offline_pages); + void AddOfflinePage(const offline_pages::OfflinePageItem& offline_page_item); + + bool initialized_; + offline_pages::OfflinePageModel* offline_page_model_; + + std::set<GURL> prefetched_urls_; + std::map<int64_t, GURL> offline_id_to_url_mapping_; + + std::vector<base::OnceCallback<void()>> initialization_completed_callbacks_; + + base::WeakPtrFactory<PrefetchedPagesTrackerImpl> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(PrefetchedPagesTrackerImpl); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_PREFETCHED_PAGES_TRACKER_IMPL_H_ diff --git a/chromium/components/ntp_snippets/remote/prefetched_pages_tracker_impl_unittest.cc b/chromium/components/ntp_snippets/remote/prefetched_pages_tracker_impl_unittest.cc new file mode 100644 index 00000000000..ddbb295cb14 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/prefetched_pages_tracker_impl_unittest.cc @@ -0,0 +1,238 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/prefetched_pages_tracker_impl.h" + +#include "base/bind.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/stringprintf.h" +#include "base/test/mock_callback.h" +#include "components/ntp_snippets/offline_pages/offline_pages_test_utils.h" +#include "components/offline_pages/core/client_namespace_constants.h" +#include "components/offline_pages/core/stub_offline_page_model.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ntp_snippets::test::FakeOfflinePageModel; +using offline_pages::MultipleOfflinePageItemCallback; +using offline_pages::OfflinePageItem; +using offline_pages::OfflinePageModelQuery; +using testing::_; +using testing::Eq; +using testing::SaveArg; +using testing::StrictMock; + +namespace ntp_snippets { + +namespace { + +class MockOfflinePageModel : public offline_pages::StubOfflinePageModel { + public: + ~MockOfflinePageModel() override = default; + + // GMock does not support movable-only types (unique_ptr in this case). + // Therefore, the call is redirected to a mock method without movable-only + // types. + void GetPagesMatchingQuery( + std::unique_ptr<OfflinePageModelQuery> query, + const MultipleOfflinePageItemCallback& callback) override { + GetPagesMatchingQuery(query.get(), callback); + } + + MOCK_METHOD2(GetPagesMatchingQuery, + void(OfflinePageModelQuery* query, + const MultipleOfflinePageItemCallback& callback)); +}; + +OfflinePageItem CreateOfflinePageItem(const GURL& url, + const std::string& name_space) { + static int id = 0; + ++id; + return OfflinePageItem( + url, id, offline_pages::ClientId(name_space, base::IntToString(id)), + base::FilePath::FromUTF8Unsafe( + base::StringPrintf("some/folder/%d.mhtml", id)), + 0, base::Time::Now()); +} + +} // namespace + +class PrefetchedPagesTrackerImplTest : public ::testing::Test { + public: + PrefetchedPagesTrackerImplTest() = default; + + FakeOfflinePageModel* fake_offline_page_model() { + return &fake_offline_page_model_; + } + + MockOfflinePageModel* mock_offline_page_model() { + return &mock_offline_page_model_; + } + + private: + FakeOfflinePageModel fake_offline_page_model_; + StrictMock<MockOfflinePageModel> mock_offline_page_model_; + DISALLOW_COPY_AND_ASSIGN(PrefetchedPagesTrackerImplTest); +}; + +TEST_F(PrefetchedPagesTrackerImplTest, + ShouldRetrievePrefetchedEarlierSuggestionsOnStartup) { + (*fake_offline_page_model()->mutable_items()) = { + CreateOfflinePageItem(GURL("http://prefetched.com"), + offline_pages::kSuggestedArticlesNamespace)}; + PrefetchedPagesTrackerImpl tracker(fake_offline_page_model()); + + ASSERT_FALSE( + tracker.PrefetchedOfflinePageExists(GURL("http://not_added_url.com"))); + EXPECT_TRUE( + tracker.PrefetchedOfflinePageExists(GURL("http://prefetched.com"))); +} + +TEST_F(PrefetchedPagesTrackerImplTest, + ShouldAddNewPrefetchedPagesWhenNotified) { + fake_offline_page_model()->mutable_items()->clear(); + PrefetchedPagesTrackerImpl tracker(fake_offline_page_model()); + + ASSERT_FALSE( + tracker.PrefetchedOfflinePageExists(GURL("http://prefetched.com"))); + tracker.OfflinePageAdded( + fake_offline_page_model(), + CreateOfflinePageItem(GURL("http://prefetched.com"), + offline_pages::kSuggestedArticlesNamespace)); + EXPECT_TRUE( + tracker.PrefetchedOfflinePageExists(GURL("http://prefetched.com"))); +} + +TEST_F(PrefetchedPagesTrackerImplTest, + ShouldIgnoreOtherTypesOfOfflinePagesWhenNotified) { + fake_offline_page_model()->mutable_items()->clear(); + PrefetchedPagesTrackerImpl tracker(fake_offline_page_model()); + + ASSERT_FALSE(tracker.PrefetchedOfflinePageExists( + GURL("http://manually_downloaded.com"))); + tracker.OfflinePageAdded( + fake_offline_page_model(), + CreateOfflinePageItem(GURL("http://manually_downloaded.com"), + offline_pages::kNTPSuggestionsNamespace)); + EXPECT_FALSE(tracker.PrefetchedOfflinePageExists( + GURL("http://manually_downloaded.com"))); +} + +TEST_F(PrefetchedPagesTrackerImplTest, + ShouldIgnoreOtherTypesOfOfflinePagesOnStartup) { + (*fake_offline_page_model()->mutable_items()) = { + CreateOfflinePageItem(GURL("http://manually_downloaded.com"), + offline_pages::kNTPSuggestionsNamespace)}; + PrefetchedPagesTrackerImpl tracker(fake_offline_page_model()); + + ASSERT_FALSE(tracker.PrefetchedOfflinePageExists( + GURL("http://manually_downloaded.com"))); + EXPECT_FALSE(tracker.PrefetchedOfflinePageExists( + GURL("http://manually_downloaded.com"))); +} + +TEST_F(PrefetchedPagesTrackerImplTest, ShouldDeletePrefetchedURLWhenNotified) { + const OfflinePageItem item = + CreateOfflinePageItem(GURL("http://prefetched.com"), + offline_pages::kSuggestedArticlesNamespace); + (*fake_offline_page_model()->mutable_items()) = {item}; + PrefetchedPagesTrackerImpl tracker(fake_offline_page_model()); + + ASSERT_TRUE( + tracker.PrefetchedOfflinePageExists(GURL("http://prefetched.com"))); + tracker.OfflinePageDeleted(offline_pages::OfflinePageModel::DeletedPageInfo( + item.offline_id, item.client_id, "" /* request_origin */)); + EXPECT_FALSE( + tracker.PrefetchedOfflinePageExists(GURL("http://prefetched.com"))); +} + +TEST_F(PrefetchedPagesTrackerImplTest, + ShouldIgnoreDeletionOfOtherTypeOfflinePagesWhenNotified) { + const OfflinePageItem prefetched_item = + CreateOfflinePageItem(GURL("http://prefetched.com"), + offline_pages::kSuggestedArticlesNamespace); + // The URL is intentionally the same. + const OfflinePageItem manually_downloaded_item = CreateOfflinePageItem( + GURL("http://prefetched.com"), offline_pages::kNTPSuggestionsNamespace); + (*fake_offline_page_model()->mutable_items()) = {prefetched_item, + manually_downloaded_item}; + PrefetchedPagesTrackerImpl tracker(fake_offline_page_model()); + + ASSERT_TRUE( + tracker.PrefetchedOfflinePageExists(GURL("http://prefetched.com"))); + tracker.OfflinePageDeleted(offline_pages::OfflinePageModel::DeletedPageInfo( + manually_downloaded_item.offline_id, manually_downloaded_item.client_id, + "" /* request_origin */)); + EXPECT_TRUE( + tracker.PrefetchedOfflinePageExists(GURL("http://prefetched.com"))); +} + +TEST_F(PrefetchedPagesTrackerImplTest, + ShouldReportAsNotInitializedBeforeInitialization) { + EXPECT_CALL(*mock_offline_page_model(), GetPagesMatchingQuery(_, _)); + PrefetchedPagesTrackerImpl tracker(mock_offline_page_model()); + EXPECT_FALSE(tracker.IsInitialized()); +} + +TEST_F(PrefetchedPagesTrackerImplTest, + ShouldReportAsInitializedAfterInitialization) { + MultipleOfflinePageItemCallback offline_pages_callback; + EXPECT_CALL(*mock_offline_page_model(), GetPagesMatchingQuery(_, _)) + .WillOnce(SaveArg<1>(&offline_pages_callback)); + PrefetchedPagesTrackerImpl tracker(mock_offline_page_model()); + + ASSERT_FALSE(tracker.IsInitialized()); + offline_pages_callback.Run(std::vector<OfflinePageItem>()); + EXPECT_TRUE(tracker.IsInitialized()); +} + +TEST_F(PrefetchedPagesTrackerImplTest, ShouldCallCallbackAfterInitialization) { + MultipleOfflinePageItemCallback offline_pages_callback; + EXPECT_CALL(*mock_offline_page_model(), GetPagesMatchingQuery(_, _)) + .WillOnce(SaveArg<1>(&offline_pages_callback)); + PrefetchedPagesTrackerImpl tracker(mock_offline_page_model()); + + base::MockCallback<base::OnceCallback<void()>> + mock_initialization_completed_callback; + tracker.AddInitializationCompletedCallback( + mock_initialization_completed_callback.Get()); + EXPECT_CALL(mock_initialization_completed_callback, Run()); + offline_pages_callback.Run(std::vector<OfflinePageItem>()); +} + +TEST_F(PrefetchedPagesTrackerImplTest, + ShouldCallMultipleCallbacksAfterInitialization) { + MultipleOfflinePageItemCallback offline_pages_callback; + EXPECT_CALL(*mock_offline_page_model(), GetPagesMatchingQuery(_, _)) + .WillOnce(SaveArg<1>(&offline_pages_callback)); + PrefetchedPagesTrackerImpl tracker(mock_offline_page_model()); + + base::MockCallback<base::OnceCallback<void()>> + first_mock_initialization_completed_callback, + second_mock_initialization_completed_callback; + tracker.AddInitializationCompletedCallback( + first_mock_initialization_completed_callback.Get()); + tracker.AddInitializationCompletedCallback( + second_mock_initialization_completed_callback.Get()); + EXPECT_CALL(first_mock_initialization_completed_callback, Run()); + EXPECT_CALL(second_mock_initialization_completed_callback, Run()); + offline_pages_callback.Run(std::vector<OfflinePageItem>()); +} + +TEST_F(PrefetchedPagesTrackerImplTest, + ShouldCallCallbackImmediatelyIfAlreadyInitialiazed) { + MultipleOfflinePageItemCallback offline_pages_callback; + EXPECT_CALL(*mock_offline_page_model(), GetPagesMatchingQuery(_, _)) + .WillOnce(SaveArg<1>(&offline_pages_callback)); + PrefetchedPagesTrackerImpl tracker(mock_offline_page_model()); + offline_pages_callback.Run(std::vector<OfflinePageItem>()); + + base::MockCallback<base::OnceCallback<void()>> + mock_initialization_completed_callback; + EXPECT_CALL(mock_initialization_completed_callback, Run()); + tracker.AddInitializationCompletedCallback( + mock_initialization_completed_callback.Get()); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/proto/ntp_snippets.proto b/chromium/components/ntp_snippets/remote/proto/ntp_snippets.proto index 823c4c1ce35..45decf8b264 100644 --- a/chromium/components/ntp_snippets/remote/proto/ntp_snippets.proto +++ b/chromium/components/ntp_snippets/remote/proto/ntp_snippets.proto @@ -27,6 +27,12 @@ message SnippetProto { optional int32 remote_category_id = 10; // The time when the snippet was fetched from the server. optional int64 fetch_date = 11; + + enum ContentType { + UNKNOWN = 0; + VIDEO = 1; + } + optional ContentType content_type = 12 [default = UNKNOWN]; } message SnippetImageProto { diff --git a/chromium/components/ntp_snippets/remote/remote_suggestion.cc b/chromium/components/ntp_snippets/remote/remote_suggestion.cc index d1159433740..fc0e4a3bcec 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestion.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestion.cc @@ -4,14 +4,12 @@ #include "components/ntp_snippets/remote/remote_suggestion.h" -#include "base/feature_list.h" #include "base/memory/ptr_util.h" #include "base/strings/string_number_conversions.h" #include "base/strings/stringprintf.h" #include "base/strings/utf_string_conversions.h" #include "base/values.h" #include "components/ntp_snippets/category.h" -#include "components/ntp_snippets/features.h" #include "components/ntp_snippets/remote/proto/ntp_snippets.pb.h" namespace { @@ -89,7 +87,8 @@ RemoteSuggestion::RemoteSuggestion(const std::vector<std::string>& ids, score_(0), is_dismissed_(false), remote_category_id_(remote_category_id), - should_notify_(false) {} + should_notify_(false), + content_type_(ContentType::UNKNOWN) {} RemoteSuggestion::~RemoteSuggestion() = default; @@ -269,6 +268,21 @@ RemoteSuggestion::CreateFromContentSuggestionsDictionary( } } + // In the JSON dictionary contentType is an optional field. The field + // content_type_ of the class |RemoteSuggestion| is by default initialized to + // ContentType::UNKNOWN. + std::string content_type; + if (dict.GetString("contentType", &content_type)) { + if (content_type == "VIDEO") { + snippet->content_type_ = ContentType::VIDEO; + } else { + // The supported values are: VIDEO, UNKNOWN. Therefore if the field is + // present the value has to be "UNKNOWN" here. + DCHECK_EQ(content_type, "UNKNOWN"); + snippet->content_type_ = ContentType::UNKNOWN; + } + } + return snippet; } @@ -328,21 +342,9 @@ std::unique_ptr<RemoteSuggestion> RemoteSuggestion::CreateFromProto( snippet->fetch_date_ = base::Time::FromInternalValue(proto.fetch_date()); } - return snippet; -} - -// static -std::unique_ptr<RemoteSuggestion> RemoteSuggestion::CreateForTesting( - const std::string& id, - int remote_category_id, - const GURL& url, - const std::string& publisher_name, - const GURL& amp_url) { - auto snippet = - MakeUnique(std::vector<std::string>(1, id), remote_category_id); - snippet->url_ = url; - snippet->publisher_name_ = publisher_name; - snippet->amp_url_ = amp_url; + if (proto.content_type() == SnippetProto_ContentType_VIDEO) { + snippet->content_type_ = ContentType::VIDEO; + } return snippet; } @@ -383,14 +385,17 @@ SnippetProto RemoteSuggestion::ToProto() const { if (!fetch_date_.is_null()) { result.set_fetch_date(fetch_date_.ToInternalValue()); } + + if (content_type_ == ContentType::VIDEO) { + result.set_content_type(SnippetProto_ContentType_VIDEO); + } return result; } ContentSuggestion RemoteSuggestion::ToContentSuggestion( Category category) const { GURL url = url_; - bool use_amp = base::FeatureList::IsEnabled(kPreferAmpUrlsFeature) && - !amp_url_.is_empty(); + bool use_amp = !amp_url_.is_empty(); if (use_amp) { url = amp_url_; } @@ -412,6 +417,9 @@ ContentSuggestion RemoteSuggestion::ToContentSuggestion( base::MakeUnique<NotificationExtra>(extra)); } suggestion.set_fetch_date(fetch_date_); + if (content_type_ == ContentType::VIDEO) { + suggestion.set_is_video_suggestion(true); + } return suggestion; } diff --git a/chromium/components/ntp_snippets/remote/remote_suggestion.h b/chromium/components/ntp_snippets/remote/remote_suggestion.h index 615c4fbe377..97bda87246d 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestion.h +++ b/chromium/components/ntp_snippets/remote/remote_suggestion.h @@ -31,6 +31,8 @@ class RemoteSuggestion { public: using PtrVector = std::vector<std::unique_ptr<RemoteSuggestion>>; + enum ContentType { UNKNOWN, VIDEO }; + ~RemoteSuggestion(); // Creates a RemoteSuggestion from a dictionary, as returned by Chrome Reader. @@ -54,14 +56,6 @@ class RemoteSuggestion { static std::unique_ptr<RemoteSuggestion> CreateFromProto( const SnippetProto& proto); - // TODO(treib): Make tests use the public interface and remove this. - static std::unique_ptr<RemoteSuggestion> CreateForTesting( - const std::string& id, - int remote_category_id, - const GURL& url, - const std::string& publisher_name, - const GURL& amp_url); - // Creates a protocol buffer corresponding to this suggestion, for persisting. SnippetProto ToProto() const; @@ -115,6 +109,8 @@ class RemoteSuggestion { bool should_notify() const { return should_notify_; } base::Time notification_deadline() const { return notification_deadline_; } + ContentType content_type() const { return content_type_; } + bool is_dismissed() const { return is_dismissed_; } void set_dismissed(bool dismissed) { is_dismissed_ = dismissed; } @@ -156,6 +152,8 @@ class RemoteSuggestion { bool should_notify_; base::Time notification_deadline_; + ContentType content_type_; + // The time when the remote suggestion was fetched from the server. base::Time fetch_date_; diff --git a/chromium/components/ntp_snippets/remote/remote_suggestion_unittest.cc b/chromium/components/ntp_snippets/remote/remote_suggestion_unittest.cc index e68e1065db3..bc79f14dddd 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestion_unittest.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestion_unittest.cc @@ -400,6 +400,7 @@ TEST(RemoteSuggestionTest, CreateFromProtoToProtoRoundtrip) { proto.set_dismissed(false); proto.set_remote_category_id(1); proto.set_fetch_date(1476364691); + proto.set_content_type(SnippetProto_ContentType_VIDEO); auto* source = proto.add_sources(); source->set_url("http://cool-suggestions.com/"); source->set_publisher_name("Great Suggestions Inc."); @@ -565,5 +566,40 @@ TEST(RemoteSuggestionTest, ToContentSuggestionWithNotificationInfo) { Eq(1467291697000)); } +TEST(RemoteSuggestionTest, ToContentSuggestionWithContentTypeVideo) { + auto json = ContentSuggestionSnippet(); + json->SetString("contentType", "VIDEO"); + auto snippet = RemoteSuggestion::CreateFromContentSuggestionsDictionary( + *json, 0, base::Time()); + ASSERT_THAT(snippet, NotNull()); + ContentSuggestion content_suggestion = snippet->ToContentSuggestion( + Category::FromKnownCategory(KnownCategories::ARTICLES)); + + EXPECT_THAT(content_suggestion.is_video_suggestion(), Eq(true)); +} + +TEST(RemoteSuggestionTest, ToContentSuggestionWithContentTypeUnknown) { + auto json = ContentSuggestionSnippet(); + json->SetString("contentType", "UNKNOWN"); + auto snippet = RemoteSuggestion::CreateFromContentSuggestionsDictionary( + *json, 0, base::Time()); + ASSERT_THAT(snippet, NotNull()); + ContentSuggestion content_suggestion = snippet->ToContentSuggestion( + Category::FromKnownCategory(KnownCategories::ARTICLES)); + + EXPECT_THAT(content_suggestion.is_video_suggestion(), Eq(false)); +} + +TEST(RemoteSuggestionTest, ToContentSuggestionWithMissingContentType) { + auto json = ContentSuggestionSnippet(); + auto snippet = RemoteSuggestion::CreateFromContentSuggestionsDictionary( + *json, 0, base::Time()); + ASSERT_THAT(snippet, NotNull()); + ContentSuggestion content_suggestion = snippet->ToContentSuggestion( + Category::FromKnownCategory(KnownCategories::ARTICLES)); + + EXPECT_THAT(content_suggestion.is_video_suggestion(), Eq(false)); +} + } // namespace } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_database.h b/chromium/components/ntp_snippets/remote/remote_suggestions_database.h index 648b124d768..34896ee5f73 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_database.h +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_database.h @@ -28,6 +28,7 @@ namespace ntp_snippets { class SnippetImageProto; class SnippetProto; +// TODO(gaschler): implement a Fake version for testing class RemoteSuggestionsDatabase { public: using SnippetsCallback = base::Callback<void(RemoteSuggestion::PtrVector)>; diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_database_unittest.cc b/chromium/components/ntp_snippets/remote/remote_suggestions_database_unittest.cc index fd3e732767a..df46935a19b 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_database_unittest.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_database_unittest.cc @@ -41,13 +41,20 @@ bool operator==(const RemoteSuggestion& lhs, const RemoteSuggestion& rhs) { namespace { std::unique_ptr<RemoteSuggestion> CreateTestSuggestion() { - return RemoteSuggestion::CreateForTesting( - "http://localhost", kArticlesRemoteId, GURL("http://localhost"), - "Publisher", GURL("http://amp")); + SnippetProto proto; + proto.add_ids("http://localhost"); + proto.set_remote_category_id(1); // Articles + auto* source = proto.add_sources(); + source->set_url("http://localhost"); + source->set_publisher_name("Publisher"); + source->set_amp_url("http://amp"); + return RemoteSuggestion::CreateFromProto(proto); } -MATCHER_P(SnippetEq, snippet, "") { - return *arg == *snippet; +// Eq matcher has to store the expected value, but RemoteSuggestion is movable- +// only. +MATCHER_P(PointeeEq, ptr_to_expected, "") { + return *arg == *ptr_to_expected; } } // namespace @@ -159,7 +166,7 @@ TEST_F(RemoteSuggestionsDatabaseTest, Save) { // Make sure they're there. EXPECT_CALL(*this, - OnSnippetsLoadedImpl(ElementsAre(SnippetEq(snippet.get())))); + OnSnippetsLoadedImpl(ElementsAre(PointeeEq(snippet.get())))); db()->LoadSnippets( base::Bind(&RemoteSuggestionsDatabaseTest::OnSnippetsLoaded, base::Unretained(this))); @@ -191,7 +198,7 @@ TEST_F(RemoteSuggestionsDatabaseTest, SavePersist) { CreateDatabase(); EXPECT_CALL(*this, - OnSnippetsLoadedImpl(ElementsAre(SnippetEq(snippet.get())))); + OnSnippetsLoadedImpl(ElementsAre(PointeeEq(snippet.get())))); db()->LoadSnippets( base::Bind(&RemoteSuggestionsDatabaseTest::OnSnippetsLoaded, base::Unretained(this))); @@ -218,7 +225,7 @@ TEST_F(RemoteSuggestionsDatabaseTest, Update) { // Make sure we get the updated version. EXPECT_CALL(*this, - OnSnippetsLoadedImpl(ElementsAre(SnippetEq(snippet.get())))); + OnSnippetsLoadedImpl(ElementsAre(PointeeEq(snippet.get())))); db()->LoadSnippets( base::Bind(&RemoteSuggestionsDatabaseTest::OnSnippetsLoaded, base::Unretained(this))); @@ -237,7 +244,7 @@ TEST_F(RemoteSuggestionsDatabaseTest, Delete) { // Make sure it's there. EXPECT_CALL(*this, - OnSnippetsLoadedImpl(ElementsAre(SnippetEq(snippet.get())))); + OnSnippetsLoadedImpl(ElementsAre(PointeeEq(snippet.get())))); db()->LoadSnippets( base::Bind(&RemoteSuggestionsDatabaseTest::OnSnippetsLoaded, base::Unretained(this))); @@ -271,7 +278,7 @@ TEST_F(RemoteSuggestionsDatabaseTest, DeleteSnippetDoesNotDeleteImage) { // Make sure they're there. EXPECT_CALL(*this, - OnSnippetsLoadedImpl(ElementsAre(SnippetEq(snippet.get())))); + OnSnippetsLoadedImpl(ElementsAre(PointeeEq(snippet.get())))); db()->LoadSnippets( base::Bind(&RemoteSuggestionsDatabaseTest::OnSnippetsLoaded, base::Unretained(this))); diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher.cc b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher.cc index b07a116dfe3..183519908a0 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher.cc @@ -4,172 +4,19 @@ #include "components/ntp_snippets/remote/remote_suggestions_fetcher.h" -#include <cstdlib> -#include <utility> - -#include "base/files/file_path.h" -#include "base/files/file_util.h" -#include "base/memory/ptr_util.h" -#include "base/metrics/histogram_macros.h" -#include "base/metrics/sparse_histogram.h" -#include "base/path_service.h" -#include "base/strings/stringprintf.h" -#include "base/strings/utf_string_conversions.h" -#include "base/time/default_clock.h" -#include "base/time/time.h" -#include "base/values.h" -#include "components/data_use_measurement/core/data_use_user_data.h" -#include "components/ntp_snippets/category.h" #include "components/ntp_snippets/features.h" #include "components/ntp_snippets/ntp_snippets_constants.h" -#include "components/ntp_snippets/remote/request_params.h" -#include "components/ntp_snippets/user_classifier.h" -#include "components/signin/core/browser/access_token_fetcher.h" -#include "components/signin/core/browser/signin_manager.h" -#include "components/signin/core/browser/signin_manager_base.h" #include "components/strings/grit/components_strings.h" #include "components/variations/variations_associated_data.h" -#include "net/url_request/url_fetcher.h" #include "ui/base/l10n/l10n_util.h" -using net::URLFetcher; -using net::URLRequestContextGetter; -using net::HttpRequestHeaders; -using net::URLRequestStatus; -using translate::LanguageModel; - namespace ntp_snippets { -using internal::JsonRequest; -using internal::FetchResult; - namespace { -const char kContentSuggestionsApiScope[] = - "https://www.googleapis.com/auth/chrome-content-suggestions"; -const char kSnippetsServerNonAuthorizedFormat[] = "%s?key=%s"; -const char kAuthorizationRequestHeaderFormat[] = "Bearer %s"; - // Variation parameter for chrome-content-suggestions backend. const char kContentSuggestionsBackend[] = "content_suggestions_backend"; -const int kFetchTimeHistogramResolution = 5; - -std::string FetchResultToString(FetchResult result) { - switch (result) { - case FetchResult::SUCCESS: - return "OK"; - case FetchResult::URL_REQUEST_STATUS_ERROR: - return "URLRequestStatus error"; - case FetchResult::HTTP_ERROR: - return "HTTP error"; - case FetchResult::JSON_PARSE_ERROR: - return "Received invalid JSON"; - case FetchResult::INVALID_SNIPPET_CONTENT_ERROR: - return "Invalid / empty list."; - case FetchResult::OAUTH_TOKEN_ERROR: - return "Error in obtaining an OAuth2 access token."; - case FetchResult::MISSING_API_KEY: - return "No API key available."; - case FetchResult::RESULT_MAX: - break; - } - NOTREACHED(); - return "Unknown error"; -} - -Status FetchResultToStatus(FetchResult result) { - switch (result) { - case FetchResult::SUCCESS: - return Status::Success(); - // Permanent errors occur if it is more likely that the error originated - // from the client. - case FetchResult::OAUTH_TOKEN_ERROR: - case FetchResult::MISSING_API_KEY: - return Status(StatusCode::PERMANENT_ERROR, FetchResultToString(result)); - // Temporary errors occur if it's more likely that the client behaved - // correctly but the server failed to respond as expected. - // TODO(fhorschig): Revisit HTTP_ERROR once the rescheduling was reworked. - case FetchResult::HTTP_ERROR: - case FetchResult::URL_REQUEST_STATUS_ERROR: - case FetchResult::INVALID_SNIPPET_CONTENT_ERROR: - case FetchResult::JSON_PARSE_ERROR: - return Status(StatusCode::TEMPORARY_ERROR, FetchResultToString(result)); - case FetchResult::RESULT_MAX: - break; - } - NOTREACHED(); - return Status(StatusCode::PERMANENT_ERROR, std::string()); -} - -// Creates suggestions from dictionary values in |list| and adds them to -// |suggestions|. Returns true on success, false if anything went wrong. -// |remote_category_id| is only used if |content_suggestions_api| is true. -bool AddSuggestionsFromListValue(bool content_suggestions_api, - int remote_category_id, - const base::ListValue& list, - RemoteSuggestion::PtrVector* suggestions, - const base::Time& fetch_time) { - for (const auto& value : list) { - const base::DictionaryValue* dict = nullptr; - if (!value.GetAsDictionary(&dict)) { - return false; - } - - std::unique_ptr<RemoteSuggestion> suggestion; - if (content_suggestions_api) { - suggestion = RemoteSuggestion::CreateFromContentSuggestionsDictionary( - *dict, remote_category_id, fetch_time); - } else { - suggestion = - RemoteSuggestion::CreateFromChromeReaderDictionary(*dict, fetch_time); - } - if (!suggestion) { - return false; - } - - suggestions->push_back(std::move(suggestion)); - } - return true; -} - -int GetMinuteOfTheDay(bool local_time, - bool reduced_resolution, - base::Clock* clock) { - base::Time now(clock->Now()); - base::Time::Exploded now_exploded{}; - local_time ? now.LocalExplode(&now_exploded) : now.UTCExplode(&now_exploded); - int now_minute = reduced_resolution - ? now_exploded.minute / kFetchTimeHistogramResolution * - kFetchTimeHistogramResolution - : now_exploded.minute; - return now_exploded.hour * 60 + now_minute; -} - -// The response from the backend might include suggestions from multiple -// categories. If only a single category was requested, this function filters -// all other categories out. -void FilterCategories( - RemoteSuggestionsFetcher::FetchedCategoriesVector* categories, - base::Optional<Category> exclusive_category) { - if (!exclusive_category.has_value()) { - return; - } - Category exclusive = exclusive_category.value(); - auto category_it = std::find_if( - categories->begin(), categories->end(), - [&exclusive](const RemoteSuggestionsFetcher::FetchedCategory& c) -> bool { - return c.category == exclusive; - }); - if (category_it == categories->end()) { - categories->clear(); - return; - } - RemoteSuggestionsFetcher::FetchedCategory category = std::move(*category_it); - categories->clear(); - categories->push_back(std::move(category)); -} - } // namespace GURL GetFetchEndpoint(version_info::Channel channel) { @@ -193,307 +40,6 @@ GURL GetFetchEndpoint(version_info::Channel channel) { return GURL{kContentSuggestionsStagingServer}; } -CategoryInfo BuildArticleCategoryInfo( - const base::Optional<base::string16>& title) { - return CategoryInfo( - title.has_value() ? title.value() - : l10n_util::GetStringUTF16( - IDS_NTP_ARTICLE_SUGGESTIONS_SECTION_HEADER), - ContentSuggestionsCardLayout::FULL_CARD, - ContentSuggestionsAdditionalAction::FETCH, - /*show_if_empty=*/true, - l10n_util::GetStringUTF16(IDS_NTP_ARTICLE_SUGGESTIONS_SECTION_EMPTY)); -} - -CategoryInfo BuildRemoteCategoryInfo(const base::string16& title, - bool allow_fetching_more_results) { - ContentSuggestionsAdditionalAction action = - ContentSuggestionsAdditionalAction::NONE; - if (allow_fetching_more_results) { - action = ContentSuggestionsAdditionalAction::FETCH; - } - return CategoryInfo( - title, ContentSuggestionsCardLayout::FULL_CARD, action, - /*show_if_empty=*/false, - // TODO(tschumann): The message for no-articles is likely wrong - // and needs to be added to the stubby protocol if we want to - // support it. - l10n_util::GetStringUTF16(IDS_NTP_ARTICLE_SUGGESTIONS_SECTION_EMPTY)); -} - -RemoteSuggestionsFetcher::FetchedCategory::FetchedCategory(Category c, - CategoryInfo&& info) - : category(c), info(info) {} - -RemoteSuggestionsFetcher::FetchedCategory::FetchedCategory(FetchedCategory&&) = - default; - -RemoteSuggestionsFetcher::FetchedCategory::~FetchedCategory() = default; - -RemoteSuggestionsFetcher::FetchedCategory& -RemoteSuggestionsFetcher::FetchedCategory::operator=(FetchedCategory&&) = - default; - -RemoteSuggestionsFetcher::RemoteSuggestionsFetcher( - SigninManagerBase* signin_manager, - OAuth2TokenService* token_service, - scoped_refptr<URLRequestContextGetter> url_request_context_getter, - PrefService* pref_service, - LanguageModel* language_model, - const ParseJSONCallback& parse_json_callback, - const GURL& api_endpoint, - const std::string& api_key, - const UserClassifier* user_classifier) - : signin_manager_(signin_manager), - token_service_(token_service), - url_request_context_getter_(std::move(url_request_context_getter)), - language_model_(language_model), - parse_json_callback_(parse_json_callback), - fetch_url_(api_endpoint), - api_key_(api_key), - clock_(new base::DefaultClock()), - user_classifier_(user_classifier) {} - RemoteSuggestionsFetcher::~RemoteSuggestionsFetcher() = default; -void RemoteSuggestionsFetcher::FetchSnippets( - const RequestParams& params, - SnippetsAvailableCallback callback) { - if (!params.interactive_request) { - UMA_HISTOGRAM_SPARSE_SLOWLY( - "NewTabPage.Snippets.FetchTimeLocal", - GetMinuteOfTheDay(/*local_time=*/true, - /*reduced_resolution=*/true, clock_.get())); - UMA_HISTOGRAM_SPARSE_SLOWLY( - "NewTabPage.Snippets.FetchTimeUTC", - GetMinuteOfTheDay(/*local_time=*/false, - /*reduced_resolution=*/true, clock_.get())); - } - - JsonRequest::Builder builder; - builder.SetLanguageModel(language_model_) - .SetParams(params) - .SetParseJsonCallback(parse_json_callback_) - .SetClock(clock_.get()) - .SetUrlRequestContextGetter(url_request_context_getter_) - .SetUserClassifier(*user_classifier_); - - if (signin_manager_->IsAuthenticated() || signin_manager_->AuthInProgress()) { - // Signed-in: get OAuth token --> fetch suggestions. - pending_requests_.emplace(std::move(builder), std::move(callback)); - StartTokenRequest(); - } else { - // Not signed in: fetch suggestions (without authentication). - FetchSnippetsNonAuthenticated(std::move(builder), std::move(callback)); - } -} - -void RemoteSuggestionsFetcher::FetchSnippetsNonAuthenticated( - JsonRequest::Builder builder, - SnippetsAvailableCallback callback) { - if (api_key_.empty()) { - // If we don't have an API key, don't even try. - FetchFinished(OptionalFetchedCategories(), std::move(callback), - FetchResult::MISSING_API_KEY, std::string()); - return; - } - // When not providing OAuth token, we need to pass the Google API key. - builder.SetUrl( - GURL(base::StringPrintf(kSnippetsServerNonAuthorizedFormat, - fetch_url_.spec().c_str(), api_key_.c_str()))); - StartRequest(std::move(builder), std::move(callback)); -} - -void RemoteSuggestionsFetcher::FetchSnippetsAuthenticated( - JsonRequest::Builder builder, - SnippetsAvailableCallback callback, - const std::string& oauth_access_token) { - // TODO(jkrcal, treib): Add unit-tests for authenticated fetches. - builder.SetUrl(fetch_url_) - .SetAuthentication(signin_manager_->GetAuthenticatedAccountId(), - base::StringPrintf(kAuthorizationRequestHeaderFormat, - oauth_access_token.c_str())); - StartRequest(std::move(builder), std::move(callback)); -} - -void RemoteSuggestionsFetcher::StartRequest( - JsonRequest::Builder builder, - SnippetsAvailableCallback callback) { - std::unique_ptr<JsonRequest> request = builder.Build(); - JsonRequest* raw_request = request.get(); - raw_request->Start(base::BindOnce(&RemoteSuggestionsFetcher::JsonRequestDone, - base::Unretained(this), std::move(request), - std::move(callback))); -} - -void RemoteSuggestionsFetcher::StartTokenRequest() { - // If there is already an ongoing token request, just wait for that. - if (token_fetcher_) { - return; - } - - OAuth2TokenService::ScopeSet scopes{kContentSuggestionsApiScope}; - token_fetcher_ = base::MakeUnique<AccessTokenFetcher>( - "ntp_snippets", signin_manager_, token_service_, scopes, - base::BindOnce(&RemoteSuggestionsFetcher::AccessTokenFetchFinished, - base::Unretained(this))); -} - -void RemoteSuggestionsFetcher::AccessTokenFetchFinished( - const GoogleServiceAuthError& error, - const std::string& access_token) { - // Delete the fetcher only after we leave this method (which is called from - // the fetcher itself). - DCHECK(token_fetcher_); - std::unique_ptr<AccessTokenFetcher> token_fetcher_deleter( - std::move(token_fetcher_)); - - if (error.state() != GoogleServiceAuthError::NONE) { - AccessTokenError(error); - return; - } - - DCHECK(!access_token.empty()); - - while (!pending_requests_.empty()) { - std::pair<JsonRequest::Builder, SnippetsAvailableCallback> - builder_and_callback = std::move(pending_requests_.front()); - pending_requests_.pop(); - FetchSnippetsAuthenticated(std::move(builder_and_callback.first), - std::move(builder_and_callback.second), - access_token); - } -} - -void RemoteSuggestionsFetcher::AccessTokenError( - const GoogleServiceAuthError& error) { - DCHECK_NE(error.state(), GoogleServiceAuthError::NONE); - - DLOG(ERROR) << "Unable to get token: " << error.ToString(); - - while (!pending_requests_.empty()) { - std::pair<JsonRequest::Builder, SnippetsAvailableCallback> - builder_and_callback = std::move(pending_requests_.front()); - - FetchFinished(OptionalFetchedCategories(), - std::move(builder_and_callback.second), - FetchResult::OAUTH_TOKEN_ERROR, - /*error_details=*/ - base::StringPrintf(" (%s)", error.ToString().c_str())); - pending_requests_.pop(); - } -} - -void RemoteSuggestionsFetcher::JsonRequestDone( - std::unique_ptr<JsonRequest> request, - SnippetsAvailableCallback callback, - std::unique_ptr<base::Value> result, - FetchResult status_code, - const std::string& error_details) { - DCHECK(request); - // Record the time when request for fetching remote content snippets finished. - const base::Time fetch_time = clock_->Now(); - - last_fetch_json_ = request->GetResponseString(); - - UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", - request->GetFetchDuration()); - - if (!result) { - FetchFinished(OptionalFetchedCategories(), std::move(callback), status_code, - error_details); - return; - } - - FetchedCategoriesVector categories; - if (!JsonToSnippets(*result, &categories, fetch_time)) { - LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_; - FetchFinished(OptionalFetchedCategories(), std::move(callback), - FetchResult::INVALID_SNIPPET_CONTENT_ERROR, std::string()); - return; - } - // Filter out unwanted categories if necessary. - // TODO(fhorschig): As soon as the server supports filtering by category, - // adjust the request instead of over-fetching and filtering here. - FilterCategories(&categories, request->exclusive_category()); - - FetchFinished(std::move(categories), std::move(callback), - FetchResult::SUCCESS, std::string()); -} - -void RemoteSuggestionsFetcher::FetchFinished( - OptionalFetchedCategories categories, - SnippetsAvailableCallback callback, - FetchResult fetch_result, - const std::string& error_details) { - DCHECK(fetch_result == FetchResult::SUCCESS || !categories.has_value()); - - last_status_ = FetchResultToString(fetch_result) + error_details; - - UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult", - static_cast<int>(fetch_result), - static_cast<int>(FetchResult::RESULT_MAX)); - - DVLOG(1) << "Fetch finished: " << last_status_; - - std::move(callback).Run(FetchResultToStatus(fetch_result), - std::move(categories)); -} - -bool RemoteSuggestionsFetcher::JsonToSnippets( - const base::Value& parsed, - FetchedCategoriesVector* categories, - const base::Time& fetch_time) { - const base::DictionaryValue* top_dict = nullptr; - if (!parsed.GetAsDictionary(&top_dict)) { - return false; - } - - const base::ListValue* categories_value = nullptr; - if (!top_dict->GetList("categories", &categories_value)) { - return false; - } - - for (const auto& v : *categories_value) { - std::string utf8_title; - int remote_category_id = -1; - const base::DictionaryValue* category_value = nullptr; - if (!(v.GetAsDictionary(&category_value) && - category_value->GetString("localizedTitle", &utf8_title) && - category_value->GetInteger("id", &remote_category_id) && - (remote_category_id > 0))) { - return false; - } - - RemoteSuggestion::PtrVector suggestions; - const base::ListValue* suggestions_list = nullptr; - // Absence of a list of suggestions is treated as an empty list, which - // is permissible. - if (category_value->GetList("suggestions", &suggestions_list)) { - if (!AddSuggestionsFromListValue( - /*content_suggestions_api=*/true, remote_category_id, - *suggestions_list, &suggestions, fetch_time)) { - return false; - } - } - Category category = Category::FromRemoteCategory(remote_category_id); - if (category.IsKnownCategory(KnownCategories::ARTICLES)) { - categories->push_back(FetchedCategory( - category, BuildArticleCategoryInfo(base::UTF8ToUTF16(utf8_title)))); - } else { - // TODO(tschumann): Right now, the backend does not yet populate this - // field. Make it mandatory once the backends provide it. - bool allow_fetching_more_results = false; - category_value->GetBoolean("allowFetchingMoreResults", - &allow_fetching_more_results); - categories->push_back(FetchedCategory( - category, BuildRemoteCategoryInfo(base::UTF8ToUTF16(utf8_title), - allow_fetching_more_results))); - } - categories->back().suggestions = std::move(suggestions); - } - - return true; -} - } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher.h b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher.h index ffe7fd9be04..e16dec659f4 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher.h +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher.h @@ -5,197 +5,52 @@ #ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_FETCHER_H_ #define COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_FETCHER_H_ -#include <memory> -#include <queue> #include <string> -#include <utility> #include <vector> #include "base/callback.h" #include "base/optional.h" -#include "base/time/clock.h" -#include "base/time/tick_clock.h" #include "components/ntp_snippets/category.h" #include "components/ntp_snippets/category_info.h" -#include "components/ntp_snippets/remote/json_request.h" +#include "components/ntp_snippets/remote/json_to_categories.h" #include "components/ntp_snippets/remote/remote_suggestion.h" #include "components/ntp_snippets/remote/request_params.h" #include "components/ntp_snippets/status.h" -#include "components/translate/core/browser/language_model.h" #include "components/version_info/version_info.h" -#include "net/url_request/url_request_context_getter.h" - -class AccessTokenFetcher; -class OAuth2TokenService; -class PrefService; -class SigninManagerBase; - -namespace base { -class Value; -} // namespace base +#include "url/gurl.h" namespace ntp_snippets { -class UserClassifier; - // Returns the appropriate API endpoint for the fetcher, in consideration of // the channel and variation parameters. GURL GetFetchEndpoint(version_info::Channel channel); -// TODO(tschumann): BuildArticleCategoryInfo() and BuildRemoteCategoryInfo() -// don't really belong into this library. However, as the fetcher is -// providing this data for server-defined remote sections it's a good starting -// point. Candiates to add to such a library would be persisting categories -// (have all category managment in one place) or turning parsed JSON into -// FetchedCategory objects (all domain-specific logic in one place). - -// Provides the CategoryInfo data for article suggestions. If |title| is -// nullopt, then the default, hard-coded title will be used. -CategoryInfo BuildArticleCategoryInfo( - const base::Optional<base::string16>& title); - -// Provides the CategoryInfo data for other remote suggestions. -CategoryInfo BuildRemoteCategoryInfo(const base::string16& title, - bool allow_fetching_more_results); - // Fetches suggestion data for the NTP from the server. -// TODO(fhorschig): Untangle cyclic dependencies by introducing a -// RemoteSuggestionsFetcherInterface. (Would be good for mock implementations, -// too!) class RemoteSuggestionsFetcher { public: - struct FetchedCategory { - Category category; - CategoryInfo info; - RemoteSuggestion::PtrVector suggestions; - - FetchedCategory(Category c, CategoryInfo&& info); - FetchedCategory(FetchedCategory&&); // = default, in .cc - ~FetchedCategory(); // = default, in .cc - FetchedCategory& operator=(FetchedCategory&&); // = default, in .cc - }; - using FetchedCategoriesVector = std::vector<FetchedCategory>; using OptionalFetchedCategories = base::Optional<FetchedCategoriesVector>; - using SnippetsAvailableCallback = base::OnceCallback<void(Status status, OptionalFetchedCategories fetched_categories)>; - RemoteSuggestionsFetcher( - SigninManagerBase* signin_manager, - OAuth2TokenService* token_service, - scoped_refptr<net::URLRequestContextGetter> url_request_context_getter, - PrefService* pref_service, - translate::LanguageModel* language_model, - const ParseJSONCallback& parse_json_callback, - const GURL& api_endpoint, - const std::string& api_key, - const UserClassifier* user_classifier); - ~RemoteSuggestionsFetcher(); + virtual ~RemoteSuggestionsFetcher(); // Initiates a fetch from the server. When done (successfully or not), calls // the callback. // // If an ongoing fetch exists, both fetches won't influence each other (i.e. // every callback will be called exactly once). - void FetchSnippets(const RequestParams& params, - SnippetsAvailableCallback callback); + virtual void FetchSnippets(const RequestParams& params, + SnippetsAvailableCallback callback) = 0; // Debug string representing the status/result of the last fetch attempt. - const std::string& last_status() const { return last_status_; } + virtual const std::string& GetLastStatusForDebugging() const = 0; // Returns the last JSON fetched from the server. - const std::string& last_json() const { return last_fetch_json_; } + virtual const std::string& GetLastJsonForDebugging() const = 0; // Returns the URL endpoint used by the fetcher. - const GURL& fetch_url() const { return fetch_url_; } - - // Overrides internal clock for testing purposes. - void SetClockForTesting(std::unique_ptr<base::Clock> clock) { - clock_ = std::move(clock); - } - - private: - FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, - BuildRequestAuthenticated); - FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, - BuildRequestUnauthenticated); - FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, - BuildRequestExcludedIds); - FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, - BuildRequestNoUserClass); - FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, - BuildRequestWithTwoLanguages); - FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, - BuildRequestWithUILanguageOnly); - FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, - BuildRequestWithOtherLanguageOnly); - friend class ChromeReaderSnippetsFetcherTest; - - void FetchSnippetsNonAuthenticated(internal::JsonRequest::Builder builder, - SnippetsAvailableCallback callback); - void FetchSnippetsAuthenticated(internal::JsonRequest::Builder builder, - SnippetsAvailableCallback callback, - const std::string& oauth_access_token); - void StartRequest(internal::JsonRequest::Builder builder, - SnippetsAvailableCallback callback); - - void StartTokenRequest(); - - void AccessTokenFetchFinished(const GoogleServiceAuthError& error, - const std::string& access_token); - void AccessTokenError(const GoogleServiceAuthError& error); - - void JsonRequestDone(std::unique_ptr<internal::JsonRequest> request, - SnippetsAvailableCallback callback, - std::unique_ptr<base::Value> result, - internal::FetchResult status_code, - const std::string& error_details); - void FetchFinished(OptionalFetchedCategories categories, - SnippetsAvailableCallback callback, - internal::FetchResult status_code, - const std::string& error_details); - - bool JsonToSnippets(const base::Value& parsed, - FetchedCategoriesVector* categories, - const base::Time& fetch_time); - - // Authentication for signed-in users. - SigninManagerBase* signin_manager_; - OAuth2TokenService* token_service_; - - std::unique_ptr<AccessTokenFetcher> token_fetcher_; - - // Holds the URL request context. - scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_; - - // Stores requests that wait for an access token. - std::queue< - std::pair<internal::JsonRequest::Builder, SnippetsAvailableCallback>> - pending_requests_; - - // Weak reference, not owned. - translate::LanguageModel* const language_model_; - - const ParseJSONCallback parse_json_callback_; - - // API endpoint for fetching suggestions. - const GURL fetch_url_; - - // API key to use for non-authenticated requests. - const std::string api_key_; - - // Allow for an injectable clock for testing. - std::unique_ptr<base::Clock> clock_; - - // Classifier that tells us how active the user is. Not owned. - const UserClassifier* user_classifier_; - - // Info on the last finished fetch. - std::string last_status_; - std::string last_fetch_json_; - - DISALLOW_COPY_AND_ASSIGN(RemoteSuggestionsFetcher); + virtual const GURL& GetFetchUrlForDebugging() const = 0; }; } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_impl.cc b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_impl.cc new file mode 100644 index 00000000000..5817f94fd36 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_impl.cc @@ -0,0 +1,344 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/remote_suggestions_fetcher_impl.h" + +#include "base/memory/ptr_util.h" +#include "base/metrics/histogram_macros.h" +#include "base/strings/stringprintf.h" +#include "base/strings/utf_string_conversions.h" +#include "base/time/default_clock.h" +#include "base/time/time.h" +#include "base/values.h" +#include "components/language/core/browser/url_language_histogram.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/ntp_snippets_constants.h" +#include "components/ntp_snippets/user_classifier.h" +#include "components/signin/core/browser/access_token_fetcher.h" +#include "components/signin/core/browser/signin_manager_base.h" +#include "net/url_request/url_fetcher.h" +#include "net/url_request/url_request_status.h" + +using language::UrlLanguageHistogram; +using net::HttpRequestHeaders; +using net::URLFetcher; +using net::URLRequestContextGetter; +using net::URLRequestStatus; + +namespace ntp_snippets { + +using internal::FetchResult; +using internal::JsonRequest; + +namespace { + +const char kSnippetsServerNonAuthorizedFormat[] = "%s?key=%s"; +const char kAuthorizationRequestHeaderFormat[] = "Bearer %s"; + +const int kFetchTimeHistogramResolution = 5; + +std::string FetchResultToString(FetchResult result) { + switch (result) { + case FetchResult::SUCCESS: + return "OK"; + case FetchResult::URL_REQUEST_STATUS_ERROR: + return "URLRequestStatus error"; + case FetchResult::HTTP_ERROR: + return "HTTP error"; + case FetchResult::JSON_PARSE_ERROR: + return "Received invalid JSON"; + case FetchResult::INVALID_SNIPPET_CONTENT_ERROR: + return "Invalid / empty list."; + case FetchResult::OAUTH_TOKEN_ERROR: + return "Error in obtaining an OAuth2 access token."; + case FetchResult::MISSING_API_KEY: + return "No API key available."; + case FetchResult::RESULT_MAX: + break; + } + NOTREACHED(); + return "Unknown error"; +} + +Status FetchResultToStatus(FetchResult result) { + switch (result) { + case FetchResult::SUCCESS: + return Status::Success(); + // Permanent errors occur if it is more likely that the error originated + // from the client. + case FetchResult::OAUTH_TOKEN_ERROR: + case FetchResult::MISSING_API_KEY: + return Status(StatusCode::PERMANENT_ERROR, FetchResultToString(result)); + // Temporary errors occur if it's more likely that the client behaved + // correctly but the server failed to respond as expected. + // TODO(fhorschig): Revisit HTTP_ERROR once the rescheduling was reworked. + case FetchResult::HTTP_ERROR: + case FetchResult::URL_REQUEST_STATUS_ERROR: + case FetchResult::INVALID_SNIPPET_CONTENT_ERROR: + case FetchResult::JSON_PARSE_ERROR: + return Status(StatusCode::TEMPORARY_ERROR, FetchResultToString(result)); + case FetchResult::RESULT_MAX: + break; + } + NOTREACHED(); + return Status(StatusCode::PERMANENT_ERROR, std::string()); +} + +int GetMinuteOfTheDay(bool local_time, + bool reduced_resolution, + base::Clock* clock) { + base::Time now(clock->Now()); + base::Time::Exploded now_exploded{}; + local_time ? now.LocalExplode(&now_exploded) : now.UTCExplode(&now_exploded); + int now_minute = reduced_resolution + ? now_exploded.minute / kFetchTimeHistogramResolution * + kFetchTimeHistogramResolution + : now_exploded.minute; + return now_exploded.hour * 60 + now_minute; +} + +// The response from the backend might include suggestions from multiple +// categories. If only a single category was requested, this function filters +// all other categories out. +void FilterCategories(FetchedCategoriesVector* categories, + base::Optional<Category> exclusive_category) { + if (!exclusive_category.has_value()) { + return; + } + Category exclusive = exclusive_category.value(); + auto category_it = + std::find_if(categories->begin(), categories->end(), + [&exclusive](const FetchedCategory& c) -> bool { + return c.category == exclusive; + }); + if (category_it == categories->end()) { + categories->clear(); + return; + } + FetchedCategory category = std::move(*category_it); + categories->clear(); + categories->push_back(std::move(category)); +} + +} // namespace + +RemoteSuggestionsFetcherImpl::RemoteSuggestionsFetcherImpl( + SigninManagerBase* signin_manager, + OAuth2TokenService* token_service, + scoped_refptr<URLRequestContextGetter> url_request_context_getter, + PrefService* pref_service, + UrlLanguageHistogram* language_histogram, + const ParseJSONCallback& parse_json_callback, + const GURL& api_endpoint, + const std::string& api_key, + const UserClassifier* user_classifier) + : signin_manager_(signin_manager), + token_service_(token_service), + url_request_context_getter_(std::move(url_request_context_getter)), + language_histogram_(language_histogram), + parse_json_callback_(parse_json_callback), + fetch_url_(api_endpoint), + api_key_(api_key), + clock_(new base::DefaultClock()), + user_classifier_(user_classifier) {} + +RemoteSuggestionsFetcherImpl::~RemoteSuggestionsFetcherImpl() = default; + +const std::string& RemoteSuggestionsFetcherImpl::GetLastStatusForDebugging() + const { + return last_status_; +} +const std::string& RemoteSuggestionsFetcherImpl::GetLastJsonForDebugging() + const { + return last_fetch_json_; +} +const GURL& RemoteSuggestionsFetcherImpl::GetFetchUrlForDebugging() const { + return fetch_url_; +} + +void RemoteSuggestionsFetcherImpl::FetchSnippets( + const RequestParams& params, + SnippetsAvailableCallback callback) { + if (!params.interactive_request) { + UMA_HISTOGRAM_SPARSE_SLOWLY( + "NewTabPage.Snippets.FetchTimeLocal", + GetMinuteOfTheDay(/*local_time=*/true, + /*reduced_resolution=*/true, clock_.get())); + UMA_HISTOGRAM_SPARSE_SLOWLY( + "NewTabPage.Snippets.FetchTimeUTC", + GetMinuteOfTheDay(/*local_time=*/false, + /*reduced_resolution=*/true, clock_.get())); + } + + JsonRequest::Builder builder; + builder.SetLanguageHistogram(language_histogram_) + .SetParams(params) + .SetParseJsonCallback(parse_json_callback_) + .SetClock(clock_.get()) + .SetUrlRequestContextGetter(url_request_context_getter_) + .SetUserClassifier(*user_classifier_); + + if (signin_manager_->IsAuthenticated() || signin_manager_->AuthInProgress()) { + // Signed-in: get OAuth token --> fetch suggestions. + pending_requests_.emplace(std::move(builder), std::move(callback)); + StartTokenRequest(); + } else { + // Not signed in: fetch suggestions (without authentication). + FetchSnippetsNonAuthenticated(std::move(builder), std::move(callback)); + } +} + +void RemoteSuggestionsFetcherImpl::FetchSnippetsNonAuthenticated( + JsonRequest::Builder builder, + SnippetsAvailableCallback callback) { + if (api_key_.empty()) { + // If we don't have an API key, don't even try. + FetchFinished(OptionalFetchedCategories(), std::move(callback), + FetchResult::MISSING_API_KEY, std::string()); + return; + } + // When not providing OAuth token, we need to pass the Google API key. + builder.SetUrl( + GURL(base::StringPrintf(kSnippetsServerNonAuthorizedFormat, + fetch_url_.spec().c_str(), api_key_.c_str()))); + StartRequest(std::move(builder), std::move(callback)); +} + +void RemoteSuggestionsFetcherImpl::FetchSnippetsAuthenticated( + JsonRequest::Builder builder, + SnippetsAvailableCallback callback, + const std::string& oauth_access_token) { + // TODO(jkrcal, treib): Add unit-tests for authenticated fetches. + builder.SetUrl(fetch_url_) + .SetAuthentication(signin_manager_->GetAuthenticatedAccountId(), + base::StringPrintf(kAuthorizationRequestHeaderFormat, + oauth_access_token.c_str())); + StartRequest(std::move(builder), std::move(callback)); +} + +void RemoteSuggestionsFetcherImpl::StartRequest( + JsonRequest::Builder builder, + SnippetsAvailableCallback callback) { + std::unique_ptr<JsonRequest> request = builder.Build(); + JsonRequest* raw_request = request.get(); + raw_request->Start(base::BindOnce( + &RemoteSuggestionsFetcherImpl::JsonRequestDone, base::Unretained(this), + std::move(request), std::move(callback))); +} + +void RemoteSuggestionsFetcherImpl::StartTokenRequest() { + // If there is already an ongoing token request, just wait for that. + if (token_fetcher_) { + return; + } + + OAuth2TokenService::ScopeSet scopes{kContentSuggestionsApiScope}; + token_fetcher_ = base::MakeUnique<AccessTokenFetcher>( + "ntp_snippets", signin_manager_, token_service_, scopes, + base::BindOnce(&RemoteSuggestionsFetcherImpl::AccessTokenFetchFinished, + base::Unretained(this))); +} + +void RemoteSuggestionsFetcherImpl::AccessTokenFetchFinished( + const GoogleServiceAuthError& error, + const std::string& access_token) { + // Delete the fetcher only after we leave this method (which is called from + // the fetcher itself). + DCHECK(token_fetcher_); + std::unique_ptr<AccessTokenFetcher> token_fetcher_deleter( + std::move(token_fetcher_)); + + if (error.state() != GoogleServiceAuthError::NONE) { + AccessTokenError(error); + return; + } + + DCHECK(!access_token.empty()); + + while (!pending_requests_.empty()) { + std::pair<JsonRequest::Builder, SnippetsAvailableCallback> + builder_and_callback = std::move(pending_requests_.front()); + pending_requests_.pop(); + FetchSnippetsAuthenticated(std::move(builder_and_callback.first), + std::move(builder_and_callback.second), + access_token); + } +} + +void RemoteSuggestionsFetcherImpl::AccessTokenError( + const GoogleServiceAuthError& error) { + DCHECK_NE(error.state(), GoogleServiceAuthError::NONE); + + DLOG(ERROR) << "Unable to get token: " << error.ToString(); + + while (!pending_requests_.empty()) { + std::pair<JsonRequest::Builder, SnippetsAvailableCallback> + builder_and_callback = std::move(pending_requests_.front()); + + FetchFinished(OptionalFetchedCategories(), + std::move(builder_and_callback.second), + FetchResult::OAUTH_TOKEN_ERROR, + /*error_details=*/ + base::StringPrintf(" (%s)", error.ToString().c_str())); + pending_requests_.pop(); + } +} + +void RemoteSuggestionsFetcherImpl::JsonRequestDone( + std::unique_ptr<JsonRequest> request, + SnippetsAvailableCallback callback, + std::unique_ptr<base::Value> result, + FetchResult status_code, + const std::string& error_details) { + DCHECK(request); + // Record the time when request for fetching remote content snippets finished. + const base::Time fetch_time = clock_->Now(); + + last_fetch_json_ = request->GetResponseString(); + + UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", + request->GetFetchDuration()); + + if (!result) { + FetchFinished(OptionalFetchedCategories(), std::move(callback), status_code, + error_details); + return; + } + + FetchedCategoriesVector categories; + if (!JsonToCategories(*result, &categories, fetch_time)) { + LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_; + FetchFinished(OptionalFetchedCategories(), std::move(callback), + FetchResult::INVALID_SNIPPET_CONTENT_ERROR, std::string()); + return; + } + // Filter out unwanted categories if necessary. + // TODO(fhorschig): As soon as the server supports filtering by category, + // adjust the request instead of over-fetching and filtering here. + FilterCategories(&categories, request->exclusive_category()); + + FetchFinished(std::move(categories), std::move(callback), + FetchResult::SUCCESS, std::string()); +} + +void RemoteSuggestionsFetcherImpl::FetchFinished( + OptionalFetchedCategories categories, + SnippetsAvailableCallback callback, + FetchResult fetch_result, + const std::string& error_details) { + DCHECK(fetch_result == FetchResult::SUCCESS || !categories.has_value()); + + last_status_ = FetchResultToString(fetch_result) + error_details; + + UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult", + static_cast<int>(fetch_result), + static_cast<int>(FetchResult::RESULT_MAX)); + + DVLOG(1) << "Fetch finished: " << last_status_; + + std::move(callback).Run(FetchResultToStatus(fetch_result), + std::move(categories)); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_impl.h b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_impl.h new file mode 100644 index 00000000000..770ea5067c0 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_impl.h @@ -0,0 +1,147 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_FETCHER_IMPL_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_FETCHER_IMPL_H_ + +#include <memory> +#include <queue> +#include <string> +#include <utility> +#include <vector> + +#include "base/callback.h" +#include "base/optional.h" +#include "base/time/clock.h" +#include "components/ntp_snippets/remote/json_request.h" +#include "components/ntp_snippets/remote/json_to_categories.h" +#include "components/ntp_snippets/remote/remote_suggestions_fetcher.h" +#include "components/ntp_snippets/remote/request_params.h" +#include "net/url_request/url_request_context_getter.h" + +class AccessTokenFetcher; +class OAuth2TokenService; +class PrefService; +class SigninManagerBase; + +namespace base { +class Value; +} // namespace base + +namespace language { +class UrlLanguageHistogram; +} // namespace language + +namespace ntp_snippets { + +class UserClassifier; + +class RemoteSuggestionsFetcherImpl : public RemoteSuggestionsFetcher { + public: + RemoteSuggestionsFetcherImpl( + SigninManagerBase* signin_manager, + OAuth2TokenService* token_service, + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter, + PrefService* pref_service, + language::UrlLanguageHistogram* language_histogram, + const ParseJSONCallback& parse_json_callback, + const GURL& api_endpoint, + const std::string& api_key, + const UserClassifier* user_classifier); + ~RemoteSuggestionsFetcherImpl() override; + + void FetchSnippets(const RequestParams& params, + SnippetsAvailableCallback callback) override; + + const std::string& GetLastStatusForDebugging() const override; + const std::string& GetLastJsonForDebugging() const override; + const GURL& GetFetchUrlForDebugging() const override; + + // Overrides internal clock for testing purposes. + void SetClockForTesting(std::unique_ptr<base::Clock> clock) { + clock_ = std::move(clock); + } + + private: + FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, + BuildRequestAuthenticated); + FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, + BuildRequestUnauthenticated); + FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, + BuildRequestExcludedIds); + FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, + BuildRequestNoUserClass); + FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, + BuildRequestWithTwoLanguages); + FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, + BuildRequestWithUILanguageOnly); + FRIEND_TEST_ALL_PREFIXES(ChromeReaderSnippetsFetcherTest, + BuildRequestWithOtherLanguageOnly); + friend class ChromeReaderSnippetsFetcherTest; + + void FetchSnippetsNonAuthenticated(internal::JsonRequest::Builder builder, + SnippetsAvailableCallback callback); + void FetchSnippetsAuthenticated(internal::JsonRequest::Builder builder, + SnippetsAvailableCallback callback, + const std::string& oauth_access_token); + void StartRequest(internal::JsonRequest::Builder builder, + SnippetsAvailableCallback callback); + + void StartTokenRequest(); + + void AccessTokenFetchFinished(const GoogleServiceAuthError& error, + const std::string& access_token); + void AccessTokenError(const GoogleServiceAuthError& error); + + void JsonRequestDone(std::unique_ptr<internal::JsonRequest> request, + SnippetsAvailableCallback callback, + std::unique_ptr<base::Value> result, + internal::FetchResult status_code, + const std::string& error_details); + void FetchFinished(OptionalFetchedCategories categories, + SnippetsAvailableCallback callback, + internal::FetchResult status_code, + const std::string& error_details); + + // Authentication for signed-in users. + SigninManagerBase* signin_manager_; + OAuth2TokenService* token_service_; + + std::unique_ptr<AccessTokenFetcher> token_fetcher_; + + // Holds the URL request context. + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_; + + // Stores requests that wait for an access token. + std::queue< + std::pair<internal::JsonRequest::Builder, SnippetsAvailableCallback>> + pending_requests_; + + // Weak reference, not owned. + language::UrlLanguageHistogram* const language_histogram_; + + const ParseJSONCallback parse_json_callback_; + + // API endpoint for fetching suggestions. + const GURL fetch_url_; + + // API key to use for non-authenticated requests. + const std::string api_key_; + + // Allow for an injectable clock for testing. + std::unique_ptr<base::Clock> clock_; + + // Classifier that tells us how active the user is. Not owned. + const UserClassifier* user_classifier_; + + // Info on the last finished fetch. + std::string last_status_; + std::string last_fetch_json_; + + DISALLOW_COPY_AND_ASSIGN(RemoteSuggestionsFetcherImpl); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_FETCHER_IMPL_H_ diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_unittest.cc b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_impl_unittest.cc index e5ffb4d0821..e4c2e2eb894 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_unittest.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_fetcher_impl_unittest.cc @@ -1,8 +1,8 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. +// Copyright 2017 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/ntp_snippets/remote/remote_suggestions_fetcher.h" +#include "components/ntp_snippets/remote/remote_suggestions_fetcher_impl.h" #include <deque> #include <map> @@ -11,12 +11,14 @@ #include "base/json/json_reader.h" #include "base/memory/ptr_util.h" +#include "base/optional.h" #include "base/test/histogram_tester.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" #include "base/time/default_clock.h" #include "base/time/time.h" #include "base/values.h" +#include "build/build_config.h" #include "components/ntp_snippets/category.h" #include "components/ntp_snippets/features.h" #include "components/ntp_snippets/ntp_snippets_constants.h" @@ -44,9 +46,11 @@ using testing::_; using testing::AllOf; using testing::ElementsAre; using testing::Eq; +using testing::Field; using testing::IsEmpty; using testing::Not; using testing::NotNull; +using testing::Property; using testing::StartsWith; const char kAPIKey[] = "fakeAPIkey"; @@ -65,20 +69,6 @@ ACTION_P(MoveArgument1PointeeTo, ptr) { *ptr = std::move(*arg1); } -MATCHER(HasValue, "") { - return static_cast<bool>(*arg); -} - -// TODO(fhorschig): When there are more helpers for the Status class, consider a -// helpers file. -MATCHER_P(HasCode, code, "") { - return arg.code == code; -} - -MATCHER(IsSuccess, "") { - return arg.IsSuccess(); -} - MATCHER(IsEmptyCategoriesList, "is an empty list of categories") { RemoteSuggestionsFetcher::OptionalFetchedCategories& fetched_categories = *arg; @@ -238,10 +228,10 @@ class FailingFakeURLFetcherFactory : public net::URLFetcherFactory { int id, const GURL& url, net::URLFetcher::RequestType request_type, - net::URLFetcherDelegate* d, + net::URLFetcherDelegate* delegate, net::NetworkTrafficAnnotationTag traffic_annotation) override { return base::MakeUnique<net::FakeURLFetcher>( - url, d, /*response_data=*/std::string(), net::HTTP_NOT_FOUND, + url, delegate, /*response_data=*/std::string(), net::HTTP_NOT_FOUND, net::URLRequestStatus::FAILED); } }; @@ -270,9 +260,9 @@ void ParseJsonDelayed(const std::string& json, } // namespace -class RemoteSuggestionsFetcherTestBase : public testing::Test { +class RemoteSuggestionsFetcherImplTestBase : public testing::Test { public: - explicit RemoteSuggestionsFetcherTestBase(const GURL& gurl) + explicit RemoteSuggestionsFetcherImplTestBase(const GURL& gurl) : default_variation_params_( {{"send_top_languages", "true"}, {"send_user_class", "true"}}), params_manager_(ntp_snippets::kArticleSuggestionsFeature.name, @@ -299,7 +289,7 @@ class RemoteSuggestionsFetcherTestBase : public testing::Test { base::MakeUnique<FakeOAuth2TokenServiceDelegate>( request_context_getter.get())); - fetcher_ = base::MakeUnique<RemoteSuggestionsFetcher>( + fetcher_ = base::MakeUnique<RemoteSuggestionsFetcherImpl>( utils_.fake_signin_manager(), fake_token_service_.get(), std::move(request_context_getter), utils_.pref_service(), nullptr, base::Bind(&ParseJsonDelayed), @@ -309,7 +299,13 @@ class RemoteSuggestionsFetcherTestBase : public testing::Test { fetcher_->SetClockForTesting(mock_task_runner_->GetMockClock()); } - void SignIn() { utils_.fake_signin_manager()->SignIn(kTestEmail); } + void SignIn() { +#if defined(OS_CHROMEOS) + utils_.fake_signin_manager()->SignIn(kTestEmail); +#else + utils_.fake_signin_manager()->SignIn(kTestEmail, "user", "password"); +#endif + } void IssueRefreshToken() { fake_token_service_->GetDelegate()->UpdateCredentials(kTestEmail, "token"); @@ -332,7 +328,7 @@ class RemoteSuggestionsFetcherTestBase : public testing::Test { base::Unretained(callback)); } - RemoteSuggestionsFetcher& fetcher() { return *fetcher_; } + RemoteSuggestionsFetcherImpl& fetcher() { return *fetcher_; } MockSnippetsAvailableCallback& mock_callback() { return mock_callback_; } void FastForwardUntilNoTasksRemain() { mock_task_runner_->FastForwardUntilNoTasksRemain(); @@ -386,20 +382,20 @@ class RemoteSuggestionsFetcherTestBase : public testing::Test { // Initialized lazily in SetFakeResponse(). std::unique_ptr<net::FakeURLFetcherFactory> fake_url_fetcher_factory_; std::unique_ptr<FakeProfileOAuth2TokenService> fake_token_service_; - std::unique_ptr<RemoteSuggestionsFetcher> fetcher_; + std::unique_ptr<RemoteSuggestionsFetcherImpl> fetcher_; std::unique_ptr<UserClassifier> user_classifier_; MockSnippetsAvailableCallback mock_callback_; const GURL test_url_; base::HistogramTester histogram_tester_; - DISALLOW_COPY_AND_ASSIGN(RemoteSuggestionsFetcherTestBase); + DISALLOW_COPY_AND_ASSIGN(RemoteSuggestionsFetcherImplTestBase); }; class RemoteSuggestionsSignedOutFetcherTest - : public RemoteSuggestionsFetcherTestBase { + : public RemoteSuggestionsFetcherImplTestBase { public: RemoteSuggestionsSignedOutFetcherTest() - : RemoteSuggestionsFetcherTestBase( + : RemoteSuggestionsFetcherImplTestBase( GURL(kTestChromeContentSuggestionsSignedOutUrl)) {} }; @@ -409,10 +405,10 @@ class RemoteSuggestionsSignedOutFetcherTest // FakeSigninManagerBase use FakeSigninManager which does not exist on // ChromeOS). crbug.com/688310 class RemoteSuggestionsSignedInFetcherTest - : public RemoteSuggestionsFetcherTestBase { + : public RemoteSuggestionsFetcherImplTestBase { public: RemoteSuggestionsSignedInFetcherTest() - : RemoteSuggestionsFetcherTestBase( + : RemoteSuggestionsFetcherImplTestBase( GURL(kTestChromeContentSuggestionsSignedInUrl)) {} }; @@ -424,7 +420,7 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldNotFetchOnCreation) { IsEmpty()); EXPECT_THAT(histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchTime"), IsEmpty()); - EXPECT_THAT(fetcher().last_status(), IsEmpty()); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), IsEmpty()); } TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldFetchSuccessfully) { @@ -448,14 +444,15 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldFetchSuccessfully) { SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); EXPECT_CALL(mock_callback(), - Run(IsSuccess(), - AllOf(IsSingleArticle("http://localhost/foobar"), - FirstCategoryHasInfo(IsCategoryInfoForArticles())))); + Run(Property(&Status::IsSuccess, true), + /*fetched_categories=*/AllOf( + IsSingleArticle("http://localhost/foobar"), + FirstCategoryHasInfo(IsCategoryInfoForArticles())))); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_status(), Eq("OK")); - EXPECT_THAT(fetcher().last_json(), Eq(kJsonStr)); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), Eq("OK")); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), Eq(kJsonStr)); EXPECT_THAT(histogram_tester().GetAllSamples( "NewTabPage.Snippets.FetchHttpResponseOrErrorCode"), ElementsAre(base::Bucket(/*min=*/200, /*count=*/1))); @@ -488,9 +485,10 @@ TEST_F(RemoteSuggestionsSignedInFetcherTest, ShouldFetchSuccessfully) { SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); EXPECT_CALL(mock_callback(), - Run(IsSuccess(), - AllOf(IsSingleArticle("http://localhost/foobar"), - FirstCategoryHasInfo(IsCategoryInfoForArticles())))); + Run(Property(&Status::IsSuccess, true), + /*fetched_categories=*/AllOf( + IsSingleArticle("http://localhost/foobar"), + FirstCategoryHasInfo(IsCategoryInfoForArticles())))); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); @@ -499,8 +497,8 @@ TEST_F(RemoteSuggestionsSignedInFetcherTest, ShouldFetchSuccessfully) { // Wait for the fake response. FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_status(), Eq("OK")); - EXPECT_THAT(fetcher().last_json(), Eq(kJsonStr)); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), Eq("OK")); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), Eq(kJsonStr)); EXPECT_THAT(histogram_tester().GetAllSamples( "NewTabPage.Snippets.FetchHttpResponseOrErrorCode"), ElementsAre(base::Bucket(/*min=*/200, /*count=*/1))); @@ -533,9 +531,10 @@ TEST_F(RemoteSuggestionsSignedInFetcherTest, ShouldRetryWhenOAuthCancelled) { SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); EXPECT_CALL(mock_callback(), - Run(IsSuccess(), - AllOf(IsSingleArticle("http://localhost/foobar"), - FirstCategoryHasInfo(IsCategoryInfoForArticles())))); + Run(Property(&Status::IsSuccess, true), + /*fetched_categories=*/AllOf( + IsSingleArticle("http://localhost/foobar"), + FirstCategoryHasInfo(IsCategoryInfoForArticles())))); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); @@ -545,8 +544,8 @@ TEST_F(RemoteSuggestionsSignedInFetcherTest, ShouldRetryWhenOAuthCancelled) { // Wait for the fake response. FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_status(), Eq("OK")); - EXPECT_THAT(fetcher().last_json(), Eq(kJsonStr)); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), Eq("OK")); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), Eq(kJsonStr)); EXPECT_THAT(histogram_tester().GetAllSamples( "NewTabPage.Snippets.FetchHttpResponseOrErrorCode"), ElementsAre(base::Bucket(/*min=*/200, /*count=*/1))); @@ -563,12 +562,14 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, EmptyCategoryIsOK) { "}]}"; SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(IsSuccess(), IsEmptyArticleList())); + EXPECT_CALL(mock_callback(), + Run(Property(&Status::IsSuccess, true), + /*fetched_categories=*/IsEmptyArticleList())); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_status(), Eq("OK")); - EXPECT_THAT(fetcher().last_json(), Eq(kJsonStr)); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), Eq("OK")); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), Eq(kJsonStr)); EXPECT_THAT(histogram_tester().GetAllSamples( "NewTabPage.Snippets.FetchHttpResponseOrErrorCode"), ElementsAre(base::Bucket(/*min=*/200, /*count=*/1))); @@ -614,7 +615,8 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ServerCategories) { SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); RemoteSuggestionsFetcher::OptionalFetchedCategories fetched_categories; - EXPECT_CALL(mock_callback(), Run(IsSuccess(), _)) + EXPECT_CALL(mock_callback(), + Run(Property(&Status::IsSuccess, true), /*fetched_categories=*/_)) .WillOnce(MoveArgument1PointeeTo(&fetched_categories)); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); @@ -639,8 +641,8 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ServerCategories) { } } - EXPECT_THAT(fetcher().last_status(), Eq("OK")); - EXPECT_THAT(fetcher().last_json(), Eq(kJsonStr)); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), Eq("OK")); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), Eq(kJsonStr)); EXPECT_THAT(histogram_tester().GetAllSamples( "NewTabPage.Snippets.FetchHttpResponseOrErrorCode"), ElementsAre(base::Bucket(/*min=*/200, /*count=*/1))); @@ -675,7 +677,8 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); RemoteSuggestionsFetcher::OptionalFetchedCategories fetched_categories; - EXPECT_CALL(mock_callback(), Run(IsSuccess(), _)) + EXPECT_CALL(mock_callback(), + Run(Property(&Status::IsSuccess, true), /*fetched_categories=*/_)) .WillOnce(MoveArgument1PointeeTo(&fetched_categories)); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); @@ -740,7 +743,8 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ExclusiveCategoryOnly) { SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); RemoteSuggestionsFetcher::OptionalFetchedCategories fetched_categories; - EXPECT_CALL(mock_callback(), Run(IsSuccess(), _)) + EXPECT_CALL(mock_callback(), + Run(Property(&Status::IsSuccess, true), /*fetched_categories=*/_)) .WillOnce(MoveArgument1PointeeTo(&fetched_categories)); RequestParams params = test_params(); @@ -763,14 +767,18 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ExclusiveCategoryOnly) { TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldNotFetchWithoutApiKey) { ResetFetcherWithAPIKey(std::string()); - EXPECT_CALL(mock_callback(), Run(HasCode(StatusCode::PERMANENT_ERROR), - /*snippets=*/Not(HasValue()))) + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::PERMANENT_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) .Times(1); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_status(), Eq("No API key available.")); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), + Eq("No API key available.")); EXPECT_THAT(histogram_tester().GetAllSamples( "NewTabPage.Snippets.FetchHttpResponseOrErrorCode"), IsEmpty()); @@ -783,12 +791,14 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, const std::string kJsonStr = "{\"categories\": []}"; SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(IsSuccess(), IsEmptyCategoriesList())); + EXPECT_CALL(mock_callback(), + Run(Property(&Status::IsSuccess, true), + /*fetched_categories=*/IsEmptyCategoriesList())); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_status(), Eq("OK")); - EXPECT_THAT(fetcher().last_json(), Eq(kJsonStr)); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), Eq("OK")); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), Eq(kJsonStr)); EXPECT_THAT( histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchResult"), ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); @@ -843,14 +853,18 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldReportUrlStatusError) { SetFakeResponse(/*response_data=*/std::string(), net::HTTP_NOT_FOUND, net::URLRequestStatus::FAILED); - EXPECT_CALL(mock_callback(), Run(HasCode(StatusCode::TEMPORARY_ERROR), - /*snippets=*/Not(HasValue()))) + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) .Times(1); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_status(), Eq("URLRequestStatus error -2")); - EXPECT_THAT(fetcher().last_json(), IsEmpty()); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), + Eq("URLRequestStatus error -2")); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), IsEmpty()); EXPECT_THAT( histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchResult"), ElementsAre(base::Bucket(/*min=*/2, /*count=*/1))); @@ -864,13 +878,16 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldReportUrlStatusError) { TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldReportHttpError) { SetFakeResponse(/*response_data=*/std::string(), net::HTTP_NOT_FOUND, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(HasCode(StatusCode::TEMPORARY_ERROR), - /*snippets=*/Not(HasValue()))) + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) .Times(1); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_json(), IsEmpty()); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), IsEmpty()); EXPECT_THAT( histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchResult"), ElementsAre(base::Bucket(/*min=*/3, /*count=*/1))); @@ -885,15 +902,18 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldReportJsonError) { const std::string kInvalidJsonStr = "{ \"recos\": []"; SetFakeResponse(/*response_data=*/kInvalidJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(HasCode(StatusCode::TEMPORARY_ERROR), - /*snippets=*/Not(HasValue()))) + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) .Times(1); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_status(), + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), StartsWith("Received invalid JSON (error ")); - EXPECT_THAT(fetcher().last_json(), Eq(kInvalidJsonStr)); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), Eq(kInvalidJsonStr)); EXPECT_THAT( histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchResult"), ElementsAre(base::Bucket(/*min=*/4, /*count=*/1))); @@ -909,13 +929,16 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldReportJsonErrorForEmptyResponse) { SetFakeResponse(/*response_data=*/std::string(), net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(HasCode(StatusCode::TEMPORARY_ERROR), - /*snippets=*/Not(HasValue()))) + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) .Times(1); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_json(), std::string()); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), std::string()); EXPECT_THAT( histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchResult"), ElementsAre(base::Bucket(/*min=*/4, /*count=*/1))); @@ -929,13 +952,62 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldReportInvalidListError) { "{\"recos\": [{ \"contentInfo\": { \"foo\" : \"bar\" }}]}"; SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(HasCode(StatusCode::TEMPORARY_ERROR), - /*snippets=*/Not(HasValue()))) + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) + .Times(1); + fetcher().FetchSnippets(test_params(), + ToSnippetsAvailableCallback(&mock_callback())); + FastForwardUntilNoTasksRemain(); + EXPECT_THAT(fetcher().GetLastJsonForDebugging(), Eq(kJsonStr)); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), + StartsWith("Invalid / empty list")); + EXPECT_THAT( + histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchResult"), + ElementsAre(base::Bucket(/*min=*/5, /*count=*/1))); + EXPECT_THAT(histogram_tester().GetAllSamples( + "NewTabPage.Snippets.FetchHttpResponseOrErrorCode"), + ElementsAre(base::Bucket(/*min=*/200, /*count=*/1))); + EXPECT_THAT(histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchTime"), + Not(IsEmpty())); +} + +TEST_F(RemoteSuggestionsSignedOutFetcherTest, + ShouldReportInvalidListErrorForIncompleteSuggestionButValidJson) { + // This is valid json, but it does not represent a valid suggestion + // (fullPageUrl is missing). + const std::string kValidJsonStr = + "{\"categories\" : [{" + " \"id\": 1," + " \"localizedTitle\": \"Articles for You\"," + " \"suggestions\" : [{" + " \"ids\" : [\"http://localhost/foobar\"]," + " \"title\" : \"Foo Barred from Baz\"," + " \"snippet\" : \"...\"," + " \"INVALID_fullPageUrl\" : \"http://localhost/foobar\"," + " \"creationTime\" : \"2016-06-30T11:01:37.000Z\"," + " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," + " \"attribution\" : \"Foo News\"," + " \"imageUrl\" : \"http://localhost/foobar.jpg\"," + " \"ampUrl\" : \"http://localhost/amp\"," + " \"faviconUrl\" : \"http://localhost/favicon.ico\" " + " }]" + "}]}"; + SetFakeResponse(/*response_data=*/kValidJsonStr, net::HTTP_OK, + net::URLRequestStatus::SUCCESS); + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) .Times(1); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); FastForwardUntilNoTasksRemain(); - EXPECT_THAT(fetcher().last_json(), Eq(kJsonStr)); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), + StartsWith("Invalid / empty list")); EXPECT_THAT( histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchResult"), ElementsAre(base::Bucket(/*min=*/5, /*count=*/1))); @@ -946,13 +1018,103 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldReportInvalidListError) { Not(IsEmpty())); } +TEST_F(RemoteSuggestionsSignedOutFetcherTest, + ShouldReportInvalidListErrorForInvalidTimestampButValidJson) { + // This is valid json, but it does not represent a valid suggestion + // (creationTime is invalid). + const std::string kValidJsonStr = + "{\"categories\" : [{" + " \"id\": 1," + " \"localizedTitle\": \"Articles for You\"," + " \"suggestions\" : [{" + " \"ids\" : [\"http://localhost/foobar\"]," + " \"title\" : \"Foo Barred from Baz\"," + " \"snippet\" : \"...\"," + " \"fullPageUrl\" : \"http://localhost/foobar\"," + " \"creationTime\" : \"INVALID_2016-06-30T11:01:37.000Z\"," + " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," + " \"attribution\" : \"Foo News\"," + " \"imageUrl\" : \"http://localhost/foobar.jpg\"," + " \"ampUrl\" : \"http://localhost/amp\"," + " \"faviconUrl\" : \"http://localhost/favicon.ico\" " + " }]" + "}]}"; + SetFakeResponse(/*response_data=*/kValidJsonStr, net::HTTP_OK, + net::URLRequestStatus::SUCCESS); + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) + .Times(1); + fetcher().FetchSnippets(test_params(), + ToSnippetsAvailableCallback(&mock_callback())); + FastForwardUntilNoTasksRemain(); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), + StartsWith("Invalid / empty list")); +} + +TEST_F(RemoteSuggestionsSignedOutFetcherTest, + ShouldReportInvalidListErrorForInvalidUrlButValidJson) { + // This is valid json, but it does not represent a valid suggestion + // (URL is invalid). + const std::string kValidJsonStr = + "{\"categories\" : [{" + " \"id\": 1," + " \"localizedTitle\": \"Articles for You\"," + " \"suggestions\" : [{" + " \"ids\" : [\"NOT A URL\"]," + " \"title\" : \"Foo Barred from Baz\"," + " \"snippet\" : \"...\"," + " \"fullPageUrl\" : \"NOT A URL\"," + " \"creationTime\" : \"2016-06-30T11:01:37.000Z\"," + " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," + " \"attribution\" : \"Foo News\"," + " \"imageUrl\" : \"http://localhost/foobar.jpg\"," + " \"ampUrl\" : \"http://localhost/amp\"," + " \"faviconUrl\" : \"http://localhost/favicon.ico\" " + " }]" + "}]}"; + SetFakeResponse(/*response_data=*/kValidJsonStr, net::HTTP_OK, + net::URLRequestStatus::SUCCESS); + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) + .Times(1); + fetcher().FetchSnippets(test_params(), + ToSnippetsAvailableCallback(&mock_callback())); + FastForwardUntilNoTasksRemain(); + EXPECT_THAT(fetcher().GetLastStatusForDebugging(), + StartsWith("Invalid / empty list")); +} + +TEST_F(RemoteSuggestionsSignedOutFetcherTest, + ShouldReportRequestFailureAsTemporaryError) { + SetFakeResponse(/*response_data=*/std::string(), net::HTTP_NOT_FOUND, + net::URLRequestStatus::FAILED); + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) + .Times(1); + fetcher().FetchSnippets(test_params(), + ToSnippetsAvailableCallback(&mock_callback())); + FastForwardUntilNoTasksRemain(); +} + // This test actually verifies that the test setup itself is sane, to prevent // hard-to-reproduce test failures. TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldReportHttpErrorForMissingBakedResponse) { InitFakeURLFetcherFactory(); - EXPECT_CALL(mock_callback(), Run(HasCode(StatusCode::TEMPORARY_ERROR), - /*snippets=*/Not(HasValue()))) + EXPECT_CALL( + mock_callback(), + Run(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + /*fetched_categories=*/Property( + &base::Optional<std::vector<FetchedCategory>>::has_value, false))) .Times(1); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); @@ -963,7 +1125,9 @@ TEST_F(RemoteSuggestionsSignedOutFetcherTest, ShouldProcessConcurrentFetches) { const std::string kJsonStr = "{ \"categories\": [] }"; SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(IsSuccess(), IsEmptyCategoriesList())) + EXPECT_CALL(mock_callback(), + Run(Property(&Status::IsSuccess, true), + /*fetched_categories=*/IsEmptyCategoriesList())) .Times(5); fetcher().FetchSnippets(test_params(), ToSnippetsAvailableCallback(&mock_callback())); diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_provider.h b/chromium/components/ntp_snippets/remote/remote_suggestions_provider.h index 5e94b8a9db6..a2b76f5fa33 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_provider.h +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_provider.h @@ -41,6 +41,9 @@ class RemoteSuggestionsProvider : public ContentSuggestionsProvider { virtual GURL GetUrlWithFavicon( const ContentSuggestion::ID& suggestion_id) const = 0; + // Whether the service is explicity disabled. + virtual bool IsDisabled() const = 0; + protected: RemoteSuggestionsProvider(Observer* observer); }; diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl.cc b/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl.cc index a3a8b9f6873..c13ee94413e 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl.cc @@ -12,6 +12,7 @@ #include "base/command_line.h" #include "base/location.h" #include "base/memory/ptr_util.h" +#include "base/metrics/field_trial_params.h" #include "base/metrics/histogram_macros.h" #include "base/metrics/sparse_histogram.h" #include "base/stl_util.h" @@ -21,7 +22,6 @@ #include "base/time/time.h" #include "base/values.h" #include "components/data_use_measurement/core/data_use_user_data.h" -#include "components/image_fetcher/core/image_decoder.h" #include "components/image_fetcher/core/image_fetcher.h" #include "components/ntp_snippets/category_rankers/category_ranker.h" #include "components/ntp_snippets/features.h" @@ -33,8 +33,6 @@ #include "components/prefs/pref_service.h" #include "components/strings/grit/components_strings.h" #include "components/variations/variations_associated_data.h" -#include "net/traffic_annotation/network_traffic_annotation.h" -#include "ui/gfx/geometry/size.h" #include "ui/gfx/image/image.h" namespace ntp_snippets { @@ -63,7 +61,26 @@ const char kCategoryContentAllowFetchingMore[] = "allow_fetching_more"; const char kOrderNewRemoteCategoriesBasedOnArticlesCategory[] = "order_new_remote_categories_based_on_articles_category"; +// Variation parameter for additional prefetched suggestions quantity. Not more +// than this number of prefetched suggestions will be kept longer. +const char kMaxAdditionalPrefetchedSuggestionsParamName[] = + "max_additional_prefetched_suggestions"; + +const int kDefaultMaxAdditionalPrefetchedSuggestions = 5; + +// Variation parameter for additional prefetched suggestions age. Only +// prefetched suggestions fetched not later than this are considered to be kept +// longer. +const char kMaxAgeForAdditionalPrefetchedSuggestionParamName[] = + "max_age_for_additional_prefetched_suggestion_minutes"; + +const base::TimeDelta kDefaultMaxAgeForAdditionalPrefetchedSuggestion = + base::TimeDelta::FromHours(36); + bool IsOrderingNewRemoteCategoriesBasedOnArticlesCategoryEnabled() { + // TODO(vitaliii): Use GetFieldTrialParamByFeature(As.*)? from + // base/metrics/field_trial_params.h. GetVariationParamByFeature(As.*)? are + // deprecated. return variations::GetVariationParamByFeatureAsBool( ntp_snippets::kArticleSuggestionsFeature, kOrderNewRemoteCategoriesBasedOnArticlesCategory, @@ -72,12 +89,11 @@ bool IsOrderingNewRemoteCategoriesBasedOnArticlesCategoryEnabled() { void AddFetchedCategoriesToRankerBasedOnArticlesCategory( CategoryRanker* ranker, - const RemoteSuggestionsFetcher::FetchedCategoriesVector& fetched_categories, + const FetchedCategoriesVector& fetched_categories, Category articles_category) { DCHECK(IsOrderingNewRemoteCategoriesBasedOnArticlesCategoryEnabled()); // Insert categories which precede "Articles" in the response. - for (const RemoteSuggestionsFetcher::FetchedCategory& fetched_category : - fetched_categories) { + for (const FetchedCategory& fetched_category : fetched_categories) { if (fetched_category.category == articles_category) { break; } @@ -99,6 +115,24 @@ void AddFetchedCategoriesToRankerBasedOnArticlesCategory( NOTREACHED() << "Articles category was not found."; } +bool IsKeepingPrefetchedSuggestionsEnabled() { + return base::FeatureList::IsEnabled(kKeepPrefetchedContentSuggestions); +} + +int GetMaxAdditionalPrefetchedSuggestions() { + return base::GetFieldTrialParamByFeatureAsInt( + kKeepPrefetchedContentSuggestions, + kMaxAdditionalPrefetchedSuggestionsParamName, + kDefaultMaxAdditionalPrefetchedSuggestions); +} + +base::TimeDelta GetMaxAgeForAdditionalPrefetchedSuggestion() { + return base::TimeDelta::FromMinutes(base::GetFieldTrialParamByFeatureAsInt( + kKeepPrefetchedContentSuggestions, + kMaxAgeForAdditionalPrefetchedSuggestionParamName, + kDefaultMaxAgeForAdditionalPrefetchedSuggestion.InMinutes())); +} + template <typename SuggestionPtrContainer> std::unique_ptr<std::vector<std::string>> GetSuggestionIDVector( const SuggestionPtrContainer& suggestions) { @@ -112,7 +146,7 @@ std::unique_ptr<std::vector<std::string>> GetSuggestionIDVector( bool HasIntersection(const std::vector<std::string>& a, const std::set<std::string>& b) { for (const std::string& item : a) { - if (base::ContainsValue(b, item)) { + if (b.count(item)) { return true; } } @@ -125,7 +159,7 @@ void EraseByPrimaryID(RemoteSuggestion::PtrVector* suggestions, base::EraseIf( *suggestions, [&ids_lookup](const std::unique_ptr<RemoteSuggestion>& suggestion) { - return base::ContainsValue(ids_lookup, suggestion->id()); + return ids_lookup.count(suggestion->id()); }); } @@ -210,130 +244,6 @@ void AddDismissedIdsToRequest(const RemoteSuggestion::PtrVector& dismissed, } // namespace -CachedImageFetcher::CachedImageFetcher( - std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, - PrefService* pref_service, - RemoteSuggestionsDatabase* database) - : image_fetcher_(std::move(image_fetcher)), - database_(database), - thumbnail_requests_throttler_( - pref_service, - RequestThrottler::RequestType::CONTENT_SUGGESTION_THUMBNAIL) { - // |image_fetcher_| can be null in tests. - if (image_fetcher_) { - image_fetcher_->SetImageFetcherDelegate(this); - image_fetcher_->SetDataUseServiceName( - data_use_measurement::DataUseUserData::NTP_SNIPPETS_THUMBNAILS); - } -} - -CachedImageFetcher::~CachedImageFetcher() {} - -void CachedImageFetcher::FetchSuggestionImage( - const ContentSuggestion::ID& suggestion_id, - const GURL& url, - const ImageFetchedCallback& callback) { - database_->LoadImage( - suggestion_id.id_within_category(), - base::Bind(&CachedImageFetcher::OnImageFetchedFromDatabase, - base::Unretained(this), callback, suggestion_id, url)); -} - -// This function gets only called for caching the image data received from the -// network. The actual decoding is done in OnImageDecodedFromDatabase(). -void CachedImageFetcher::OnImageDataFetched( - const std::string& id_within_category, - const std::string& image_data) { - if (image_data.empty()) { - return; - } - database_->SaveImage(id_within_category, image_data); -} - -void CachedImageFetcher::OnImageDecodingDone( - const ImageFetchedCallback& callback, - const std::string& id_within_category, - const gfx::Image& image, - const image_fetcher::RequestMetadata& metadata) { - callback.Run(image); -} - -void CachedImageFetcher::OnImageFetchedFromDatabase( - const ImageFetchedCallback& callback, - const ContentSuggestion::ID& suggestion_id, - const GURL& url, - std::string data) { // SnippetImageCallback requires by-value. - // The image decoder is null in tests. - if (image_fetcher_->GetImageDecoder() && !data.empty()) { - image_fetcher_->GetImageDecoder()->DecodeImage( - data, - // We're not dealing with multi-frame images. - /*desired_image_frame_size=*/gfx::Size(), - base::Bind(&CachedImageFetcher::OnImageDecodedFromDatabase, - base::Unretained(this), callback, suggestion_id, url)); - return; - } - // Fetching from the DB failed; start a network fetch. - FetchImageFromNetwork(suggestion_id, url, callback); -} - -void CachedImageFetcher::OnImageDecodedFromDatabase( - const ImageFetchedCallback& callback, - const ContentSuggestion::ID& suggestion_id, - const GURL& url, - const gfx::Image& image) { - if (!image.IsEmpty()) { - callback.Run(image); - return; - } - // If decoding the image failed, delete the DB entry. - database_->DeleteImage(suggestion_id.id_within_category()); - FetchImageFromNetwork(suggestion_id, url, callback); -} - -void CachedImageFetcher::FetchImageFromNetwork( - const ContentSuggestion::ID& suggestion_id, - const GURL& url, - const ImageFetchedCallback& callback) { - if (url.is_empty() || !thumbnail_requests_throttler_.DemandQuotaForRequest( - /*interactive_request=*/true)) { - // Return an empty image. Directly, this is never synchronous with the - // original FetchSuggestionImage() call - an asynchronous database query has - // happened in the meantime. - callback.Run(gfx::Image()); - return; - } - - net::NetworkTrafficAnnotationTag traffic_annotation = - net::DefineNetworkTrafficAnnotation("remote_suggestions_provider", R"( - semantics { - sender: "Content Suggestion Thumbnail Fetch" - description: - "Retrieves thumbnails for content suggestions, for display on the " - "New Tab page or Chrome Home." - trigger: - "Triggered when the user looks at a content suggestion (and its " - "thumbnail isn't cached yet)." - data: "None." - destination: GOOGLE_OWNED_SERVICE - } - policy { - cookies_allowed: false - setting: "Currently not available, but in progress: crbug.com/703684" - chrome_policy { - NTPContentSuggestionsEnabled { - policy_options {mode: MANDATORY} - NTPContentSuggestionsEnabled: false - } - } - })"); - image_fetcher_->StartOrQueueNetworkRequest( - suggestion_id.id_within_category(), url, - base::Bind(&CachedImageFetcher::OnImageDecodingDone, - base::Unretained(this), callback), - traffic_annotation); -} - RemoteSuggestionsProviderImpl::RemoteSuggestionsProviderImpl( Observer* observer, PrefService* pref_service, @@ -343,7 +253,8 @@ RemoteSuggestionsProviderImpl::RemoteSuggestionsProviderImpl( std::unique_ptr<RemoteSuggestionsFetcher> suggestions_fetcher, std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, std::unique_ptr<RemoteSuggestionsDatabase> database, - std::unique_ptr<RemoteSuggestionsStatusService> status_service) + std::unique_ptr<RemoteSuggestionsStatusService> status_service, + std::unique_ptr<PrefetchedPagesTracker> prefetched_pages_tracker) : RemoteSuggestionsProvider(observer), state_(State::NOT_INITED), pref_service_(pref_service), @@ -359,7 +270,8 @@ RemoteSuggestionsProviderImpl::RemoteSuggestionsProviderImpl( fetch_when_ready_(false), fetch_when_ready_interactive_(false), clear_history_dependent_state_when_initialized_(false), - clock_(base::MakeUnique<base::DefaultClock>()) { + clock_(base::MakeUnique<base::DefaultClock>()), + prefetched_pages_tracker_(std::move(prefetched_pages_tracker)) { RestoreCategoriesFromPrefs(); // The articles category always exists. Add it if we didn't get it from prefs. // TODO(treib): Rethink this. @@ -436,6 +348,10 @@ GURL RemoteSuggestionsProviderImpl::GetUrlWithFavicon( return ContentSuggestion::GetFaviconDomain(suggestion->url()); } +bool RemoteSuggestionsProviderImpl::IsDisabled() const { + return state_ == State::DISABLED; +} + void RemoteSuggestionsProviderImpl::FetchSuggestions( bool interactive_request, const FetchStatusCallback& callback) { @@ -739,11 +655,6 @@ void RemoteSuggestionsProviderImpl::OnFetchMoreFinished( // |fetched_category.suggestions|. ArchiveSuggestions(existing_content, &fetched_category.suggestions); - // TODO(tschumann): We should properly honor the existing category state, - // e.g. to make sure we don't serve results after the sign-out. Revisit this: - // Should Nuke also cancel outstanding requests, or do we want to check the - // status? - UpdateCategoryStatus(category, CategoryStatus::AVAILABLE); fetching_callback.Run(Status::Success(), std::move(result)); } @@ -758,6 +669,16 @@ void RemoteSuggestionsProviderImpl::OnFetchFinished( return; } + if (IsKeepingPrefetchedSuggestionsEnabled() && prefetched_pages_tracker_ && + !prefetched_pages_tracker_->IsInitialized()) { + // Wait until the tracker is initialized. + prefetched_pages_tracker_->AddInitializationCompletedCallback( + base::BindOnce(&RemoteSuggestionsProviderImpl::OnFetchFinished, + base::Unretained(this), callback, interactive_request, + status, std::move(fetched_categories))); + return; + } + // Record the fetch time of a successfull background fetch. if (!interactive_request && status.IsSuccess()) { pref_service_->SetInt64(prefs::kLastSuccessfulBackgroundFetchTime, @@ -782,8 +703,7 @@ void RemoteSuggestionsProviderImpl::OnFetchFinished( // TODO(treib): Reorder |category_contents_| to match the order we received // from the server. crbug.com/653816 bool response_includes_article_category = false; - for (RemoteSuggestionsFetcher::FetchedCategory& fetched_category : - *fetched_categories) { + for (FetchedCategory& fetched_category : *fetched_categories) { // TODO(tschumann): Remove this histogram once we only talk to the content // suggestions cloud backend. if (fetched_category.category == articles_category_) { @@ -799,7 +719,8 @@ void RemoteSuggestionsProviderImpl::OnFetchFinished( content->included_in_last_server_response = true; SanitizeReceivedSuggestions(content->dismissed, &fetched_category.suggestions); - IntegrateSuggestions(content, std::move(fetched_category.suggestions)); + IntegrateSuggestions(fetched_category.category, content, + std::move(fetched_category.suggestions)); } // Add new remote categories to the ranker. @@ -808,8 +729,7 @@ void RemoteSuggestionsProviderImpl::OnFetchFinished( AddFetchedCategoriesToRankerBasedOnArticlesCategory( category_ranker_, *fetched_categories, articles_category_); } else { - for (const RemoteSuggestionsFetcher::FetchedCategory& fetched_category : - *fetched_categories) { + for (const FetchedCategory& fetched_category : *fetched_categories) { category_ranker_->AppendCategoryIfNecessary(fetched_category.category); } } @@ -876,6 +796,7 @@ void RemoteSuggestionsProviderImpl::SanitizeReceivedSuggestions( } void RemoteSuggestionsProviderImpl::IntegrateSuggestions( + Category category, CategoryContent* content, RemoteSuggestion::PtrVector new_suggestions) { DCHECK(ready()); @@ -896,6 +817,51 @@ void RemoteSuggestionsProviderImpl::IntegrateSuggestions( // IDs though). EraseByPrimaryID(&content->suggestions, *GetSuggestionIDVector(new_suggestions)); + + // If enabled, keep some older prefetched article suggestions, otherwise the + // user has little time to see them. + if (IsKeepingPrefetchedSuggestionsEnabled() && + category == articles_category_ && prefetched_pages_tracker_) { + DCHECK(prefetched_pages_tracker_->IsInitialized()); + + // Select suggestions to keep. + std::sort(content->suggestions.begin(), content->suggestions.end(), + [](const std::unique_ptr<RemoteSuggestion>& first, + const std::unique_ptr<RemoteSuggestion>& second) { + return first->fetch_date() > second->fetch_date(); + }); + std::vector<std::unique_ptr<RemoteSuggestion>> + additional_prefetched_suggestions, other_suggestions; + for (auto& remote_suggestion : content->suggestions) { + const GURL& url = remote_suggestion->amp_url().is_empty() + ? remote_suggestion->url() + : remote_suggestion->amp_url(); + if (prefetched_pages_tracker_->PrefetchedOfflinePageExists(url) && + clock_->Now() - remote_suggestion->fetch_date() < + GetMaxAgeForAdditionalPrefetchedSuggestion() && + static_cast<int>(additional_prefetched_suggestions.size()) < + GetMaxAdditionalPrefetchedSuggestions()) { + additional_prefetched_suggestions.push_back( + std::move(remote_suggestion)); + } else { + other_suggestions.push_back(std::move(remote_suggestion)); + } + } + + // Mix them into the new set according to their score. + for (auto& remote_suggestion : additional_prefetched_suggestions) { + new_suggestions.push_back(std::move(remote_suggestion)); + } + std::sort(new_suggestions.begin(), new_suggestions.end(), + [](const std::unique_ptr<RemoteSuggestion>& first, + const std::unique_ptr<RemoteSuggestion>& second) { + return first->score() > second->score(); + }); + + // Treat remaining suggestions as usual. + content->suggestions = std::move(other_suggestions); + } + // Do not delete the thumbnail images as they are still handy on open NTPs. database_->DeleteSnippets(GetSuggestionIDVector(content->suggestions)); // Note, that ArchiveSuggestions will clear |content->suggestions|. @@ -997,6 +963,8 @@ void RemoteSuggestionsProviderImpl::ClearSuggestions() { } void RemoteSuggestionsProviderImpl::NukeAllSuggestions() { + // TODO(tschumann): Should Nuke also cancel outstanding requests? Or should we + // only block the results of such outstanding requests? for (const auto& item : category_contents_) { Category category = item.first; const CategoryContent& content = item.second; diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl.h b/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl.h index d84d00d4548..ef70fc46b17 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl.h +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl.h @@ -20,11 +20,13 @@ #include "base/optional.h" #include "base/time/clock.h" #include "base/time/time.h" -#include "components/image_fetcher/core/image_fetcher_delegate.h" #include "components/ntp_snippets/category.h" #include "components/ntp_snippets/category_status.h" #include "components/ntp_snippets/content_suggestion.h" #include "components/ntp_snippets/content_suggestions_provider.h" +#include "components/ntp_snippets/remote/cached_image_fetcher.h" +#include "components/ntp_snippets/remote/json_to_categories.h" +#include "components/ntp_snippets/remote/prefetched_pages_tracker.h" #include "components/ntp_snippets/remote/remote_suggestion.h" #include "components/ntp_snippets/remote/remote_suggestions_fetcher.h" #include "components/ntp_snippets/remote/remote_suggestions_provider.h" @@ -35,13 +37,8 @@ class PrefRegistrySimple; class PrefService; -namespace gfx { -class Image; -} // namespace gfx - namespace image_fetcher { class ImageFetcher; -struct RequestMetadata; } // namespace image_fetcher namespace ntp_snippets { @@ -50,58 +47,6 @@ class CategoryRanker; class RemoteSuggestionsDatabase; class RemoteSuggestionsScheduler; -// CachedImageFetcher takes care of fetching images from the network and caching -// them in the database. -// TODO(tschumann): Move into a separate library and inject the -// CachedImageFetcher into the RemoteSuggestionsProvider. This allows us to get -// rid of exposing this member for testing and lets us test the caching logic -// separately. -class CachedImageFetcher : public image_fetcher::ImageFetcherDelegate { - public: - // |pref_service| and |database| need to outlive the created image fetcher - // instance. - CachedImageFetcher(std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, - PrefService* pref_service, - RemoteSuggestionsDatabase* database); - ~CachedImageFetcher() override; - - // Fetches the image for a suggestion. The fetcher will first issue a lookup - // to the underlying cache with a fallback to the network. - void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, - const GURL& image_url, - const ImageFetchedCallback& callback); - - private: - // image_fetcher::ImageFetcherDelegate implementation. - void OnImageDataFetched(const std::string& id_within_category, - const std::string& image_data) override; - - void OnImageDecodingDone(const ImageFetchedCallback& callback, - const std::string& id_within_category, - const gfx::Image& image, - const image_fetcher::RequestMetadata& metadata); - void OnImageFetchedFromDatabase( - const ImageFetchedCallback& callback, - const ContentSuggestion::ID& suggestion_id, - const GURL& image_url, - // SnippetImageCallback requires by-value (not const ref). - std::string data); - void OnImageDecodedFromDatabase(const ImageFetchedCallback& callback, - const ContentSuggestion::ID& suggestion_id, - const GURL& url, - const gfx::Image& image); - void FetchImageFromNetwork(const ContentSuggestion::ID& suggestion_id, - const GURL& url, - const ImageFetchedCallback& callback); - - std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher_; - RemoteSuggestionsDatabase* database_; - // Request throttler for limiting requests to thumbnail images. - RequestThrottler thumbnail_requests_throttler_; - - DISALLOW_COPY_AND_ASSIGN(CachedImageFetcher); -}; - // Retrieves fresh content data (articles) from the server, stores them and // provides them as content suggestions. // This class is final because it does things in its constructor which make it @@ -122,7 +67,8 @@ class RemoteSuggestionsProviderImpl final : public RemoteSuggestionsProvider { std::unique_ptr<RemoteSuggestionsFetcher> suggestions_fetcher, std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, std::unique_ptr<RemoteSuggestionsDatabase> database, - std::unique_ptr<RemoteSuggestionsStatusService> status_service); + std::unique_ptr<RemoteSuggestionsStatusService> status_service, + std::unique_ptr<PrefetchedPagesTracker> prefetched_pages_tracker); ~RemoteSuggestionsProviderImpl() override; @@ -148,6 +94,8 @@ class RemoteSuggestionsProviderImpl final : public RemoteSuggestionsProvider { GURL GetUrlWithFavicon( const ContentSuggestion::ID& suggestion_id) const override; + bool IsDisabled() const override; + // ContentSuggestionsProvider implementation. CategoryStatus GetCategoryStatus(Category category) override; CategoryInfo GetCategoryInfo(Category category) override; @@ -333,8 +281,9 @@ class RemoteSuggestionsProviderImpl final : public RemoteSuggestionsProvider { void SanitizeReceivedSuggestions(const RemoteSuggestion::PtrVector& dismissed, RemoteSuggestion::PtrVector* suggestions); - // Adds newly available suggestions to |content|. - void IntegrateSuggestions(CategoryContent* content, + // Adds newly available suggestions to |content| corresponding to |category|. + void IntegrateSuggestions(Category category, + CategoryContent* content, RemoteSuggestion::PtrVector new_suggestions); // Dismisses a suggestion within a given category content. @@ -463,6 +412,9 @@ class RemoteSuggestionsProviderImpl final : public RemoteSuggestionsProvider { // A clock for getting the time. This allows to inject a clock in tests. std::unique_ptr<base::Clock> clock_; + // Prefetched pages tracker to query which urls have been prefetched. + std::unique_ptr<PrefetchedPagesTracker> prefetched_pages_tracker_; + DISALLOW_COPY_AND_ASSIGN(RemoteSuggestionsProviderImpl); }; diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl_unittest.cc b/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl_unittest.cc index 0759b4f1cb6..116dbad428b 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl_unittest.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_provider_impl_unittest.cc @@ -4,7 +4,9 @@ #include "components/ntp_snippets/remote/remote_suggestions_provider_impl.h" +#include <map> #include <memory> +#include <string> #include <utility> #include <vector> @@ -20,6 +22,7 @@ #include "base/strings/string_number_conversions.h" #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" +#include "base/strings/utf_string_conversions.h" #include "base/test/histogram_tester.h" #include "base/test/simple_test_clock.h" #include "base/threading/thread_task_runner_handle.h" @@ -38,10 +41,12 @@ #include "components/ntp_snippets/features.h" #include "components/ntp_snippets/ntp_snippets_constants.h" #include "components/ntp_snippets/pref_names.h" +#include "components/ntp_snippets/remote/json_to_categories.h" #include "components/ntp_snippets/remote/persistent_scheduler.h" +#include "components/ntp_snippets/remote/proto/ntp_snippets.pb.h" #include "components/ntp_snippets/remote/remote_suggestion.h" #include "components/ntp_snippets/remote/remote_suggestions_database.h" -#include "components/ntp_snippets/remote/remote_suggestions_fetcher.h" +#include "components/ntp_snippets/remote/remote_suggestions_fetcher_impl.h" #include "components/ntp_snippets/remote/remote_suggestions_scheduler.h" #include "components/ntp_snippets/remote/test_utils.h" #include "components/ntp_snippets/user_classifier.h" @@ -50,8 +55,6 @@ #include "components/signin/core/browser/fake_signin_manager.h" #include "components/variations/variations_params_manager.h" #include "net/traffic_annotation/network_traffic_annotation_test_helper.h" -#include "net/url_request/test_url_fetcher_factory.h" -#include "net/url_request/url_request_test_util.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gmock_mutant.h" #include "testing/gtest/include/gtest/gtest.h" @@ -62,9 +65,11 @@ using image_fetcher::ImageFetcher; using image_fetcher::ImageFetcherDelegate; using testing::_; +using testing::Contains; using testing::CreateFunctor; using testing::ElementsAre; using testing::Eq; +using testing::Field; using testing::InSequence; using testing::Invoke; using testing::IsEmpty; @@ -72,6 +77,7 @@ using testing::Mock; using testing::MockFunction; using testing::NiceMock; using testing::Not; +using testing::Property; using testing::Return; using testing::SaveArg; using testing::SizeIs; @@ -83,33 +89,24 @@ namespace ntp_snippets { namespace { -MATCHER_P(IdEq, value, "") { - return arg->id() == value; +ACTION_P(MoveFirstArgumentPointeeTo, ptr) { + // 0-based indexation. + *ptr = std::move(*arg0); } -MATCHER_P(IdWithinCategoryEq, expected_id, "") { - return arg.id().id_within_category() == expected_id; +ACTION_P(MoveSecondArgumentPointeeTo, ptr) { + // 0-based indexation. + *ptr = std::move(*arg1); } -MATCHER_P(HasCode, code, "") { - return arg.code == code; -} +const int kMaxExcludedDismissedIds = 100; const base::Time::Exploded kDefaultCreationTime = {2015, 11, 4, 25, 13, 46, 45}; -const char kTestContentSuggestionsServerEndpoint[] = - "https://alpha-chromecontentsuggestions-pa.sandbox.googleapis.com/v1/" - "suggestions/fetch"; -const char kAPIKey[] = "fakeAPIkey"; -const char kTestContentSuggestionsServerWithAPIKey[] = - "https://alpha-chromecontentsuggestions-pa.sandbox.googleapis.com/v1/" - "suggestions/fetch?key=fakeAPIkey"; const char kSuggestionUrl[] = "http://localhost/foobar"; const char kSuggestionTitle[] = "Title"; const char kSuggestionText[] = "Suggestion"; -const char kSuggestionSalientImage[] = "http://localhost/salient_image"; const char kSuggestionPublisherName[] = "Foo News"; -const char kSuggestionAmpUrl[] = "http://localhost/amp"; const char kSuggestionUrl2[] = "http://foo.com/bar"; @@ -117,6 +114,12 @@ const char kTestJsonDefaultCategoryTitle[] = "Some title"; const int kUnknownRemoteCategoryId = 1234; +// Different from default values to confirm that variation param values are +// used. +const int kMaxAdditionalPrefetchedSuggestions = 7; +const base::TimeDelta kMaxAgeForAdditionalPrefetchedSuggestion = + base::TimeDelta::FromHours(48); + base::Time GetDefaultCreationTime() { base::Time out_time; EXPECT_TRUE(base::Time::FromUTCExploded(kDefaultCreationTime, &out_time)); @@ -127,163 +130,194 @@ base::Time GetDefaultExpirationTime() { return base::Time::Now() + base::TimeDelta::FromHours(1); } -std::string GetCategoryJson(const std::vector<std::string>& suggestions, - int remote_category_id, - const std::string& category_title) { - return base::StringPrintf( - " {\n" - " \"id\": %d,\n" - " \"localizedTitle\": \"%s\",\n" - " \"suggestions\": [%s]\n" - " }\n", - remote_category_id, category_title.c_str(), - base::JoinString(suggestions, ", ").c_str()); -} - -class MultiCategoryJsonBuilder { +// TODO(vitaliii): Remove this and use RemoteSuggestionBuilder instead. +std::unique_ptr<RemoteSuggestion> CreateTestRemoteSuggestion( + const std::string& url) { + SnippetProto snippet_proto; + snippet_proto.add_ids(url); + snippet_proto.set_title("title"); + snippet_proto.set_snippet("snippet"); + snippet_proto.set_salient_image_url(url + "p.jpg"); + snippet_proto.set_publish_date(GetDefaultCreationTime().ToTimeT()); + snippet_proto.set_expiry_date(GetDefaultExpirationTime().ToTimeT()); + snippet_proto.set_remote_category_id(1); + auto* source = snippet_proto.add_sources(); + source->set_url(url); + source->set_publisher_name("Publisher"); + source->set_amp_url(url + "amp"); + return RemoteSuggestion::CreateFromProto(snippet_proto); +} + +class RemoteSuggestionBuilder { public: - MultiCategoryJsonBuilder() {} - - MultiCategoryJsonBuilder& AddCategoryWithCustomTitle( - const std::vector<std::string>& suggestions, - int remote_category_id, - const std::string& category_title) { - category_json_.push_back( - GetCategoryJson(suggestions, remote_category_id, category_title)); + RemoteSuggestionBuilder() = default; + + RemoteSuggestionBuilder& AddId(const std::string& id) { + if (!ids_) { + ids_ = std::vector<std::string>(); + } + ids_->push_back(id); return *this; } - - MultiCategoryJsonBuilder& AddCategory( - const std::vector<std::string>& suggestions, - int remote_category_id) { - return AddCategoryWithCustomTitle( - suggestions, remote_category_id, - "Title" + base::IntToString(remote_category_id)); + RemoteSuggestionBuilder& SetTitle(const std::string& title) { + title_ = title; + return *this; + } + RemoteSuggestionBuilder& SetSnippet(const std::string& snippet) { + snippet_ = snippet; + return *this; + } + RemoteSuggestionBuilder& SetImageUrl(const std::string& image_url) { + salient_image_url_ = image_url; + return *this; + } + RemoteSuggestionBuilder& SetPublishDate(const base::Time& publish_date) { + publish_date_ = publish_date; + return *this; + } + RemoteSuggestionBuilder& SetExpiryDate(const base::Time& expiry_date) { + expiry_date_ = expiry_date; + return *this; + } + RemoteSuggestionBuilder& SetScore(double score) { + score_ = score; + return *this; + } + RemoteSuggestionBuilder& SetIsDismissed(bool is_dismissed) { + is_dismissed_ = is_dismissed; + return *this; + } + RemoteSuggestionBuilder& SetRemoteCategoryId(int remote_category_id) { + remote_category_id_ = remote_category_id; + return *this; + } + RemoteSuggestionBuilder& SetUrl(const std::string& url) { + url_ = url; + return *this; + } + RemoteSuggestionBuilder& SetPublisher(const std::string& publisher) { + publisher_name_ = publisher; + return *this; + } + RemoteSuggestionBuilder& SetAmpUrl(const std::string& amp_url) { + amp_url_ = amp_url; + return *this; + } + RemoteSuggestionBuilder& SetFetchDate(const base::Time& fetch_date) { + fetch_date_ = fetch_date; + return *this; } - std::string Build() { - return base::StringPrintf( - "{\n" - " \"categories\": [\n" - "%s\n" - " ]\n" - "}\n", - base::JoinString(category_json_, " ,\n").c_str()); + std::unique_ptr<RemoteSuggestion> Build() const { + SnippetProto proto; + proto.set_title(title_.value_or("Title")); + proto.set_snippet(snippet_.value_or("Snippet")); + proto.set_salient_image_url( + salient_image_url_.value_or("http://image_url.com/")); + proto.set_publish_date( + publish_date_.value_or(GetDefaultCreationTime()).ToInternalValue()); + proto.set_expiry_date( + expiry_date_.value_or(GetDefaultExpirationTime()).ToInternalValue()); + proto.set_score(score_.value_or(1)); + proto.set_dismissed(is_dismissed_.value_or(false)); + proto.set_remote_category_id(remote_category_id_.value_or(1)); + auto* source = proto.add_sources(); + source->set_url(url_.value_or("http://url.com/")); + source->set_publisher_name(publisher_name_.value_or("Publisher")); + source->set_amp_url(amp_url_.value_or("http://amp_url.com/")); + proto.set_fetch_date( + fetch_date_.value_or(base::Time::Now()).ToInternalValue()); + for (const auto& id : + ids_.value_or(std::vector<std::string>{source->url()})) { + proto.add_ids(id); + } + return RemoteSuggestion::CreateFromProto(proto); } private: - std::vector<std::string> category_json_; + base::Optional<std::vector<std::string>> ids_; + base::Optional<std::string> title_; + base::Optional<std::string> snippet_; + base::Optional<std::string> salient_image_url_; + base::Optional<base::Time> publish_date_; + base::Optional<base::Time> expiry_date_; + base::Optional<double> score_; + base::Optional<bool> is_dismissed_; + base::Optional<int> remote_category_id_; + base::Optional<std::string> url_; + base::Optional<std::string> publisher_name_; + base::Optional<std::string> amp_url_; + base::Optional<base::Time> fetch_date_; }; -// TODO(vitaliii): Remove these convenience functions as they do not provide -// that much value and add additional redirections obscuring the code. -std::string GetTestJson(const std::vector<std::string>& suggestions, - const std::string& category_title) { - return MultiCategoryJsonBuilder() - .AddCategoryWithCustomTitle(suggestions, /*remote_category_id=*/1, - category_title) - .Build(); -} - -std::string GetTestJson(const std::vector<std::string>& suggestions) { - return GetTestJson(suggestions, kTestJsonDefaultCategoryTitle); -} - -std::string FormatTime(const base::Time& t) { - base::Time::Exploded x; - t.UTCExplode(&x); - return base::StringPrintf("%04d-%02d-%02dT%02d:%02d:%02dZ", x.year, x.month, - x.day_of_month, x.hour, x.minute, x.second); -} - -std::string GetSuggestionWithUrlAndTimesAndSource( - const std::vector<std::string>& ids, - const std::string& url, - const base::Time& creation_time, - const base::Time& expiry_time, - const std::string& publisher, - const std::string& amp_url) { - const std::string ids_string = base::JoinString(ids, "\",\n \""); - return base::StringPrintf( - "{\n" - " \"ids\": [\n" - " \"%s\"\n" - " ],\n" - " \"title\": \"%s\",\n" - " \"snippet\": \"%s\",\n" - " \"fullPageUrl\": \"%s\",\n" - " \"creationTime\": \"%s\",\n" - " \"expirationTime\": \"%s\",\n" - " \"attribution\": \"%s\",\n" - " \"imageUrl\": \"%s\",\n" - " \"ampUrl\": \"%s\"\n" - " }", - ids_string.c_str(), kSuggestionTitle, kSuggestionText, url.c_str(), - FormatTime(creation_time).c_str(), FormatTime(expiry_time).c_str(), - publisher.c_str(), kSuggestionSalientImage, amp_url.c_str()); -} - -std::string GetSuggestionWithSources(const std::string& source_url, - const std::string& publisher, - const std::string& amp_url) { - return GetSuggestionWithUrlAndTimesAndSource( - {kSuggestionUrl}, source_url, GetDefaultCreationTime(), - GetDefaultExpirationTime(), publisher, amp_url); -} - -std::string GetSuggestionWithUrlAndTimes( - const std::string& url, - const base::Time& content_creation_time, - const base::Time& expiry_time) { - return GetSuggestionWithUrlAndTimesAndSource( - {url}, url, content_creation_time, expiry_time, kSuggestionPublisherName, - kSuggestionAmpUrl); -} - -std::string GetSuggestionWithTimes(const base::Time& content_creation_time, - const base::Time& expiry_time) { - return GetSuggestionWithUrlAndTimes(kSuggestionUrl, content_creation_time, - expiry_time); -} - -std::string GetSuggestionWithUrl(const std::string& url) { - return GetSuggestionWithUrlAndTimes(url, GetDefaultCreationTime(), - GetDefaultExpirationTime()); -} - -std::string GetSuggestion() { - return GetSuggestionWithUrlAndTimes(kSuggestionUrl, GetDefaultCreationTime(), - GetDefaultExpirationTime()); -} - -std::string GetSuggestionN(int n) { - return GetSuggestionWithUrlAndTimes( - base::StringPrintf("%s/%d", kSuggestionUrl, n), GetDefaultCreationTime(), - GetDefaultExpirationTime()); -} - -std::string GetExpiredSuggestion() { - return GetSuggestionWithTimes(GetDefaultCreationTime(), base::Time::Now()); -} +class FetchedCategoryBuilder { + public: + FetchedCategoryBuilder() = default; -std::string GetInvalidSuggestion() { - std::string json_str = GetSuggestion(); - // Make the json invalid by removing the final closing brace. - return json_str.substr(0, json_str.size() - 1); -} + FetchedCategoryBuilder& SetCategory(Category category) { + category_ = category; + return *this; + } + FetchedCategoryBuilder& SetTitle(const std::string& title) { + title_ = base::UTF8ToUTF16(title); + return *this; + } + FetchedCategoryBuilder& SetCardLayout( + ContentSuggestionsCardLayout card_layout) { + card_layout_ = card_layout; + return *this; + } + FetchedCategoryBuilder& SetAdditionalAction( + ContentSuggestionsAdditionalAction additional_action) { + additional_action_ = additional_action; + return *this; + } + FetchedCategoryBuilder& SetShowIfEmpty(bool show_if_empty) { + show_if_empty_ = show_if_empty; + return *this; + } + FetchedCategoryBuilder& SetNoSuggestionsMessage( + const std::string& no_suggestions_message) { + no_suggestions_message_ = base::UTF8ToUTF16(no_suggestions_message); + return *this; + } + FetchedCategoryBuilder& AddSuggestionViaBuilder( + const RemoteSuggestionBuilder& builder) { + if (!suggestion_builders_) { + suggestion_builders_ = std::vector<RemoteSuggestionBuilder>(); + } + suggestion_builders_->push_back(builder); + return *this; + } -std::string GetIncompleteSuggestion() { - std::string json_str = GetSuggestion(); - // Rename the "url" entry. The result is syntactically valid json that will - // fail to parse as suggestions. - size_t pos = json_str.find("\"fullPageUrl\""); - if (pos == std::string::npos) { - NOTREACHED(); - return std::string(); + FetchedCategory Build() const { + FetchedCategory result = FetchedCategory( + category_.value_or(Category::FromRemoteCategory(1)), + CategoryInfo( + title_.value_or(base::UTF8ToUTF16("Category title")), + card_layout_.value_or(ContentSuggestionsCardLayout::FULL_CARD), + additional_action_.value_or( + ContentSuggestionsAdditionalAction::FETCH), + show_if_empty_.value_or(false), + no_suggestions_message_.value_or( + base::UTF8ToUTF16("No suggestions message")))); + + if (suggestion_builders_) { + for (const auto& suggestion_builder : *suggestion_builders_) + result.suggestions.push_back(suggestion_builder.Build()); + } + return result; } - json_str[pos + 1] = 'x'; - return json_str; -} + + private: + base::Optional<Category> category_; + base::Optional<base::string16> title_; + base::Optional<ContentSuggestionsCardLayout> card_layout_; + base::Optional<ContentSuggestionsAdditionalAction> additional_action_; + base::Optional<bool> show_if_empty_; + base::Optional<base::string16> no_suggestions_message_; + base::Optional<std::vector<RemoteSuggestionBuilder>> suggestion_builders_; +}; using ServeImageCallback = base::Callback<void( const std::string&, @@ -303,49 +337,22 @@ void ServeOneByOneImage( notify->OnImageDataFetched(id, "1-by-1-image-data"); } -gfx::Image FetchImage(RemoteSuggestionsProviderImpl* service, +gfx::Image FetchImage(RemoteSuggestionsProviderImpl* provider, const ContentSuggestion::ID& suggestion_id) { gfx::Image result; base::RunLoop run_loop; - service->FetchSuggestionImage(suggestion_id, - base::Bind( - [](base::Closure signal, gfx::Image* output, - const gfx::Image& loaded) { - *output = loaded; - signal.Run(); - }, - run_loop.QuitClosure(), &result)); + provider->FetchSuggestionImage( + suggestion_id, base::Bind( + [](base::Closure signal, gfx::Image* output, + const gfx::Image& loaded) { + *output = loaded; + signal.Run(); + }, + run_loop.QuitClosure(), &result)); run_loop.Run(); return result; } -void ParseJson(const std::string& json, - const SuccessCallback& success_callback, - const ErrorCallback& error_callback) { - base::JSONReader json_reader; - std::unique_ptr<base::Value> value = json_reader.ReadToValue(json); - if (value) { - success_callback.Run(std::move(value)); - } else { - error_callback.Run(json_reader.GetErrorMessage()); - } -} - -// Factory for FakeURLFetcher objects that always generate errors. -class FailingFakeURLFetcherFactory : public net::URLFetcherFactory { - public: - std::unique_ptr<net::URLFetcher> CreateURLFetcher( - int id, - const GURL& url, - net::URLFetcher::RequestType request_type, - net::URLFetcherDelegate* d, - net::NetworkTrafficAnnotationTag traffic_annotation) override { - return base::MakeUnique<net::FakeURLFetcher>( - url, d, /*response_data=*/std::string(), net::HTTP_NOT_FOUND, - net::URLRequestStatus::FAILED); - } -}; - class MockImageFetcher : public ImageFetcher { public: MOCK_METHOD1(SetImageFetcherDelegate, void(ImageFetcherDelegate*)); @@ -389,9 +396,42 @@ class MockScheduler : public RemoteSuggestionsScheduler { MOCK_METHOD1(OnInteractiveFetchFinished, void(Status fetch_status)); MOCK_METHOD0(OnBrowserForegrounded, void()); MOCK_METHOD0(OnBrowserColdStart, void()); - MOCK_METHOD0(OnNTPOpened, void()); + MOCK_METHOD0(OnSuggestionsSurfaceOpened, void()); MOCK_METHOD0(OnPersistentSchedulerWakeUp, void()); - MOCK_METHOD0(RescheduleFetching, void()); + MOCK_METHOD0(OnBrowserUpgraded, void()); +}; + +class MockRemoteSuggestionsFetcher : public RemoteSuggestionsFetcher { + public: + // GMock does not support movable-only types (SnippetsAvailableCallback is + // OnceCallback), therefore, the call is redirected to a mock method with a + // pointer to the callback. + void FetchSnippets(const RequestParams& params, + SnippetsAvailableCallback callback) override { + FetchSnippets(params, &callback); + } + MOCK_METHOD2(FetchSnippets, + void(const RequestParams& params, + SnippetsAvailableCallback* callback)); + MOCK_CONST_METHOD0(GetLastStatusForDebugging, const std::string&()); + MOCK_CONST_METHOD0(GetLastJsonForDebugging, const std::string&()); + MOCK_CONST_METHOD0(GetFetchUrlForDebugging, const GURL&()); +}; + +class MockPrefetchedPagesTracker : public PrefetchedPagesTracker { + public: + MOCK_CONST_METHOD0(IsInitialized, bool()); + + // GMock does not support movable-only types (e.g. OnceCallback), therefore, + // the call is redirected to a mock method with a pointer to the callback. + void AddInitializationCompletedCallback( + base::OnceCallback<void()> callback) override { + AddInitializationCompletedCallback(&callback); + } + MOCK_METHOD1(AddInitializationCompletedCallback, + void(base::OnceCallback<void()>* callback)); + + MOCK_CONST_METHOD1(PrefetchedOfflinePageExists, bool(const GURL& url)); }; } // namespace @@ -399,17 +439,10 @@ class MockScheduler : public RemoteSuggestionsScheduler { class RemoteSuggestionsProviderImplTest : public ::testing::Test { public: RemoteSuggestionsProviderImplTest() - : params_manager_(ntp_snippets::kArticleSuggestionsFeature.name, - {{"content_suggestions_backend", - kTestContentSuggestionsServerEndpoint}}, - {ntp_snippets::kArticleSuggestionsFeature.name}), - fake_url_fetcher_factory_( - /*default_factory=*/&failing_url_fetcher_factory_), - test_url_(kTestContentSuggestionsServerWithAPIKey), - category_ranker_(base::MakeUnique<ConstantCategoryRanker>()), + : category_ranker_(base::MakeUnique<ConstantCategoryRanker>()), user_classifier_(/*pref_service=*/nullptr, base::MakeUnique<base::DefaultClock>()), - suggestions_fetcher_(nullptr), + mock_suggestions_fetcher_(nullptr), image_fetcher_(nullptr), scheduler_(base::MakeUnique<NiceMock<MockScheduler>>()), database_(nullptr) { @@ -428,28 +461,31 @@ class RemoteSuggestionsProviderImplTest : public ::testing::Test { } // TODO(vitaliii): Rewrite this function to initialize a test class member - // instead of creating a new service. - std::unique_ptr<RemoteSuggestionsProviderImpl> MakeSuggestionsProvider( - bool set_empty_response = true) { - auto service = MakeSuggestionsProviderWithoutInitialization(); - WaitForSuggestionsProviderInitialization(service.get(), set_empty_response); - return service; + // instead of creating a new provider. + std::unique_ptr<RemoteSuggestionsProviderImpl> MakeSuggestionsProvider() { + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + WaitForSuggestionsProviderInitialization(provider.get()); + return provider; } std::unique_ptr<RemoteSuggestionsProviderImpl> - MakeSuggestionsProviderWithoutInitialization() { + MakeSuggestionsProviderWithoutInitialization( + bool use_mock_prefetched_pages_tracker) { scoped_refptr<base::SingleThreadTaskRunner> task_runner( base::ThreadTaskRunnerHandle::Get()); - scoped_refptr<net::TestURLRequestContextGetter> request_context_getter = - new net::TestURLRequestContextGetter(task_runner.get()); utils_.ResetSigninManager(); - auto suggestions_fetcher = base::MakeUnique<RemoteSuggestionsFetcher>( - utils_.fake_signin_manager(), /*token_service=*/nullptr, - std::move(request_context_getter), utils_.pref_service(), nullptr, - base::Bind(&ParseJson), GetFetchEndpoint(version_info::Channel::STABLE), - kAPIKey, &user_classifier_); - suggestions_fetcher_ = suggestions_fetcher.get(); + auto mock_suggestions_fetcher = + base::MakeUnique<StrictMock<MockRemoteSuggestionsFetcher>>(); + mock_suggestions_fetcher_ = mock_suggestions_fetcher.get(); + + std::unique_ptr<PrefetchedPagesTracker> prefetched_pages_tracker; + if (use_mock_prefetched_pages_tracker) { + prefetched_pages_tracker = + base::MakeUnique<StrictMock<MockPrefetchedPagesTracker>>(); + } + prefetched_pages_tracker_ = prefetched_pages_tracker.get(); auto image_fetcher = base::MakeUnique<NiceMock<MockImageFetcher>>(); @@ -464,43 +500,36 @@ class RemoteSuggestionsProviderImplTest : public ::testing::Test { database_ = database.get(); return base::MakeUnique<RemoteSuggestionsProviderImpl>( observer_.get(), utils_.pref_service(), "fr", category_ranker_.get(), - scheduler_.get(), std::move(suggestions_fetcher), + scheduler_.get(), std::move(mock_suggestions_fetcher), std::move(image_fetcher), std::move(database), base::MakeUnique<RemoteSuggestionsStatusService>( - utils_.fake_signin_manager(), utils_.pref_service(), - std::string())); + utils_.fake_signin_manager(), utils_.pref_service(), std::string()), + std::move(prefetched_pages_tracker)); } std::unique_ptr<RemoteSuggestionsProviderImpl> MakeSuggestionsProviderWithoutInitializationWithStrictScheduler() { scheduler_ = base::MakeUnique<StrictMock<MockScheduler>>(); - return MakeSuggestionsProviderWithoutInitialization(); + return MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); } void WaitForSuggestionsProviderInitialization( - RemoteSuggestionsProviderImpl* service, - bool set_empty_response) { + RemoteSuggestionsProviderImpl* provider) { EXPECT_EQ(RemoteSuggestionsProviderImpl::State::NOT_INITED, - service->state_); - - // Add an initial fetch response, as the service tries to fetch when there - // is nothing in the DB. - if (set_empty_response) { - SetUpFetchResponse(GetTestJson(std::vector<std::string>())); - } + provider->state_); // TODO(treib): Find a better way to wait for initialization to finish. base::RunLoop().RunUntilIdle(); EXPECT_NE(RemoteSuggestionsProviderImpl::State::NOT_INITED, - service->state_); + provider->state_); } void ResetSuggestionsProvider( - std::unique_ptr<RemoteSuggestionsProviderImpl>* service, - bool set_empty_response) { - service->reset(); + std::unique_ptr<RemoteSuggestionsProviderImpl>* provider) { + provider->reset(); observer_.reset(); - *service = MakeSuggestionsProvider(set_empty_response); + *provider = MakeSuggestionsProvider(); } void SetCategoryRanker(std::unique_ptr<CategoryRanker> category_ranker) { @@ -528,10 +557,12 @@ class RemoteSuggestionsProviderImplTest : public ::testing::Test { } protected: - const GURL& test_url() { return test_url_; } FakeContentSuggestionsProviderObserver& observer() { return *observer_; } - RemoteSuggestionsFetcher* suggestions_fetcher() { - return suggestions_fetcher_; + StrictMock<MockRemoteSuggestionsFetcher>* mock_suggestions_fetcher() { + return mock_suggestions_fetcher_; + } + PrefetchedPagesTracker* prefetched_pages_tracker() { + return prefetched_pages_tracker_; } // TODO(tschumann): Make this a strict-mock. We want to avoid unneccesary // network requests. @@ -541,66 +572,82 @@ class RemoteSuggestionsProviderImplTest : public ::testing::Test { RemoteSuggestionsDatabase* database() { return database_; } MockScheduler* scheduler() { return scheduler_.get(); } - // Provide the json to be returned by the fake fetcher. - void SetUpFetchResponse(const std::string& json) { - fake_url_fetcher_factory_.SetFakeResponse(test_url_, json, net::HTTP_OK, - net::URLRequestStatus::SUCCESS); - } - - // Have the fake fetcher fail due to a HTTP error like a 404. - void SetUpHttpError() { - fake_url_fetcher_factory_.SetFakeResponse(test_url_, /*json=*/std::string(), - net::HTTP_NOT_FOUND, - net::URLRequestStatus::SUCCESS); - } - - void LoadFromJSONString(RemoteSuggestionsProviderImpl* service, - const std::string& json) { - SetUpFetchResponse(json); - service->FetchSuggestions(/*interactive_request=*/true, - RemoteSuggestionsProvider::FetchStatusCallback()); - base::RunLoop().RunUntilIdle(); + void FetchTheseSuggestions( + RemoteSuggestionsProviderImpl* provider, + bool interactive_request, + Status status, + base::Optional<std::vector<FetchedCategory>> fetched_categories) { + RemoteSuggestionsFetcher::SnippetsAvailableCallback snippets_callback; + EXPECT_CALL(*mock_suggestions_fetcher(), FetchSnippets(_, _)) + .WillOnce(MoveSecondArgumentPointeeTo(&snippets_callback)) + .RetiresOnSaturation(); + provider->FetchSuggestions( + interactive_request, RemoteSuggestionsProvider::FetchStatusCallback()); + std::move(snippets_callback) + .Run(Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); } - void LoadMoreFromJSONString(RemoteSuggestionsProviderImpl* service, - const Category& category, - const std::string& json, - const std::set<std::string>& known_ids, - FetchDoneCallback callback) { - SetUpFetchResponse(json); + void FetchMoreTheseSuggestions( + RemoteSuggestionsProviderImpl* provider, + const Category& category, + const std::set<std::string>& known_suggestion_ids, + FetchDoneCallback fetch_done_callback, + Status status, + base::Optional<std::vector<FetchedCategory>> fetched_categories) { + RemoteSuggestionsFetcher::SnippetsAvailableCallback snippets_callback; + EXPECT_CALL(*mock_suggestions_fetcher(), FetchSnippets(_, _)) + .WillOnce(MoveSecondArgumentPointeeTo(&snippets_callback)) + .RetiresOnSaturation(); EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) .WillOnce(Return(true)) .RetiresOnSaturation(); - service->Fetch(category, known_ids, callback); - base::RunLoop().RunUntilIdle(); + provider->Fetch(category, known_suggestion_ids, fetch_done_callback); + std::move(snippets_callback) + .Run(Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); } void SetOrderNewRemoteCategoriesBasedOnArticlesCategoryParam(bool value) { // params_manager supports only one // |SetVariationParamsWithFeatureAssociations| at a time, so we clear - // previous settings first and then set everything we need. + // previous settings first to make this explicit. params_manager_.ClearAllVariationParams(); params_manager_.SetVariationParamsWithFeatureAssociations( kArticleSuggestionsFeature.name, {{"order_new_remote_categories_based_on_articles_category", - value ? "true" : "false"}, - {"content_suggestions_backend", - kTestContentSuggestionsServerEndpoint}}, + value ? "true" : "false"}}, {kArticleSuggestionsFeature.name}); } + void EnableKeepingPrefetchedContentSuggestions( + int max_additional_prefetched_suggestions, + const base::TimeDelta& max_age_for_additional_prefetched_suggestion) { + // params_manager supports only one + // |SetVariationParamsWithFeatureAssociations| at a time, so we clear + // previous settings first to make this explicit. + params_manager_.ClearAllVariationParams(); + params_manager_.SetVariationParamsWithFeatureAssociations( + kKeepPrefetchedContentSuggestions.name, + { + {"max_additional_prefetched_suggestions", + base::IntToString(max_additional_prefetched_suggestions)}, + {"max_age_for_additional_prefetched_suggestion_minutes", + base::IntToString( + max_age_for_additional_prefetched_suggestion.InMinutes())}, + }, + {kKeepPrefetchedContentSuggestions.name}); + } + private: variations::testing::VariationParamsManager params_manager_; test::RemoteSuggestionsTestUtils utils_; base::MessageLoop message_loop_; - FailingFakeURLFetcherFactory failing_url_fetcher_factory_; - // Instantiation of factory automatically sets itself as URLFetcher's factory. - net::FakeURLFetcherFactory fake_url_fetcher_factory_; - const GURL test_url_; std::unique_ptr<CategoryRanker> category_ranker_; UserClassifier user_classifier_; std::unique_ptr<FakeContentSuggestionsProviderObserver> observer_; - RemoteSuggestionsFetcher* suggestions_fetcher_; + StrictMock<MockRemoteSuggestionsFetcher>* mock_suggestions_fetcher_; + PrefetchedPagesTracker* prefetched_pages_tracker_; NiceMock<MockImageFetcher>* image_fetcher_; FakeImageDecoder image_decoder_; std::unique_ptr<MockScheduler> scheduler_; @@ -612,15 +659,27 @@ class RemoteSuggestionsProviderImplTest : public ::testing::Test { }; TEST_F(RemoteSuggestionsProviderImplTest, Full) { - std::string json_str(GetTestJson({GetSuggestion()})); - - auto service = MakeSuggestionsProvider(); - - LoadFromJSONString(service.get(), json_str); + auto provider = MakeSuggestionsProvider(); + + // TODO(vitaliii): Inline the vector creation in FetchTheseSuggestions. + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId(kSuggestionUrl) + .SetTitle(kSuggestionTitle) + .SetSnippet(kSuggestionText) + .SetPublishDate(GetDefaultCreationTime()) + .SetPublisher(kSuggestionPublisherName)) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), SizeIs(1)); - ASSERT_THAT(service->GetSuggestionsForTesting(articles_category()), + ASSERT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); const ContentSuggestion& suggestion = @@ -640,27 +699,35 @@ TEST_F(RemoteSuggestionsProviderImplTest, CategoryTitle) { // Don't send an initial response -- we want to test what happens without any // server status. - auto service = MakeSuggestionsProvider(/*set_empty_response=*/false); + auto provider = MakeSuggestionsProvider(); // The articles category should be there by default, and have a title. - CategoryInfo info_before = service->GetCategoryInfo(articles_category()); + CategoryInfo info_before = provider->GetCategoryInfo(articles_category()); ASSERT_THAT(info_before.title(), Not(IsEmpty())); ASSERT_THAT(info_before.title(), Not(Eq(test_default_title))); EXPECT_THAT(info_before.additional_action(), Eq(ContentSuggestionsAdditionalAction::FETCH)); EXPECT_THAT(info_before.show_if_empty(), Eq(true)); - std::string json_str_with_title(GetTestJson({GetSuggestion()})); - LoadFromJSONString(service.get(), json_str_with_title); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .SetTitle(base::UTF16ToUTF8(test_default_title)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder()) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), SizeIs(1)); - ASSERT_THAT(service->GetSuggestionsForTesting(articles_category()), + ASSERT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); // The response contained a title, |kTestJsonDefaultCategoryTitle|. // Make sure we updated the title in the CategoryInfo. - CategoryInfo info_with_title = service->GetCategoryInfo(articles_category()); + CategoryInfo info_with_title = provider->GetCategoryInfo(articles_category()); EXPECT_THAT(info_before.title(), Not(Eq(info_with_title.title()))); EXPECT_THAT(test_default_title, Eq(info_with_title.title())); EXPECT_THAT(info_before.additional_action(), @@ -669,13 +736,33 @@ TEST_F(RemoteSuggestionsProviderImplTest, CategoryTitle) { } TEST_F(RemoteSuggestionsProviderImplTest, MultipleCategories) { - auto service = MakeSuggestionsProvider(); - std::string json_str = - MultiCategoryJsonBuilder() - .AddCategory({GetSuggestionN(0)}, /*remote_category_id=*/1) - .AddCategory({GetSuggestionN(1)}, /*remote_category_id=*/2) - .Build(); - LoadFromJSONString(service.get(), json_str); + auto provider = MakeSuggestionsProvider(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(1)) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder() + .AddId(base::StringPrintf("%s/%d", kSuggestionUrl, 0)) + .SetTitle(kSuggestionTitle) + .SetSnippet(kSuggestionText) + .SetPublishDate(GetDefaultCreationTime()) + .SetPublisher(kSuggestionPublisherName)) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(2)) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder() + .AddId(base::StringPrintf("%s/%d", kSuggestionUrl, 1)) + .SetTitle(kSuggestionTitle) + .SetSnippet(kSuggestionText) + .SetPublishDate(GetDefaultCreationTime()) + .SetPublisher(kSuggestionPublisherName)) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); ASSERT_THAT(observer().statuses(), Eq(std::map<Category, CategoryStatus, Category::CompareByID>{ @@ -683,9 +770,9 @@ TEST_F(RemoteSuggestionsProviderImplTest, MultipleCategories) { {other_category(), CategoryStatus::AVAILABLE}, })); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); - EXPECT_THAT(service->GetSuggestionsForTesting(other_category()), SizeIs(1)); + EXPECT_THAT(provider->GetSuggestionsForTesting(other_category()), SizeIs(1)); ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), SizeIs(1)); @@ -717,25 +804,34 @@ TEST_F(RemoteSuggestionsProviderImplTest, MultipleCategories) { } TEST_F(RemoteSuggestionsProviderImplTest, ArticleCategoryInfo) { - auto service = MakeSuggestionsProvider(); - CategoryInfo article_info = service->GetCategoryInfo(articles_category()); + auto provider = MakeSuggestionsProvider(); + CategoryInfo article_info = provider->GetCategoryInfo(articles_category()); EXPECT_THAT(article_info.additional_action(), Eq(ContentSuggestionsAdditionalAction::FETCH)); EXPECT_THAT(article_info.show_if_empty(), Eq(true)); } TEST_F(RemoteSuggestionsProviderImplTest, ExperimentalCategoryInfo) { - auto service = MakeSuggestionsProvider(); - std::string json_str = - MultiCategoryJsonBuilder() - .AddCategory({GetSuggestionN(0)}, /*remote_category_id=*/1) - .AddCategory({GetSuggestionN(1)}, kUnknownRemoteCategoryId) - .Build(); + auto provider = MakeSuggestionsProvider(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(1)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("1")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(kUnknownRemoteCategoryId)) + .SetAdditionalAction(ContentSuggestionsAdditionalAction::NONE) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("2")) + .Build()); // Load data with multiple categories so that a new experimental category gets // registered. - LoadFromJSONString(service.get(), json_str); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); - CategoryInfo info = service->GetCategoryInfo(unknown_category()); + CategoryInfo info = provider->GetCategoryInfo(unknown_category()); EXPECT_THAT(info.additional_action(), Eq(ContentSuggestionsAdditionalAction::NONE)); EXPECT_THAT(info.show_if_empty(), Eq(false)); @@ -745,12 +841,22 @@ TEST_F(RemoteSuggestionsProviderImplTest, AddRemoteCategoriesToCategoryRanker) { auto mock_ranker = base::MakeUnique<MockCategoryRanker>(); MockCategoryRanker* raw_mock_ranker = mock_ranker.get(); SetCategoryRanker(std::move(mock_ranker)); - std::string json_str = - MultiCategoryJsonBuilder() - .AddCategory({GetSuggestionN(0)}, /*remote_category_id=*/11) - .AddCategory({GetSuggestionN(1)}, /*remote_category_id=*/13) - .AddCategory({GetSuggestionN(2)}, /*remote_category_id=*/12) - .Build(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(11)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("11")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(13)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("13")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(12)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("12")) + .Build()); { // The order of categories is determined by the order in which they are // added. Thus, the latter is tested here. @@ -762,8 +868,10 @@ TEST_F(RemoteSuggestionsProviderImplTest, AddRemoteCategoriesToCategoryRanker) { EXPECT_CALL(*raw_mock_ranker, AppendCategoryIfNecessary(Category::FromRemoteCategory(12))); } - auto service = MakeSuggestionsProvider(/*set_empty_response=*/false); - LoadFromJSONString(service.get(), json_str); + auto provider = MakeSuggestionsProvider(); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); } TEST_F(RemoteSuggestionsProviderImplTest, @@ -772,14 +880,32 @@ TEST_F(RemoteSuggestionsProviderImplTest, auto mock_ranker = base::MakeUnique<MockCategoryRanker>(); MockCategoryRanker* raw_mock_ranker = mock_ranker.get(); SetCategoryRanker(std::move(mock_ranker)); - std::string json_str = - MultiCategoryJsonBuilder() - .AddCategory({GetSuggestionN(0)}, /*remote_category_id=*/14) - .AddCategory({GetSuggestionN(1)}, /*remote_category_id=*/13) - .AddCategory({GetSuggestionN(2)}, /*remote_category_id=*/1) - .AddCategory({GetSuggestionN(3)}, /*remote_category_id=*/12) - .AddCategory({GetSuggestionN(4)}, /*remote_category_id=*/11) - .Build(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(14)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("14")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(13)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("13")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(1)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("1")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(12)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("12")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(11)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("11")) + .Build()); { InSequence s; EXPECT_CALL(*raw_mock_ranker, @@ -795,8 +921,10 @@ TEST_F(RemoteSuggestionsProviderImplTest, InsertCategoryAfterIfNecessary(Category::FromRemoteCategory(12), articles_category())); } - auto service = MakeSuggestionsProvider(/*set_empty_response=*/false); - LoadFromJSONString(service.get(), json_str); + auto provider = MakeSuggestionsProvider(); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); } TEST_F( @@ -806,81 +934,105 @@ TEST_F( auto mock_ranker = base::MakeUnique<MockCategoryRanker>(); MockCategoryRanker* raw_mock_ranker = mock_ranker.get(); SetCategoryRanker(std::move(mock_ranker)); - std::string json_str = - MultiCategoryJsonBuilder() - .AddCategory({GetSuggestionN(0)}, /*remote_category_id=*/11) - .Build(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(11)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("11")) + .Build()); EXPECT_CALL(*raw_mock_ranker, InsertCategoryBeforeIfNecessary(_, _)).Times(0); EXPECT_CALL(*raw_mock_ranker, AppendCategoryIfNecessary(Category::FromRemoteCategory(11))); - auto service = MakeSuggestionsProvider(/*set_empty_response=*/false); - LoadFromJSONString(service.get(), json_str); + auto provider = MakeSuggestionsProvider(); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); } TEST_F(RemoteSuggestionsProviderImplTest, PersistCategoryInfos) { - auto service = MakeSuggestionsProvider(); - // TODO(vitaliii): Use |articles_category()| instead of constant ID below. - std::string json_str = - MultiCategoryJsonBuilder() - .AddCategoryWithCustomTitle( - {GetSuggestionN(0)}, /*remote_category_id=*/1, "Articles for You") - .AddCategoryWithCustomTitle({GetSuggestionN(1)}, - kUnknownRemoteCategoryId, "Other Things") - .Build(); - LoadFromJSONString(service.get(), json_str); + auto provider = MakeSuggestionsProvider(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("1")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(kUnknownRemoteCategoryId)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("2")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); ASSERT_EQ(observer().StatusForCategory(articles_category()), CategoryStatus::AVAILABLE); - ASSERT_EQ(observer().StatusForCategory(unknown_category()), + ASSERT_EQ(observer().StatusForCategory( + Category::FromRemoteCategory(kUnknownRemoteCategoryId)), CategoryStatus::AVAILABLE); CategoryInfo info_articles_before = - service->GetCategoryInfo(articles_category()); - CategoryInfo info_unknown_before = - service->GetCategoryInfo(unknown_category()); + provider->GetCategoryInfo(articles_category()); + CategoryInfo info_unknown_before = provider->GetCategoryInfo( + Category::FromRemoteCategory(kUnknownRemoteCategoryId)); - // Recreate the service to simulate a Chrome restart. - ResetSuggestionsProvider(&service, /*set_empty_response=*/true); + // Recreate the provider to simulate a Chrome restart. + ResetSuggestionsProvider(&provider); // The categories should have been restored. ASSERT_NE(observer().StatusForCategory(articles_category()), CategoryStatus::NOT_PROVIDED); - ASSERT_NE(observer().StatusForCategory(unknown_category()), + ASSERT_NE(observer().StatusForCategory( + Category::FromRemoteCategory(kUnknownRemoteCategoryId)), CategoryStatus::NOT_PROVIDED); EXPECT_EQ(observer().StatusForCategory(articles_category()), CategoryStatus::AVAILABLE); - EXPECT_EQ(observer().StatusForCategory(unknown_category()), + EXPECT_EQ(observer().StatusForCategory( + Category::FromRemoteCategory(kUnknownRemoteCategoryId)), CategoryStatus::AVAILABLE); CategoryInfo info_articles_after = - service->GetCategoryInfo(articles_category()); - CategoryInfo info_unknown_after = - service->GetCategoryInfo(unknown_category()); + provider->GetCategoryInfo(articles_category()); + CategoryInfo info_unknown_after = provider->GetCategoryInfo( + Category::FromRemoteCategory(kUnknownRemoteCategoryId)); EXPECT_EQ(info_articles_before.title(), info_articles_after.title()); EXPECT_EQ(info_unknown_before.title(), info_unknown_after.title()); } TEST_F(RemoteSuggestionsProviderImplTest, PersistRemoteCategoryOrder) { - // We create a service with a normal ranker to store the order. - std::string json_str = - MultiCategoryJsonBuilder() - .AddCategory({GetSuggestionN(0)}, /*remote_category_id=*/11) - .AddCategory({GetSuggestionN(1)}, /*remote_category_id=*/13) - .AddCategory({GetSuggestionN(2)}, /*remote_category_id=*/12) - .Build(); - auto service = MakeSuggestionsProvider(/*set_empty_response=*/false); - LoadFromJSONString(service.get(), json_str); - - // We manually recreate the service to simulate Chrome restart and enforce a - // mock ranker. The response is cleared to ensure that the order is not - // fetched. - SetUpFetchResponse(""); + // We create a provider with a normal ranker to store the order. + auto provider = MakeSuggestionsProvider(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(11)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("11")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(13)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("13")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(12)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("12")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + // We manually recreate the provider to simulate Chrome restart and enforce a + // mock ranker. auto mock_ranker = base::MakeUnique<MockCategoryRanker>(); MockCategoryRanker* raw_mock_ranker = mock_ranker.get(); SetCategoryRanker(std::move(mock_ranker)); + // Ensure that the order is not fetched. + EXPECT_CALL(*mock_suggestions_fetcher(), FetchSnippets(_, _)).Times(0); { // The order of categories is determined by the order in which they are // added. Thus, the latter is tested here. @@ -896,24 +1048,34 @@ TEST_F(RemoteSuggestionsProviderImplTest, PersistRemoteCategoryOrder) { EXPECT_CALL(*raw_mock_ranker, AppendCategoryIfNecessary(Category::FromRemoteCategory(12))); } - ResetSuggestionsProvider(&service, /*set_empty_response=*/false); + ResetSuggestionsProvider(&provider); } TEST_F(RemoteSuggestionsProviderImplTest, PersistSuggestions) { - auto service = MakeSuggestionsProvider(); - std::string json_str = - MultiCategoryJsonBuilder() - .AddCategory({GetSuggestionN(0)}, /*remote_category_id=*/1) - .AddCategory({GetSuggestionN(2)}, /*remote_category_id=*/2) - .Build(); - LoadFromJSONString(service.get(), json_str); + auto provider = MakeSuggestionsProvider(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(1)) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("1").SetRemoteCategoryId(1)) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(2)) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("2").SetRemoteCategoryId(2)) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), SizeIs(1)); ASSERT_THAT(observer().SuggestionsForCategory(other_category()), SizeIs(1)); - // Recreate the service to simulate a Chrome restart. - ResetSuggestionsProvider(&service, /*set_empty_response=*/true); + // Recreate the provider to simulate a Chrome restart. + ResetSuggestionsProvider(&provider); // The suggestions in both categories should have been restored. EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), @@ -923,29 +1085,36 @@ TEST_F(RemoteSuggestionsProviderImplTest, PersistSuggestions) { TEST_F(RemoteSuggestionsProviderImplTest, DontNotifyIfNotAvailable) { // Get some suggestions into the database. - auto service = MakeSuggestionsProvider(); - std::string json_str = - MultiCategoryJsonBuilder() - .AddCategory({GetSuggestionN(0)}, - /*remote_category_id=*/1) - .AddCategory({GetSuggestionN(1)}, /*remote_category_id=*/2) - .Build(); - LoadFromJSONString(service.get(), json_str); + auto provider = MakeSuggestionsProvider(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(1)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("1")) + .Build()); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(Category::FromRemoteCategory(2)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("2")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), SizeIs(1)); ASSERT_THAT(observer().SuggestionsForCategory(other_category()), SizeIs(1)); - service.reset(); + provider.reset(); // Set the pref that disables remote suggestions. pref_service()->SetBoolean(prefs::kEnableSnippets, false); - // Recreate the service to simulate a Chrome start. - ResetSuggestionsProvider(&service, /*set_empty_response=*/true); + // Recreate the provider to simulate a Chrome start. + ResetSuggestionsProvider(&provider); ASSERT_THAT(RemoteSuggestionsProviderImpl::State::DISABLED, - Eq(service->state_)); + Eq(provider->state_)); // Now the observer should not have received any suggestions. EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), @@ -954,294 +1123,336 @@ TEST_F(RemoteSuggestionsProviderImplTest, DontNotifyIfNotAvailable) { } TEST_F(RemoteSuggestionsProviderImplTest, Clear) { - auto service = MakeSuggestionsProvider(); - - std::string json_str(GetTestJson({GetSuggestion()})); - - LoadFromJSONString(service.get(), json_str); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("1")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); - service->ClearCachedSuggestions(articles_category()); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + provider->ClearCachedSuggestions(articles_category()); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), IsEmpty()); } TEST_F(RemoteSuggestionsProviderImplTest, ReplaceSuggestions) { - auto service = MakeSuggestionsProvider(); + auto provider = MakeSuggestionsProvider(); std::string first("http://first"); - LoadFromJSONString(service.get(), GetTestJson({GetSuggestionWithUrl(first)})); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), - ElementsAre(IdEq(first))); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId(first)) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), + ElementsAre(Pointee(Property(&RemoteSuggestion::id, first)))); std::string second("http://second"); - LoadFromJSONString(service.get(), - GetTestJson({GetSuggestionWithUrl(second)})); + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId(second)) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); // The suggestions loaded last replace all that was loaded previously. - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), - ElementsAre(IdEq(second))); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), + ElementsAre(Pointee(Property(&RemoteSuggestion::id, second)))); } -TEST_F(RemoteSuggestionsProviderImplTest, LoadsAdditionalSuggestions) { - auto service = MakeSuggestionsProvider(); +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldResolveFetchedSuggestionThumbnail) { + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("id")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + ASSERT_THAT(provider->GetSuggestionsForTesting(articles_category()), + ElementsAre(Pointee(Property(&RemoteSuggestion::id, "id")))); - LoadFromJSONString(service.get(), - GetTestJson({GetSuggestionWithUrl("http://first")})); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), - ElementsAre(IdEq("http://first"))); + image_decoder()->SetDecodedImage(gfx::test::CreateImage(1, 1)); + ServeImageCallback serve_one_by_one_image_callback = + base::Bind(&ServeOneByOneImage, &provider->GetImageFetcherForTesting()); + EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _, _)) + .WillOnce(WithArgs<0, 2>( + Invoke(CreateFunctor(serve_one_by_one_image_callback)))); + + gfx::Image image = FetchImage(provider.get(), MakeArticleID("id")); + ASSERT_FALSE(image.IsEmpty()); + EXPECT_EQ(1, image.Width()); +} + +TEST_F(RemoteSuggestionsProviderImplTest, ShouldFetchMore) { + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("first")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + ASSERT_THAT(provider->GetSuggestionsForTesting(articles_category()), + ElementsAre(Pointee(Property(&RemoteSuggestion::id, "first")))); auto expect_only_second_suggestion_received = base::Bind([](Status status, std::vector<ContentSuggestion> suggestions) { EXPECT_THAT(suggestions, SizeIs(1)); - EXPECT_THAT(suggestions[0].id().id_within_category(), - Eq("http://second")); + EXPECT_THAT(suggestions[0].id().id_within_category(), Eq("second")); }); - LoadMoreFromJSONString(service.get(), articles_category(), - GetTestJson({GetSuggestionWithUrl("http://second")}), - /*known_ids=*/std::set<std::string>(), - expect_only_second_suggestion_received); + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("second")) + .Build()); + FetchMoreTheseSuggestions( + provider.get(), articles_category(), + /*known_suggestion_ids=*/std::set<std::string>(), + /*fetch_done_callback=*/expect_only_second_suggestion_received, + Status(StatusCode::SUCCESS, "message"), std::move(fetched_categories)); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldResolveFetchedMoreSuggestionThumbnail) { + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId("id")) + .Build()); + + auto assert_only_first_suggestion_received = + base::Bind([](Status status, std::vector<ContentSuggestion> suggestions) { + ASSERT_THAT(suggestions, SizeIs(1)); + ASSERT_THAT(suggestions[0].id().id_within_category(), Eq("id")); + }); + FetchMoreTheseSuggestions( + provider.get(), articles_category(), + /*known_suggestion_ids=*/std::set<std::string>(), + /*fetch_done_callback=*/assert_only_first_suggestion_received, + Status(StatusCode::SUCCESS, "message"), std::move(fetched_categories)); - // Verify we can resolve the image of the new suggestions. - ServeImageCallback cb = - base::Bind(&ServeOneByOneImage, &service->GetImageFetcherForTesting()); - EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _, _)) - .Times(2) - .WillRepeatedly(WithArgs<0, 2>(Invoke(CreateFunctor(cb)))); image_decoder()->SetDecodedImage(gfx::test::CreateImage(1, 1)); - gfx::Image image = FetchImage(service.get(), MakeArticleID("http://first")); - EXPECT_FALSE(image.IsEmpty()); - EXPECT_EQ(1, image.Width()); + ServeImageCallback serve_one_by_one_image_callback = + base::Bind(&ServeOneByOneImage, &provider->GetImageFetcherForTesting()); + EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _, _)) + .WillOnce(WithArgs<0, 2>( + Invoke(CreateFunctor(serve_one_by_one_image_callback)))); - image = FetchImage(service.get(), MakeArticleID("http://second")); - EXPECT_FALSE(image.IsEmpty()); + gfx::Image image = FetchImage(provider.get(), MakeArticleID("id")); + ASSERT_FALSE(image.IsEmpty()); EXPECT_EQ(1, image.Width()); } -// The tests TestMergingFetchedMoreSuggestionsFillup and -// TestMergingFetchedMoreSuggestionsReplaceAll simulate the following user -// story: -// 1) fetch suggestions in NTP A -// 2) fetch more suggestions in NTP A. -// 3) open new NTP B: See the first 10 results from step 1). -// 4) fetch more suggestions in NTP B. Make sure the results are independent -// from step 2) -// TODO(tschumann): Test step 4) on a higher level instead of peeking into the -// internal 'dismissed' data. The proper check is to make sure we tell the -// backend to exclude these suggestions. +// Imagine that we have surfaces A and B. The user fetches more in A, this +// should not add any suggestions to B. TEST_F(RemoteSuggestionsProviderImplTest, - FewMoreFetchedSuggestionsShouldNotInterfere) { - auto service = MakeSuggestionsProvider(/*set_empty_response=*/false); - LoadFromJSONString(service.get(), - GetTestJson({GetSuggestionWithUrl("http://id-1"), - GetSuggestionWithUrl("http://id-2"), - GetSuggestionWithUrl("http://id-3"), - GetSuggestionWithUrl("http://id-4"), - GetSuggestionWithUrl("http://id-5"), - GetSuggestionWithUrl("http://id-6"), - GetSuggestionWithUrl("http://id-7"), - GetSuggestionWithUrl("http://id-8"), - GetSuggestionWithUrl("http://id-9"), - GetSuggestionWithUrl("http://id-10")})); - EXPECT_THAT( - observer().SuggestionsForCategory(articles_category()), - ElementsAre( - IdWithinCategoryEq("http://id-1"), IdWithinCategoryEq("http://id-2"), - IdWithinCategoryEq("http://id-3"), IdWithinCategoryEq("http://id-4"), - IdWithinCategoryEq("http://id-5"), IdWithinCategoryEq("http://id-6"), - IdWithinCategoryEq("http://id-7"), IdWithinCategoryEq("http://id-8"), - IdWithinCategoryEq("http://id-9"), - IdWithinCategoryEq("http://id-10"))); - - auto expect_receiving_two_new_suggestions = - base::Bind([](Status status, std::vector<ContentSuggestion> suggestions) { - ASSERT_THAT(suggestions, SizeIs(2)); - EXPECT_THAT(suggestions[0], IdWithinCategoryEq("http://more-id-1")); - EXPECT_THAT(suggestions[1], IdWithinCategoryEq("http://more-id-2")); - }); - LoadMoreFromJSONString( - service.get(), articles_category(), - GetTestJson({GetSuggestionWithUrl("http://more-id-1"), - GetSuggestionWithUrl("http://more-id-2")}), - /*known_ids=*/ - {"http://id-1", "http://id-2", "http://id-3", "http://id-4", - "http://id-5", "http://id-6", "http://id-7", "http://id-8", - "http://id-9", "http://id-10"}, - expect_receiving_two_new_suggestions); - - // Verify that the observer still has the old set. - EXPECT_THAT( - observer().SuggestionsForCategory(articles_category()), - ElementsAre( - IdWithinCategoryEq("http://id-1"), IdWithinCategoryEq("http://id-2"), - IdWithinCategoryEq("http://id-3"), IdWithinCategoryEq("http://id-4"), - IdWithinCategoryEq("http://id-5"), IdWithinCategoryEq("http://id-6"), - IdWithinCategoryEq("http://id-7"), IdWithinCategoryEq("http://id-8"), - IdWithinCategoryEq("http://id-9"), - IdWithinCategoryEq("http://id-10"))); - - // No interference from previous Fetch more: we can receive two other ones. - expect_receiving_two_new_suggestions = + ShouldNotChangeSuggestionsInOtherSurfacesWhenFetchingMore) { + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + WaitForSuggestionsProviderInitialization(provider.get()); + + // Fetch a suggestion. + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://old.com/")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + ElementsAre(Property(&ContentSuggestion::id, + MakeArticleID("http://old.com/")))); + + // Now fetch more, but first prepare a response. + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://fetched-more.com/")) + .Build()); + + // The surface issuing the fetch more gets response via callback. + auto assert_receiving_one_new_suggestion = base::Bind([](Status status, std::vector<ContentSuggestion> suggestions) { - ASSERT_THAT(suggestions, SizeIs(2)); - EXPECT_THAT(suggestions[0], IdWithinCategoryEq("http://more-id-3")); - EXPECT_THAT(suggestions[1], IdWithinCategoryEq("http://more-id-4")); + ASSERT_THAT(suggestions, SizeIs(1)); + ASSERT_THAT(suggestions[0].id().id_within_category(), + Eq("http://fetched-more.com/")); }); - LoadMoreFromJSONString( - service.get(), articles_category(), - GetTestJson({GetSuggestionWithUrl("http://more-id-3"), - GetSuggestionWithUrl("http://more-id-4")}), - /*known_ids=*/ - {"http://id-1", "http://id-2", "http://id-3", "http://id-4", - "http://id-5", "http://id-6", "http://id-7", "http://id-8", - "http://id-9", "http://id-10"}, - expect_receiving_two_new_suggestions); + FetchMoreTheseSuggestions( + provider.get(), articles_category(), + /*known_suggestion_ids=*/{"http://old.com/"}, + /*fetch_done_callback=*/assert_receiving_one_new_suggestion, + Status(StatusCode::SUCCESS, "message"), std::move(fetched_categories)); + + // Other surfaces should remain the same. + EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), + ElementsAre(Property(&ContentSuggestion::id, + MakeArticleID("http://old.com/")))); } +// Imagine that we have surfaces A and B. The user fetches more in A. This +// should not affect the next fetch more in B, i.e. assuming the same server +// response the same suggestions must be fetched in B if the user fetches more +// there as well. TEST_F(RemoteSuggestionsProviderImplTest, - TenMoreFetchedSuggestionsShouldNotInterfere) { - auto service = MakeSuggestionsProvider(/*set_empty_response=*/false); - LoadFromJSONString(service.get(), - GetTestJson({GetSuggestionWithUrl("http://id-1"), - GetSuggestionWithUrl("http://id-2"), - GetSuggestionWithUrl("http://id-3"), - GetSuggestionWithUrl("http://id-4"), - GetSuggestionWithUrl("http://id-5"), - GetSuggestionWithUrl("http://id-6"), - GetSuggestionWithUrl("http://id-7"), - GetSuggestionWithUrl("http://id-8"), - GetSuggestionWithUrl("http://id-9"), - GetSuggestionWithUrl("http://id-10")})); - EXPECT_THAT( - observer().SuggestionsForCategory(articles_category()), - ElementsAre( - IdWithinCategoryEq("http://id-1"), IdWithinCategoryEq("http://id-2"), - IdWithinCategoryEq("http://id-3"), IdWithinCategoryEq("http://id-4"), - IdWithinCategoryEq("http://id-5"), IdWithinCategoryEq("http://id-6"), - IdWithinCategoryEq("http://id-7"), IdWithinCategoryEq("http://id-8"), - IdWithinCategoryEq("http://id-9"), - IdWithinCategoryEq("http://id-10"))); - - auto expect_receiving_ten_new_suggestions = + ShouldNotAffectFetchMoreInOtherSurfacesWhenFetchingMore) { + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + WaitForSuggestionsProviderInitialization(provider.get()); + + // Fetch more on the surface A. + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back(FetchedCategory( + articles_category(), + BuildRemoteCategoryInfo(base::UTF8ToUTF16("title"), + /*allow_fetching_more_results=*/true))); + fetched_categories[0].suggestions.push_back( + CreateTestRemoteSuggestion("http://fetched-more.com/")); + + auto assert_receiving_one_new_suggestion = base::Bind([](Status status, std::vector<ContentSuggestion> suggestions) { - EXPECT_THAT(suggestions, - ElementsAre(IdWithinCategoryEq("http://more-id-1"), - IdWithinCategoryEq("http://more-id-2"), - IdWithinCategoryEq("http://more-id-3"), - IdWithinCategoryEq("http://more-id-4"), - IdWithinCategoryEq("http://more-id-5"), - IdWithinCategoryEq("http://more-id-6"), - IdWithinCategoryEq("http://more-id-7"), - IdWithinCategoryEq("http://more-id-8"), - IdWithinCategoryEq("http://more-id-9"), - IdWithinCategoryEq("http://more-id-10"))); + ASSERT_THAT(suggestions, SizeIs(1)); + ASSERT_THAT(suggestions[0].id().id_within_category(), + Eq("http://fetched-more.com/")); }); - LoadMoreFromJSONString( - service.get(), articles_category(), - GetTestJson({GetSuggestionWithUrl("http://more-id-1"), - GetSuggestionWithUrl("http://more-id-2"), - GetSuggestionWithUrl("http://more-id-3"), - GetSuggestionWithUrl("http://more-id-4"), - GetSuggestionWithUrl("http://more-id-5"), - GetSuggestionWithUrl("http://more-id-6"), - GetSuggestionWithUrl("http://more-id-7"), - GetSuggestionWithUrl("http://more-id-8"), - GetSuggestionWithUrl("http://more-id-9"), - GetSuggestionWithUrl("http://more-id-10")}), - /*known_ids=*/ - {"http://id-1", "http://id-2", "http://id-3", "http://id-4", - "http://id-5", "http://id-6", "http://id-7", "http://id-8", - "http://id-9", "http://id-10"}, - expect_receiving_ten_new_suggestions); - EXPECT_THAT( - observer().SuggestionsForCategory(articles_category()), - ElementsAre( - IdWithinCategoryEq("http://id-1"), IdWithinCategoryEq("http://id-2"), - IdWithinCategoryEq("http://id-3"), IdWithinCategoryEq("http://id-4"), - IdWithinCategoryEq("http://id-5"), IdWithinCategoryEq("http://id-6"), - IdWithinCategoryEq("http://id-7"), IdWithinCategoryEq("http://id-8"), - IdWithinCategoryEq("http://id-9"), - IdWithinCategoryEq("http://id-10"))); - - // This time, test receiving the same set. - expect_receiving_ten_new_suggestions = + RemoteSuggestionsFetcher::SnippetsAvailableCallback snippets_callback; + EXPECT_CALL(*mock_suggestions_fetcher(), FetchSnippets(_, _)) + .WillOnce(MoveSecondArgumentPointeeTo(&snippets_callback)) + .RetiresOnSaturation(); + EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) + .WillOnce(Return(true)) + .RetiresOnSaturation(); + provider->Fetch(articles_category(), + /*known_suggestion_ids=*/std::set<std::string>(), + assert_receiving_one_new_suggestion); + std::move(snippets_callback) + .Run(Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + // Now fetch more on the surface B. The response is the same as before. + fetched_categories.clear(); + fetched_categories.push_back(FetchedCategory( + articles_category(), + BuildRemoteCategoryInfo(base::UTF8ToUTF16("title"), + /*allow_fetching_more_results=*/true))); + fetched_categories[0].suggestions.push_back( + CreateTestRemoteSuggestion("http://fetched-more.com/")); + + // B should receive the same suggestion as was fetched more on A. + auto expect_receiving_same_suggestion = base::Bind([](Status status, std::vector<ContentSuggestion> suggestions) { - EXPECT_THAT(suggestions, - ElementsAre(IdWithinCategoryEq("http://more-id-1"), - IdWithinCategoryEq("http://more-id-2"), - IdWithinCategoryEq("http://more-id-3"), - IdWithinCategoryEq("http://more-id-4"), - IdWithinCategoryEq("http://more-id-5"), - IdWithinCategoryEq("http://more-id-6"), - IdWithinCategoryEq("http://more-id-7"), - IdWithinCategoryEq("http://more-id-8"), - IdWithinCategoryEq("http://more-id-9"), - IdWithinCategoryEq("http://more-id-10"))); + ASSERT_THAT(suggestions, SizeIs(1)); + EXPECT_THAT(suggestions[0].id().id_within_category(), + Eq("http://fetched-more.com/")); }); - LoadMoreFromJSONString( - service.get(), articles_category(), - GetTestJson({GetSuggestionWithUrl("http://more-id-1"), - GetSuggestionWithUrl("http://more-id-2"), - GetSuggestionWithUrl("http://more-id-3"), - GetSuggestionWithUrl("http://more-id-4"), - GetSuggestionWithUrl("http://more-id-5"), - GetSuggestionWithUrl("http://more-id-6"), - GetSuggestionWithUrl("http://more-id-7"), - GetSuggestionWithUrl("http://more-id-8"), - GetSuggestionWithUrl("http://more-id-9"), - GetSuggestionWithUrl("http://more-id-10")}), - /*known_ids=*/ - {"http://id-1", "http://id-2", "http://id-3", "http://id-4", - "http://id-5", "http://id-6", "http://id-7", "http://id-8", - "http://id-9", "http://id-10"}, - expect_receiving_ten_new_suggestions); + // The provider should not ask the fetcher to exclude the suggestion fetched + // more on A. + EXPECT_CALL(*mock_suggestions_fetcher(), + FetchSnippets(Field(&RequestParams::excluded_ids, + Not(Contains("http://fetched-more.com/"))), + _)) + .WillOnce(MoveSecondArgumentPointeeTo(&snippets_callback)) + .RetiresOnSaturation(); + EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) + .WillOnce(Return(true)) + .RetiresOnSaturation(); + provider->Fetch(articles_category(), + /*known_suggestion_ids=*/std::set<std::string>(), + expect_receiving_same_suggestion); + std::move(snippets_callback) + .Run(Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); } TEST_F(RemoteSuggestionsProviderImplTest, ClearHistoryShouldDeleteArchivedSuggestions) { - auto service = MakeSuggestionsProvider(/*set_empty_response=*/false); + auto provider = MakeSuggestionsProvider(); // First get suggestions into the archived state which happens through // subsequent fetches. Then we verify the entries are gone from the 'archived' // state by trying to load their images (and we shouldn't even know the URLs // anymore). - LoadFromJSONString(service.get(), - GetTestJson({GetSuggestionWithUrl("http://id-1"), - GetSuggestionWithUrl("http://id-2")})); - LoadFromJSONString(service.get(), - GetTestJson({GetSuggestionWithUrl("http://new-id-1"), - GetSuggestionWithUrl("http://new-id-2")})); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://id-1")) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://id-2")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://new-id-1")) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://new-id-2")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); // Make sure images of both batches are available. This is to sanity check our // assumptions for the test are right. ServeImageCallback cb = - base::Bind(&ServeOneByOneImage, &service->GetImageFetcherForTesting()); + base::Bind(&ServeOneByOneImage, &provider->GetImageFetcherForTesting()); EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _, _)) .Times(2) .WillRepeatedly(WithArgs<0, 2>(Invoke(CreateFunctor(cb)))); image_decoder()->SetDecodedImage(gfx::test::CreateImage(1, 1)); - gfx::Image image = FetchImage(service.get(), MakeArticleID("http://id-1")); + gfx::Image image = FetchImage(provider.get(), MakeArticleID("http://id-1")); ASSERT_FALSE(image.IsEmpty()); ASSERT_EQ(1, image.Width()); - image = FetchImage(service.get(), MakeArticleID("http://new-id-1")); + image = FetchImage(provider.get(), MakeArticleID("http://new-id-1")); ASSERT_FALSE(image.IsEmpty()); ASSERT_EQ(1, image.Width()); - service->ClearHistory(base::Time::UnixEpoch(), base::Time::Max(), - base::Callback<bool(const GURL& url)>()); + provider->ClearHistory(base::Time::UnixEpoch(), base::Time::Max(), + base::Callback<bool(const GURL& url)>()); // Make sure images of both batches are gone. // Verify we cannot resolve the image of the new suggestions. image_decoder()->SetDecodedImage(gfx::test::CreateImage(1, 1)); EXPECT_TRUE( - FetchImage(service.get(), MakeArticleID("http://id-1")).IsEmpty()); + FetchImage(provider.get(), MakeArticleID("http://id-1")).IsEmpty()); EXPECT_TRUE( - FetchImage(service.get(), MakeArticleID("http://new-id-1")).IsEmpty()); + FetchImage(provider.get(), MakeArticleID("http://new-id-1")).IsEmpty()); } -// TODO(tschumann): We don't have test making sure the RemoteSuggestionsFetcher -// actually gets the proper parameters. Add tests with an injected -// RemoteSuggestionsFetcher to verify the parameters, including proper handling -// of dismissed and known_ids. - namespace { // Workaround for gMock's lack of support for movable types. @@ -1255,340 +1466,378 @@ void SuggestionsLoaded( } // namespace TEST_F(RemoteSuggestionsProviderImplTest, ReturnFetchRequestEmptyBeforeInit) { - auto service = MakeSuggestionsProviderWithoutInitialization(); + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + RemoteSuggestionsFetcher::SnippetsAvailableCallback snippets_callback; + EXPECT_CALL(*mock_suggestions_fetcher(), FetchSnippets(_, _)).Times(0); MockFunction<void(Status, const std::vector<ContentSuggestion>&)> loaded; - EXPECT_CALL(loaded, Call(HasCode(StatusCode::TEMPORARY_ERROR), IsEmpty())); - service->Fetch(articles_category(), std::set<std::string>(), - base::Bind(&SuggestionsLoaded, &loaded)); + EXPECT_CALL(loaded, Call(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + IsEmpty())); + provider->Fetch(articles_category(), std::set<std::string>(), + base::Bind(&SuggestionsLoaded, &loaded)); base::RunLoop().RunUntilIdle(); } -TEST_F(RemoteSuggestionsProviderImplTest, ReturnTemporaryErrorForInvalidJson) { - auto service = MakeSuggestionsProvider(); - - MockFunction<void(Status, const std::vector<ContentSuggestion>&)> loaded; - EXPECT_CALL(loaded, Call(HasCode(StatusCode::TEMPORARY_ERROR), IsEmpty())); - LoadMoreFromJSONString(service.get(), articles_category(), - "invalid json string}]}", - /*known_ids=*/std::set<std::string>(), - base::Bind(&SuggestionsLoaded, &loaded)); - EXPECT_THAT(suggestions_fetcher()->last_status(), - StartsWith("Received invalid JSON")); -} - -TEST_F(RemoteSuggestionsProviderImplTest, - ReturnTemporaryErrorForInvalidSuggestion) { - auto service = MakeSuggestionsProvider(); - - MockFunction<void(Status, const std::vector<ContentSuggestion>&)> loaded; - EXPECT_CALL(loaded, Call(HasCode(StatusCode::TEMPORARY_ERROR), IsEmpty())); - LoadMoreFromJSONString(service.get(), articles_category(), - GetTestJson({GetIncompleteSuggestion()}), - /*known_ids=*/std::set<std::string>(), - base::Bind(&SuggestionsLoaded, &loaded)); - EXPECT_THAT(suggestions_fetcher()->last_status(), - StartsWith("Invalid / empty list")); -} - TEST_F(RemoteSuggestionsProviderImplTest, - ReturnTemporaryErrorForRequestFailure) { - // Created SuggestionsProvider will fail by default with unsuccessful request. - auto service = MakeSuggestionsProvider(/*set_empty_response=*/false); + ShouldForwardTemporaryErrorFromFetcher) { + auto provider = MakeSuggestionsProvider(); + RemoteSuggestionsFetcher::SnippetsAvailableCallback snippets_callback; MockFunction<void(Status, const std::vector<ContentSuggestion>&)> loaded; - EXPECT_CALL(loaded, Call(HasCode(StatusCode::TEMPORARY_ERROR), IsEmpty())); - service->Fetch(articles_category(), - /*known_ids=*/std::set<std::string>(), - base::Bind(&SuggestionsLoaded, &loaded)); - base::RunLoop().RunUntilIdle(); -} - -TEST_F(RemoteSuggestionsProviderImplTest, ReturnTemporaryErrorForHttpFailure) { - auto service = MakeSuggestionsProvider(); - SetUpHttpError(); - - MockFunction<void(Status, const std::vector<ContentSuggestion>&)> loaded; - EXPECT_CALL(loaded, Call(HasCode(StatusCode::TEMPORARY_ERROR), IsEmpty())); - service->Fetch(articles_category(), - /*known_ids=*/std::set<std::string>(), - base::Bind(&SuggestionsLoaded, &loaded)); - base::RunLoop().RunUntilIdle(); -} - -TEST_F(RemoteSuggestionsProviderImplTest, LoadInvalidJson) { - auto service = MakeSuggestionsProvider(); - - LoadFromJSONString(service.get(), GetTestJson({GetInvalidSuggestion()})); - EXPECT_THAT(suggestions_fetcher()->last_status(), - StartsWith("Received invalid JSON")); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), - IsEmpty()); + EXPECT_CALL(*mock_suggestions_fetcher(), FetchSnippets(_, _)) + .WillOnce(MoveSecondArgumentPointeeTo(&snippets_callback)); + EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) + .WillOnce(Return(true)) + .RetiresOnSaturation(); + provider->Fetch(articles_category(), + /*known_ids=*/std::set<std::string>(), + base::Bind(&SuggestionsLoaded, &loaded)); + + EXPECT_CALL(loaded, Call(Field(&Status::code, StatusCode::TEMPORARY_ERROR), + IsEmpty())); + ASSERT_FALSE(snippets_callback.is_null()); + std::move(snippets_callback) + .Run(Status(StatusCode::TEMPORARY_ERROR, "Received invalid JSON"), + base::nullopt); } TEST_F(RemoteSuggestionsProviderImplTest, - LoadInvalidJsonWithExistingSuggestions) { - auto service = MakeSuggestionsProvider(); - - LoadFromJSONString(service.get(), GetTestJson({GetSuggestion()})); - ASSERT_THAT(service->GetSuggestionsForTesting(articles_category()), - SizeIs(1)); - ASSERT_EQ("OK", suggestions_fetcher()->last_status()); - - LoadFromJSONString(service.get(), GetTestJson({GetInvalidSuggestion()})); - EXPECT_THAT(suggestions_fetcher()->last_status(), - StartsWith("Received invalid JSON")); - // This should not have changed the existing suggestions. - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), - SizeIs(1)); -} - -TEST_F(RemoteSuggestionsProviderImplTest, LoadIncompleteJson) { - auto service = MakeSuggestionsProvider(); - - LoadFromJSONString(service.get(), GetTestJson({GetIncompleteSuggestion()})); - EXPECT_EQ("Invalid / empty list.", suggestions_fetcher()->last_status()); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + ShouldNotAddNewSuggestionsAfterFetchError) { + auto provider = MakeSuggestionsProvider(); + + FetchTheseSuggestions( + provider.get(), /*interactive_request=*/false, + Status(StatusCode::TEMPORARY_ERROR, "Received invalid JSON"), + base::nullopt); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), IsEmpty()); } TEST_F(RemoteSuggestionsProviderImplTest, - LoadIncompleteJsonWithExistingSuggestions) { - auto service = MakeSuggestionsProvider(); + ShouldNotClearOldSuggestionsAfterFetchError) { + auto provider = MakeSuggestionsProvider(); - LoadFromJSONString(service.get(), GetTestJson({GetSuggestion()})); - ASSERT_THAT(service->GetSuggestionsForTesting(articles_category()), - SizeIs(1)); - - LoadFromJSONString(service.get(), GetTestJson({GetIncompleteSuggestion()})); - EXPECT_EQ("Invalid / empty list.", suggestions_fetcher()->last_status()); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back(FetchedCategory( + articles_category(), + BuildRemoteCategoryInfo(base::UTF8ToUTF16("title"), + /*allow_fetching_more_results=*/true))); + fetched_categories[0].suggestions.push_back( + CreateTestRemoteSuggestion(base::StringPrintf("http://abc.com/"))); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/false, + Status(StatusCode::SUCCESS, "success message"), + std::move(fetched_categories)); + + ASSERT_THAT( + provider->GetSuggestionsForTesting(articles_category()), + ElementsAre(Pointee(Property(&RemoteSuggestion::id, "http://abc.com/")))); + + FetchTheseSuggestions( + provider.get(), /*interactive_request=*/false, + Status(StatusCode::TEMPORARY_ERROR, "Received invalid JSON"), + base::nullopt); // This should not have changed the existing suggestions. - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), - SizeIs(1)); + EXPECT_THAT( + provider->GetSuggestionsForTesting(articles_category()), + ElementsAre(Pointee(Property(&RemoteSuggestion::id, "http://abc.com/")))); } TEST_F(RemoteSuggestionsProviderImplTest, Dismiss) { - auto service = MakeSuggestionsProvider(); - - std::string json_str(GetTestJson( - {GetSuggestionWithSources("http://site.com", "Source 1", "")})); - - LoadFromJSONString(service.get(), json_str); - - ASSERT_THAT(service->GetSuggestionsForTesting(articles_category()), + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + const FetchedCategoryBuilder category_builder = + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://site.com")); + fetched_categories.push_back(category_builder.Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + ASSERT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); // Load the image to store it in the database. ServeImageCallback cb = - base::Bind(&ServeOneByOneImage, &service->GetImageFetcherForTesting()); + base::Bind(&ServeOneByOneImage, &provider->GetImageFetcherForTesting()); EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _, _)) .WillOnce(WithArgs<0, 2>(Invoke(CreateFunctor(cb)))); image_decoder()->SetDecodedImage(gfx::test::CreateImage(1, 1)); - gfx::Image image = FetchImage(service.get(), MakeArticleID(kSuggestionUrl)); + gfx::Image image = + FetchImage(provider.get(), MakeArticleID("http://site.com")); EXPECT_FALSE(image.IsEmpty()); EXPECT_EQ(1, image.Width()); // Dismissing a non-existent suggestion shouldn't do anything. - service->DismissSuggestion(MakeArticleID("http://othersite.com")); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + provider->DismissSuggestion(MakeArticleID("http://othersite.com")); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); // Dismiss the suggestion. - service->DismissSuggestion(MakeArticleID(kSuggestionUrl)); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + provider->DismissSuggestion(MakeArticleID("http://site.com")); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), IsEmpty()); // Verify we can still load the image of the discarded suggestion (other NTPs // might still reference it). This should come from the database -- no network // fetch necessary. image_decoder()->SetDecodedImage(gfx::test::CreateImage(1, 1)); - image = FetchImage(service.get(), MakeArticleID(kSuggestionUrl)); + image = FetchImage(provider.get(), MakeArticleID("http://site.com")); EXPECT_FALSE(image.IsEmpty()); EXPECT_EQ(1, image.Width()); // Make sure that fetching the same suggestion again does not re-add it. - LoadFromJSONString(service.get(), json_str); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + fetched_categories.clear(); + fetched_categories.push_back(category_builder.Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), IsEmpty()); - // The suggestion should stay dismissed even after re-creating the service. - ResetSuggestionsProvider(&service, /*set_empty_response=*/true); - LoadFromJSONString(service.get(), json_str); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + // The suggestion should stay dismissed even after re-creating the provider. + ResetSuggestionsProvider(&provider); + fetched_categories.clear(); + fetched_categories.push_back(category_builder.Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), IsEmpty()); // The suggestion can be added again after clearing dismissed suggestions. - service->ClearDismissedSuggestionsForDebugging(articles_category()); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + provider->ClearDismissedSuggestionsForDebugging(articles_category()); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), IsEmpty()); - LoadFromJSONString(service.get(), json_str); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + fetched_categories.clear(); + fetched_categories.push_back(category_builder.Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); } TEST_F(RemoteSuggestionsProviderImplTest, GetDismissed) { - auto service = MakeSuggestionsProvider(); - - LoadFromJSONString(service.get(), GetTestJson({GetSuggestion()})); - - service->DismissSuggestion(MakeArticleID(kSuggestionUrl)); - - service->GetDismissedSuggestionsForDebugging( + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://site.com")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + provider->DismissSuggestion(MakeArticleID("http://site.com")); + + provider->GetDismissedSuggestionsForDebugging( articles_category(), base::Bind( - [](RemoteSuggestionsProviderImpl* service, + [](RemoteSuggestionsProviderImpl* provider, RemoteSuggestionsProviderImplTest* test, std::vector<ContentSuggestion> dismissed_suggestions) { EXPECT_EQ(1u, dismissed_suggestions.size()); for (auto& suggestion : dismissed_suggestions) { - EXPECT_EQ(test->MakeArticleID(kSuggestionUrl), suggestion.id()); + EXPECT_EQ(test->MakeArticleID("http://site.com"), + suggestion.id()); } }, - service.get(), this)); + provider.get(), this)); base::RunLoop().RunUntilIdle(); // There should be no dismissed suggestion after clearing the list. - service->ClearDismissedSuggestionsForDebugging(articles_category()); - service->GetDismissedSuggestionsForDebugging( + provider->ClearDismissedSuggestionsForDebugging(articles_category()); + provider->GetDismissedSuggestionsForDebugging( articles_category(), base::Bind( - [](RemoteSuggestionsProviderImpl* service, + [](RemoteSuggestionsProviderImpl* provider, RemoteSuggestionsProviderImplTest* test, std::vector<ContentSuggestion> dismissed_suggestions) { EXPECT_EQ(0u, dismissed_suggestions.size()); }, - service.get(), this)); + provider.get(), this)); base::RunLoop().RunUntilIdle(); } -TEST_F(RemoteSuggestionsProviderImplTest, CreationTimestampParseFail) { - auto service = MakeSuggestionsProvider(); - - std::string json = GetSuggestionWithTimes(GetDefaultCreationTime(), - GetDefaultExpirationTime()); - base::ReplaceFirstSubstringAfterOffset( - &json, 0, FormatTime(GetDefaultCreationTime()), "aaa1448459205"); - std::string json_str(GetTestJson({json})); - - LoadFromJSONString(service.get(), json_str); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), - IsEmpty()); -} - TEST_F(RemoteSuggestionsProviderImplTest, RemoveExpiredDismissedContent) { - auto service = MakeSuggestionsProvider(); - - std::string json_str1(GetTestJson({GetExpiredSuggestion()})); - // Load it. - LoadFromJSONString(service.get(), json_str1); + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://first/") + .SetExpiryDate(base::Time::Now())) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); // Load the image to store it in the database. // TODO(tschumann): Introduce some abstraction to nicely work with image // fetching expectations. ServeImageCallback cb = - base::Bind(&ServeOneByOneImage, &service->GetImageFetcherForTesting()); + base::Bind(&ServeOneByOneImage, &provider->GetImageFetcherForTesting()); EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _, _)) .WillOnce(WithArgs<0, 2>(Invoke(CreateFunctor(cb)))); image_decoder()->SetDecodedImage(gfx::test::CreateImage(1, 1)); - gfx::Image image = FetchImage(service.get(), MakeArticleID(kSuggestionUrl)); + gfx::Image image = FetchImage(provider.get(), MakeArticleID("http://first/")); EXPECT_FALSE(image.IsEmpty()); EXPECT_EQ(1, image.Width()); // Dismiss the suggestion - service->DismissSuggestion( - ContentSuggestion::ID(articles_category(), kSuggestionUrl)); + provider->DismissSuggestion( + ContentSuggestion::ID(articles_category(), "http://first/")); // Load a different suggestion - this will clear the expired dismissed ones. - std::string json_str2(GetTestJson({GetSuggestionWithUrl(kSuggestionUrl2)})); - LoadFromJSONString(service.get(), json_str2); - - EXPECT_THAT(service->GetDismissedSuggestionsForTesting(articles_category()), + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://second/")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + EXPECT_THAT(provider->GetDismissedSuggestionsForTesting(articles_category()), IsEmpty()); // Verify the image got removed, too. EXPECT_TRUE( - FetchImage(service.get(), MakeArticleID(kSuggestionUrl)).IsEmpty()); + FetchImage(provider.get(), MakeArticleID("http://first/")).IsEmpty()); } TEST_F(RemoteSuggestionsProviderImplTest, ExpiredContentNotRemoved) { - auto service = MakeSuggestionsProvider(); - - std::string json_str(GetTestJson({GetExpiredSuggestion()})); - - LoadFromJSONString(service.get(), json_str); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().SetExpiryDate(base::Time::Now())) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); } TEST_F(RemoteSuggestionsProviderImplTest, TestSingleSource) { - auto service = MakeSuggestionsProvider(); - - std::string json_str(GetTestJson({GetSuggestionWithSources( - "http://source1.com", "Source 1", "http://source1.amp.com")})); - - LoadFromJSONString(service.get(), json_str); - ASSERT_THAT(service->GetSuggestionsForTesting(articles_category()), + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://source1.com") + .SetUrl("http://source1.com") + .SetPublisher("Source 1") + .SetAmpUrl("http://source1.amp.com")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + ASSERT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); const RemoteSuggestion& suggestion = - *service->GetSuggestionsForTesting(articles_category()).front(); - EXPECT_EQ(suggestion.id(), kSuggestionUrl); + *provider->GetSuggestionsForTesting(articles_category()).front(); + EXPECT_EQ(suggestion.id(), "http://source1.com"); EXPECT_EQ(suggestion.url(), GURL("http://source1.com")); EXPECT_EQ(suggestion.publisher_name(), std::string("Source 1")); EXPECT_EQ(suggestion.amp_url(), GURL("http://source1.amp.com")); } -TEST_F(RemoteSuggestionsProviderImplTest, TestSingleSourceWithMalformedUrl) { - auto service = MakeSuggestionsProvider(); - - std::string json_str(GetTestJson({GetSuggestionWithSources( - "ceci n'est pas un url", "Source 1", "http://source1.amp.com")})); - - LoadFromJSONString(service.get(), json_str); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), - IsEmpty()); -} - TEST_F(RemoteSuggestionsProviderImplTest, TestSingleSourceWithMissingData) { - auto service = MakeSuggestionsProvider(); - - std::string json_str( - GetTestJson({GetSuggestionWithSources("http://source1.com", "", "")})); - - LoadFromJSONString(service.get(), json_str); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().SetPublisher("").SetAmpUrl("")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), IsEmpty()); } TEST_F(RemoteSuggestionsProviderImplTest, LogNumArticlesHistogram) { - auto service = MakeSuggestionsProvider(); + auto provider = MakeSuggestionsProvider(); base::HistogramTester tester; - LoadFromJSONString(service.get(), GetTestJson({GetInvalidSuggestion()})); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::TEMPORARY_ERROR, "message"), + base::nullopt); EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); - - // Invalid JSON shouldn't contribute to NumArticlesFetched. + // Fetch error shouldn't contribute to NumArticlesFetched. EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), IsEmpty()); - // Valid JSON with empty list. - LoadFromJSONString(service.get(), GetTestJson(std::vector<std::string>())); + // Emptry categories list. + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::vector<FetchedCategory>()); EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), ElementsAre(base::Bucket(/*min=*/0, /*count=*/2))); EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), + IsEmpty()); + + // Empty articles category. + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder().SetCategory(articles_category()).Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/3))); + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); // Suggestion list should be populated with size 1. - LoadFromJSONString(service.get(), GetTestJson({GetSuggestion()})); + const FetchedCategoryBuilder category_builder = + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://site.com/")); + fetched_categories.clear(); + fetched_categories.push_back(category_builder.Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/2), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/3), base::Bucket(/*min=*/1, /*count=*/1))); EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), ElementsAre(base::Bucket(/*min=*/0, /*count=*/1), base::Bucket(/*min=*/1, /*count=*/1))); // Duplicate suggestion shouldn't increase the list size. - LoadFromJSONString(service.get(), GetTestJson({GetSuggestion()})); + fetched_categories.clear(); + fetched_categories.push_back(category_builder.Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/2), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/3), base::Bucket(/*min=*/1, /*count=*/2))); EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), ElementsAre(base::Bucket(/*min=*/0, /*count=*/1), @@ -1599,10 +1848,14 @@ TEST_F(RemoteSuggestionsProviderImplTest, LogNumArticlesHistogram) { // Dismissing a suggestion should decrease the list size. This will only be // logged after the next fetch. - service->DismissSuggestion(MakeArticleID(kSuggestionUrl)); - LoadFromJSONString(service.get(), GetTestJson({GetSuggestion()})); + provider->DismissSuggestion(MakeArticleID("http://site.com/")); + fetched_categories.clear(); + fetched_categories.push_back(category_builder.Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/3), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/4), base::Bucket(/*min=*/1, /*count=*/2))); // Dismissed suggestions shouldn't influence NumArticlesFetched. EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), @@ -1614,10 +1867,8 @@ TEST_F(RemoteSuggestionsProviderImplTest, LogNumArticlesHistogram) { } TEST_F(RemoteSuggestionsProviderImplTest, DismissShouldRespectAllKnownUrls) { - auto service = MakeSuggestionsProvider(); + auto provider = MakeSuggestionsProvider(); - const base::Time creation = GetDefaultCreationTime(); - const base::Time expiry = GetDefaultExpirationTime(); const std::vector<std::string> source_urls = { "http://mashable.com/2016/05/11/stolen", "http://www.aol.com/article/2016/05/stolen-doggie"}; @@ -1627,35 +1878,64 @@ TEST_F(RemoteSuggestionsProviderImplTest, DismissShouldRespectAllKnownUrls) { "http://t2.gstatic.com/images?q=tbn:3"}; // Add the suggestion from the mashable domain. - LoadFromJSONString(service.get(), - GetTestJson({GetSuggestionWithUrlAndTimesAndSource( - source_urls, source_urls[0], creation, expiry, - publishers[0], amp_urls[0])})); - ASSERT_THAT(service->GetSuggestionsForTesting(articles_category()), + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId(source_urls[0]) + .AddId(source_urls[1]) + .SetUrl(source_urls[0]) + .SetAmpUrl(amp_urls[0]) + .SetPublisher(publishers[0])) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + ASSERT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(1)); // Dismiss the suggestion via the mashable source corpus ID. - service->DismissSuggestion(MakeArticleID(source_urls[0])); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + provider->DismissSuggestion(MakeArticleID(source_urls[0])); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), IsEmpty()); // The same article from the AOL domain should now be detected as dismissed. - LoadFromJSONString(service.get(), - GetTestJson({GetSuggestionWithUrlAndTimesAndSource( - source_urls, source_urls[1], creation, expiry, - publishers[1], amp_urls[1])})); - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId(source_urls[0]) + .AddId(source_urls[1]) + .SetUrl(source_urls[1]) + .SetAmpUrl(amp_urls[1]) + .SetPublisher(publishers[1])) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), IsEmpty()); } TEST_F(RemoteSuggestionsProviderImplTest, ImageReturnedWithTheSameId) { - auto service = MakeSuggestionsProvider(); - - LoadFromJSONString(service.get(), GetTestJson({GetSuggestion()})); + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId(kSuggestionUrl)) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); gfx::Image image; MockFunction<void(const gfx::Image&)> image_fetched; ServeImageCallback cb = - base::Bind(&ServeOneByOneImage, &service->GetImageFetcherForTesting()); + base::Bind(&ServeOneByOneImage, &provider->GetImageFetcherForTesting()); { InSequence s; EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _, _)) @@ -1663,7 +1943,7 @@ TEST_F(RemoteSuggestionsProviderImplTest, ImageReturnedWithTheSameId) { EXPECT_CALL(image_fetched, Call(_)).WillOnce(SaveArg<0>(&image)); } - service->FetchSuggestionImage( + provider->FetchSuggestionImage( MakeArticleID(kSuggestionUrl), base::Bind(&MockFunction<void(const gfx::Image&)>::Call, base::Unretained(&image_fetched))); @@ -1673,15 +1953,15 @@ TEST_F(RemoteSuggestionsProviderImplTest, ImageReturnedWithTheSameId) { } TEST_F(RemoteSuggestionsProviderImplTest, EmptyImageReturnedForNonExistentId) { - auto service = MakeSuggestionsProvider(); + auto provider = MakeSuggestionsProvider(); // Create a non-empty image so that we can test the image gets updated. gfx::Image image = gfx::test::CreateImage(1, 1); MockFunction<void(const gfx::Image&)> image_fetched; EXPECT_CALL(image_fetched, Call(_)).WillOnce(SaveArg<0>(&image)); - service->FetchSuggestionImage( - MakeArticleID(kSuggestionUrl2), + provider->FetchSuggestionImage( + MakeArticleID("nonexistent"), base::Bind(&MockFunction<void(const gfx::Image&)>::Call, base::Unretained(&image_fetched))); @@ -1694,7 +1974,7 @@ TEST_F(RemoteSuggestionsProviderImplTest, // Testing that the provider is not accessing the database is tricky. // Therefore, we simply put in some data making sure that if the provider asks // the database, it will get a wrong answer. - auto service = MakeSuggestionsProvider(); + auto provider = MakeSuggestionsProvider(); ContentSuggestion::ID unknown_id = MakeArticleID(kSuggestionUrl2); database()->SaveImage(unknown_id.id_within_category(), "some image blob"); @@ -1706,7 +1986,7 @@ TEST_F(RemoteSuggestionsProviderImplTest, MockFunction<void(const gfx::Image&)> image_fetched; EXPECT_CALL(image_fetched, Call(_)).WillOnce(SaveArg<0>(&image)); - service->FetchSuggestionImage( + provider->FetchSuggestionImage( MakeArticleID(kSuggestionUrl2), base::Bind(&MockFunction<void(const gfx::Image&)>::Call, base::Unretained(&image_fetched))); @@ -1716,30 +1996,38 @@ TEST_F(RemoteSuggestionsProviderImplTest, } TEST_F(RemoteSuggestionsProviderImplTest, ClearHistoryRemovesAllSuggestions) { - auto service = MakeSuggestionsProvider(); - - std::string first_suggestion = GetSuggestionWithUrl("http://url1.com"); - std::string second_suggestion = GetSuggestionWithUrl("http://url2.com"); - std::string json_str = GetTestJson({first_suggestion, second_suggestion}); - LoadFromJSONString(service.get(), json_str); - ASSERT_THAT(service->GetSuggestionsForTesting(articles_category()), + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://first/")) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://second/")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + ASSERT_THAT(provider->GetSuggestionsForTesting(articles_category()), SizeIs(2)); - service->DismissSuggestion(MakeArticleID("http://url1.com")); + provider->DismissSuggestion(MakeArticleID("http://first/")); ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), Not(IsEmpty())); - ASSERT_THAT(service->GetDismissedSuggestionsForTesting(articles_category()), + ASSERT_THAT(provider->GetDismissedSuggestionsForTesting(articles_category()), SizeIs(1)); base::Time begin = base::Time::FromTimeT(123), end = base::Time::FromTimeT(456); base::Callback<bool(const GURL& url)> filter; - service->ClearHistory(begin, end, filter); + provider->ClearHistory(begin, end, filter); // Verify that the observer received the update with the empty data as well. EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), IsEmpty()); - EXPECT_THAT(service->GetDismissedSuggestionsForTesting(articles_category()), + EXPECT_THAT(provider->GetDismissedSuggestionsForTesting(articles_category()), IsEmpty()); } @@ -1747,90 +2035,117 @@ TEST_F(RemoteSuggestionsProviderImplTest, ShouldKeepArticlesCategoryAvailableAfterClearHistory) { // If the provider marks that category as NOT_PROVIDED, then it won't be shown // at all in the UI and the user cannot load new data :-/. - auto service = MakeSuggestionsProvider(); + auto provider = MakeSuggestionsProvider(); ASSERT_THAT(observer().StatusForCategory(articles_category()), Eq(CategoryStatus::AVAILABLE)); - service->ClearHistory(base::Time::UnixEpoch(), base::Time::Max(), - base::Callback<bool(const GURL& url)>()); + provider->ClearHistory(base::Time::UnixEpoch(), base::Time::Max(), + base::Callback<bool(const GURL& url)>()); EXPECT_THAT(observer().StatusForCategory(articles_category()), Eq(CategoryStatus::AVAILABLE)); } TEST_F(RemoteSuggestionsProviderImplTest, ShouldClearOrphanedImagesOnRestart) { - auto service = MakeSuggestionsProvider(); - - LoadFromJSONString(service.get(), GetTestJson({GetSuggestion()})); + auto provider = MakeSuggestionsProvider(); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId(kSuggestionUrl)) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); ServeImageCallback cb = - base::Bind(&ServeOneByOneImage, &service->GetImageFetcherForTesting()); + base::Bind(&ServeOneByOneImage, &provider->GetImageFetcherForTesting()); EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _, _)) .WillOnce(WithArgs<0, 2>(Invoke(CreateFunctor(cb)))); image_decoder()->SetDecodedImage(gfx::test::CreateImage(1, 1)); - gfx::Image image = FetchImage(service.get(), MakeArticleID(kSuggestionUrl)); + gfx::Image image = FetchImage(provider.get(), MakeArticleID(kSuggestionUrl)); EXPECT_EQ(1, image.Width()); EXPECT_FALSE(image.IsEmpty()); // Send new suggestion which don't include the suggestion referencing the // image. - LoadFromJSONString(service.get(), - GetTestJson({GetSuggestionWithUrl( - "http://something.com/pletely/unrelated")})); + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId( + "http://something.com/pletely/unrelated")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); // The image should still be available until a restart happens. EXPECT_FALSE( - FetchImage(service.get(), MakeArticleID(kSuggestionUrl)).IsEmpty()); - ResetSuggestionsProvider(&service, /*set_empty_response=*/true); + FetchImage(provider.get(), MakeArticleID(kSuggestionUrl)).IsEmpty()); + ResetSuggestionsProvider(&provider); // After the restart, the image should be garbage collected. EXPECT_TRUE( - FetchImage(service.get(), MakeArticleID(kSuggestionUrl)).IsEmpty()); + FetchImage(provider.get(), MakeArticleID(kSuggestionUrl)).IsEmpty()); } TEST_F(RemoteSuggestionsProviderImplTest, ShouldHandleMoreThanMaxSuggestionsInResponse) { - auto service = MakeSuggestionsProvider(); + auto provider = MakeSuggestionsProvider(); - std::vector<std::string> suggestions; - for (int i = 0; i < service->GetMaxSuggestionCountForTesting() + 1; ++i) { - suggestions.push_back(GetSuggestionWithUrl( + std::vector<FetchedCategory> fetched_categories; + FetchedCategoryBuilder category_builder; + category_builder.SetCategory(articles_category()); + for (int i = 0; i < provider->GetMaxSuggestionCountForTesting() + 1; ++i) { + category_builder.AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId( base::StringPrintf("http://localhost/suggestion-id-%d", i))); } - LoadFromJSONString(service.get(), GetTestJson(suggestions)); + fetched_categories.push_back(category_builder.Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); // TODO(tschumann): We should probably trim out any additional results and // only serve the MaxSuggestionCount items. - EXPECT_THAT(service->GetSuggestionsForTesting(articles_category()), - SizeIs(service->GetMaxSuggestionCountForTesting() + 1)); + EXPECT_THAT(provider->GetSuggestionsForTesting(articles_category()), + SizeIs(provider->GetMaxSuggestionCountForTesting() + 1)); } TEST_F(RemoteSuggestionsProviderImplTest, StoreLastSuccessfullBackgroundFetchTime) { // On initialization of the RemoteSuggestionsProviderImpl a background fetch - // is triggered since the suggestions DB is empty. Therefore the service must + // is triggered since the suggestions DB is empty. Therefore the provider must // not be initialized until the test clock is set. - auto service = MakeSuggestionsProviderWithoutInitialization(); + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); auto simple_test_clock = base::MakeUnique<base::SimpleTestClock>(); base::SimpleTestClock* simple_test_clock_ptr = simple_test_clock.get(); - service->SetClockForTesting(std::move(simple_test_clock)); + provider->SetClockForTesting(std::move(simple_test_clock)); // Test that the preference is correctly initialized with the default value 0. EXPECT_EQ( 0, pref_service()->GetInt64(prefs::kLastSuccessfulBackgroundFetchTime)); - WaitForSuggestionsProviderInitialization(service.get(), - /*set_empty_response=*/true); + WaitForSuggestionsProviderInitialization(provider.get()); EXPECT_EQ( simple_test_clock_ptr->Now().ToInternalValue(), pref_service()->GetInt64(prefs::kLastSuccessfulBackgroundFetchTime)); // Advance the time and check whether the time was updated correctly after the // background fetch. - simple_test_clock_ptr->Advance(TimeDelta::FromHours(1)); + simple_test_clock_ptr->Advance(base::TimeDelta::FromHours(1)); - service->RefetchInTheBackground( + RemoteSuggestionsFetcher::SnippetsAvailableCallback snippets_callback; + EXPECT_CALL(*mock_suggestions_fetcher(), FetchSnippets(_, _)) + .WillOnce(MoveSecondArgumentPointeeTo(&snippets_callback)) + .RetiresOnSaturation(); + provider->RefetchInTheBackground( RemoteSuggestionsProvider::FetchStatusCallback()); base::RunLoop().RunUntilIdle(); + std::move(snippets_callback) + .Run(Status(StatusCode::SUCCESS, "message"), base::nullopt); // TODO(jkrcal): Move together with the pref storage into the scheduler. EXPECT_EQ( simple_test_clock_ptr->Now().ToInternalValue(), @@ -1840,26 +2155,25 @@ TEST_F(RemoteSuggestionsProviderImplTest, } TEST_F(RemoteSuggestionsProviderImplTest, CallsSchedulerWhenReady) { - auto service = + auto provider = MakeSuggestionsProviderWithoutInitializationWithStrictScheduler(); // Should be called when becoming ready. EXPECT_CALL(*scheduler(), OnProviderActivated()); - WaitForSuggestionsProviderInitialization(service.get(), - /*set_empty_response=*/true); + WaitForSuggestionsProviderInitialization(provider.get()); } TEST_F(RemoteSuggestionsProviderImplTest, CallsSchedulerOnError) { - auto service = + auto provider = MakeSuggestionsProviderWithoutInitializationWithStrictScheduler(); // Should be called on error. EXPECT_CALL(*scheduler(), OnProviderDeactivated()); - service->EnterState(RemoteSuggestionsProviderImpl::State::ERROR_OCCURRED); + provider->EnterState(RemoteSuggestionsProviderImpl::State::ERROR_OCCURRED); } TEST_F(RemoteSuggestionsProviderImplTest, CallsSchedulerWhenDisabled) { - auto service = + auto provider = MakeSuggestionsProviderWithoutInitializationWithStrictScheduler(); // Should be called when becoming disabled. First deactivate and only after @@ -1867,52 +2181,736 @@ TEST_F(RemoteSuggestionsProviderImplTest, CallsSchedulerWhenDisabled) { { InSequence s; EXPECT_CALL(*scheduler(), OnProviderDeactivated()); - ASSERT_THAT(service->ready(), Eq(false)); + ASSERT_THAT(provider->ready(), Eq(false)); EXPECT_CALL(*scheduler(), OnSuggestionsCleared()); } - service->EnterState(RemoteSuggestionsProviderImpl::State::DISABLED); + provider->EnterState(RemoteSuggestionsProviderImpl::State::DISABLED); } TEST_F(RemoteSuggestionsProviderImplTest, CallsSchedulerWhenHistoryCleared) { - auto service = + auto provider = MakeSuggestionsProviderWithoutInitializationWithStrictScheduler(); - // Initiate the service so that it is already READY. + // Initiate the provider so that it is already READY. EXPECT_CALL(*scheduler(), OnProviderActivated()); - WaitForSuggestionsProviderInitialization(service.get(), - /*set_empty_response=*/true); + WaitForSuggestionsProviderInitialization(provider.get()); // The scheduler should be notified of clearing the history. EXPECT_CALL(*scheduler(), OnHistoryCleared()); - service->ClearHistory(GetDefaultCreationTime(), GetDefaultExpirationTime(), - base::Callback<bool(const GURL& url)>()); + provider->ClearHistory(GetDefaultCreationTime(), GetDefaultExpirationTime(), + base::Callback<bool(const GURL& url)>()); } TEST_F(RemoteSuggestionsProviderImplTest, CallsSchedulerWhenSignedIn) { - auto service = + auto provider = MakeSuggestionsProviderWithoutInitializationWithStrictScheduler(); - // Initiate the service so that it is already READY. + // Initiate the provider so that it is already READY. EXPECT_CALL(*scheduler(), OnProviderActivated()); - WaitForSuggestionsProviderInitialization(service.get(), - /*set_empty_response=*/true); + WaitForSuggestionsProviderInitialization(provider.get()); // The scheduler should be notified of clearing the history. EXPECT_CALL(*scheduler(), OnSuggestionsCleared()); - service->OnStatusChanged(RemoteSuggestionsStatus::ENABLED_AND_SIGNED_IN, - RemoteSuggestionsStatus::ENABLED_AND_SIGNED_OUT); + provider->OnStatusChanged(RemoteSuggestionsStatus::ENABLED_AND_SIGNED_IN, + RemoteSuggestionsStatus::ENABLED_AND_SIGNED_OUT); } TEST_F(RemoteSuggestionsProviderImplTest, CallsSchedulerWhenSignedOut) { - auto service = + auto provider = MakeSuggestionsProviderWithoutInitializationWithStrictScheduler(); - // Initiate the service so that it is already READY. + // Initiate the provider so that it is already READY. EXPECT_CALL(*scheduler(), OnProviderActivated()); - WaitForSuggestionsProviderInitialization(service.get(), - /*set_empty_response=*/true); + WaitForSuggestionsProviderInitialization(provider.get()); // The scheduler should be notified of clearing the history. EXPECT_CALL(*scheduler(), OnSuggestionsCleared()); - service->OnStatusChanged(RemoteSuggestionsStatus::ENABLED_AND_SIGNED_OUT, - RemoteSuggestionsStatus::ENABLED_AND_SIGNED_IN); + provider->OnStatusChanged(RemoteSuggestionsStatus::ENABLED_AND_SIGNED_OUT, + RemoteSuggestionsStatus::ENABLED_AND_SIGNED_IN); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldExcludeKnownSuggestionsWithoutTruncatingWhenFetchingMore) { + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + WaitForSuggestionsProviderInitialization(provider.get()); + + std::set<std::string> known_ids; + for (int i = 0; i < 200; ++i) { + known_ids.insert(base::IntToString(i)); + } + + EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) + .WillOnce(Return(true)) + .RetiresOnSaturation(); + EXPECT_CALL(*mock_suggestions_fetcher(), + FetchSnippets(Field(&RequestParams::excluded_ids, known_ids), _)); + provider->Fetch( + articles_category(), known_ids, + base::Bind([](Status status_code, + std::vector<ContentSuggestion> suggestions) {})); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldExcludeDismissedSuggestionsWhenFetchingMore) { + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + WaitForSuggestionsProviderInitialization(provider.get()); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId("http://abc.com/")) + .Build()); + ASSERT_TRUE(fetched_categories[0].suggestions[0]->is_complete()); + + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + provider->DismissSuggestion(MakeArticleID("http://abc.com/")); + + std::set<std::string> expected_excluded_ids({"http://abc.com/"}); + EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) + .WillOnce(Return(true)) + .RetiresOnSaturation(); + EXPECT_CALL( + *mock_suggestions_fetcher(), + FetchSnippets(Field(&RequestParams::excluded_ids, expected_excluded_ids), + _)); + provider->Fetch( + articles_category(), std::set<std::string>(), + base::Bind([](Status status_code, + std::vector<ContentSuggestion> suggestions) {})); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldTruncateExcludedDismissedSuggestionsWhenFetchingMore) { + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + WaitForSuggestionsProviderInitialization(provider.get()); + + std::vector<FetchedCategory> fetched_categories; + FetchedCategoryBuilder category_builder; + category_builder.SetCategory(articles_category()); + const int kSuggestionsCount = kMaxExcludedDismissedIds + 1; + for (int i = 0; i < kSuggestionsCount; ++i) { + category_builder.AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId( + base::StringPrintf("http://abc.com/%d/", i))); + } + fetched_categories.push_back(category_builder.Build()); + + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + // Dismiss them. + for (int i = 0; i < kSuggestionsCount; ++i) { + provider->DismissSuggestion( + MakeArticleID(base::StringPrintf("http://abc.com/%d/", i))); + } + + EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) + .WillOnce(Return(true)) + .RetiresOnSaturation(); + EXPECT_CALL(*mock_suggestions_fetcher(), + FetchSnippets(Field(&RequestParams::excluded_ids, + SizeIs(kMaxExcludedDismissedIds)), + _)); + provider->Fetch( + articles_category(), std::set<std::string>(), + base::Bind([](Status status_code, + std::vector<ContentSuggestion> suggestions) {})); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldPreferLatestExcludedDismissedSuggestionsWhenFetchingMore) { + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + WaitForSuggestionsProviderInitialization(provider.get()); + + std::vector<FetchedCategory> fetched_categories; + FetchedCategoryBuilder category_builder; + category_builder.SetCategory(articles_category()); + const int kSuggestionsCount = kMaxExcludedDismissedIds + 1; + for (int i = 0; i < kSuggestionsCount; ++i) { + category_builder.AddSuggestionViaBuilder(RemoteSuggestionBuilder().AddId( + base::StringPrintf("http://abc.com/%d/", i))); + } + fetched_categories.push_back(category_builder.Build()); + + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + // Dismiss them in reverse order. + std::string first_dismissed_suggestion_id; + for (int i = kSuggestionsCount - 1; i >= 0; --i) { + const std::string id = base::StringPrintf("http://abc.com/%d/", i); + provider->DismissSuggestion(MakeArticleID(id)); + if (first_dismissed_suggestion_id.empty()) { + first_dismissed_suggestion_id = id; + } + } + + EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) + .WillOnce(Return(true)) + .RetiresOnSaturation(); + // The oldest dismissed suggestion should be absent, because there are + // |kMaxExcludedDismissedIds| newer dismissed suggestions. + EXPECT_CALL(*mock_suggestions_fetcher(), + FetchSnippets(Field(&RequestParams::excluded_ids, + Not(Contains(first_dismissed_suggestion_id))), + _)); + provider->Fetch( + articles_category(), std::set<std::string>(), + base::Bind([](Status status_code, + std::vector<ContentSuggestion> suggestions) {})); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldExcludeDismissedSuggestionsFromAllCategoriesWhenFetchingMore) { + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + WaitForSuggestionsProviderInitialization(provider.get()); + + // Add article suggestions. + std::vector<FetchedCategory> fetched_categories; + FetchedCategoryBuilder first_category_builder; + first_category_builder.SetCategory(articles_category()); + const int kSuggestionsPerCategory = 2; + for (int i = 0; i < kSuggestionsPerCategory; ++i) { + first_category_builder.AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId( + base::StringPrintf("http://abc.com/%d/", i))); + } + fetched_categories.push_back(first_category_builder.Build()); + // Add other category suggestions. + FetchedCategoryBuilder second_category_builder; + second_category_builder.SetCategory(other_category()); + for (int i = 0; i < kSuggestionsPerCategory; ++i) { + second_category_builder.AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId( + base::StringPrintf("http://other.com/%d/", i))); + } + fetched_categories.push_back(second_category_builder.Build()); + + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + // Dismiss all suggestions. + std::set<std::string> expected_excluded_ids; + for (int i = 0; i < kSuggestionsPerCategory; ++i) { + const std::string article_id = base::StringPrintf("http://abc.com/%d/", i); + provider->DismissSuggestion(MakeArticleID(article_id)); + expected_excluded_ids.insert(article_id); + const std::string other_id = base::StringPrintf("http://other.com/%d/", i); + provider->DismissSuggestion(MakeOtherID(other_id)); + expected_excluded_ids.insert(other_id); + } + + EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) + .WillOnce(Return(true)) + .RetiresOnSaturation(); + // Dismissed suggestions from all categories must be excluded (but not only + // target category). + EXPECT_CALL( + *mock_suggestions_fetcher(), + FetchSnippets(Field(&RequestParams::excluded_ids, expected_excluded_ids), + _)); + provider->Fetch( + articles_category(), std::set<std::string>(), + base::Bind([](Status status_code, + std::vector<ContentSuggestion> suggestions) {})); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldPreferTargetCategoryExcludedDismissedSuggestionsWhenFetchingMore) { + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/false); + WaitForSuggestionsProviderInitialization(provider.get()); + + // Add article suggestions. + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back(FetchedCategory( + articles_category(), + BuildRemoteCategoryInfo(base::UTF8ToUTF16("title"), + /*allow_fetching_more_results=*/true))); + + for (int i = 0; i < kMaxExcludedDismissedIds; ++i) { + fetched_categories[0].suggestions.push_back(CreateTestRemoteSuggestion( + base::StringPrintf("http://abc.com/%d/", i))); + } + // Add other category suggestion. + fetched_categories.push_back(FetchedCategory( + other_category(), + BuildRemoteCategoryInfo(base::UTF8ToUTF16("title"), + /*allow_fetching_more_results=*/true))); + fetched_categories[1].suggestions.push_back( + CreateTestRemoteSuggestion("http://other.com/")); + + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + // Dismiss article suggestions first. + for (int i = 0; i < kMaxExcludedDismissedIds; ++i) { + provider->DismissSuggestion( + MakeArticleID(base::StringPrintf("http://abc.com/%d/", i))); + } + + // Then dismiss other category suggestion. + provider->DismissSuggestion(MakeOtherID("http://other.com/")); + + EXPECT_CALL(*scheduler(), AcquireQuotaForInteractiveFetch()) + .WillOnce(Return(true)) + .RetiresOnSaturation(); + // The other category dismissed suggestion should be absent, because the fetch + // is for articles and there are |kMaxExcludedDismissedIds| dismissed + // suggestions there. + EXPECT_CALL(*mock_suggestions_fetcher(), + FetchSnippets(Field(&RequestParams::excluded_ids, + Not(Contains("http://other.com/"))), + _)); + provider->Fetch( + articles_category(), std::set<std::string>(), + base::Bind([](Status status_code, + std::vector<ContentSuggestion> suggestions) {})); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldFetchNormallyWithoutPrefetchedPagesTracker) { + EnableKeepingPrefetchedContentSuggestions( + kMaxAdditionalPrefetchedSuggestions, + kMaxAgeForAdditionalPrefetchedSuggestion); + + auto provider = MakeSuggestionsProvider(); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder()) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldKeepPrefetchedSuggestionsAfterFetchWhenEnabled) { + EnableKeepingPrefetchedContentSuggestions( + kMaxAdditionalPrefetchedSuggestions, + kMaxAgeForAdditionalPrefetchedSuggestion); + + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/true); + auto* mock_tracker = static_cast<StrictMock<MockPrefetchedPagesTracker>*>( + prefetched_pages_tracker()); + WaitForSuggestionsProviderInitialization(provider.get()); + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://prefetched.com") + .SetUrl("http://prefetched.com") + .SetAmpUrl("http://amp.prefetched.com")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); + + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_tracker, + PrefetchedOfflinePageExists(GURL("http://amp.prefetched.com"))) + .WillOnce(Return(true)); + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://other.com") + .SetUrl("http://other.com") + .SetAmpUrl("http://amp.other.com")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + EXPECT_THAT( + observer().SuggestionsForCategory(articles_category()), + UnorderedElementsAre( + Property(&ContentSuggestion::id, + MakeArticleID("http://prefetched.com")), + Property(&ContentSuggestion::id, MakeArticleID("http://other.com")))); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldIgnoreNotPrefetchedSuggestionsAfterFetchWhenEnabled) { + EnableKeepingPrefetchedContentSuggestions( + kMaxAdditionalPrefetchedSuggestions, + kMaxAgeForAdditionalPrefetchedSuggestion); + + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/true); + auto* mock_tracker = static_cast<StrictMock<MockPrefetchedPagesTracker>*>( + prefetched_pages_tracker()); + WaitForSuggestionsProviderInitialization(provider.get()); + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder() + .AddId("http://not_prefetched.com") + .SetUrl("http://not_prefetched.com") + .SetAmpUrl("http://amp.not_prefetched.com")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); + + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_tracker, PrefetchedOfflinePageExists( + GURL("http://amp.not_prefetched.com"))) + .WillOnce(Return(false)); + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://other.com") + .SetUrl("http://other.com") + .SetAmpUrl("http://amp.other.com")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), + UnorderedElementsAre(Property( + &ContentSuggestion::id, MakeArticleID("http://other.com")))); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldLimitKeptPrefetchedSuggestionsAfterFetchWhenEnabled) { + EnableKeepingPrefetchedContentSuggestions( + kMaxAdditionalPrefetchedSuggestions, + kMaxAgeForAdditionalPrefetchedSuggestion); + + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/true); + auto* mock_tracker = static_cast<StrictMock<MockPrefetchedPagesTracker>*>( + prefetched_pages_tracker()); + WaitForSuggestionsProviderInitialization(provider.get()); + + const int prefetched_suggestions_count = + 2 * kMaxAdditionalPrefetchedSuggestions + 1; + std::vector<FetchedCategory> fetched_categories; + FetchedCategoryBuilder category_builder; + category_builder.SetCategory(articles_category()); + for (int i = 0; i < prefetched_suggestions_count; ++i) { + const std::string url = base::StringPrintf("http://prefetched.com/%d", i); + category_builder.AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId(url).SetUrl(url).SetAmpUrl( + base::StringPrintf("http://amp.prefetched.com/%d", i))); + } + fetched_categories.push_back(category_builder.Build()); + + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(prefetched_suggestions_count)); + + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + for (int i = 0; i < prefetched_suggestions_count; ++i) { + EXPECT_CALL(*mock_tracker, + PrefetchedOfflinePageExists(GURL( + base::StringPrintf("http://amp.prefetched.com/%d", i)))) + .WillOnce(Return(true)); + } + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder() + .AddId("http://not_prefetched.com") + .SetUrl("http://not_prefetched.com") + .SetAmpUrl("http://amp.not_prefetched.com")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(kMaxAdditionalPrefetchedSuggestions + 1)); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldMixInPrefetchedSuggestionsByScoreAfterFetchWhenEnabled) { + EnableKeepingPrefetchedContentSuggestions( + kMaxAdditionalPrefetchedSuggestions, + kMaxAgeForAdditionalPrefetchedSuggestion); + + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/true); + auto* mock_tracker = static_cast<StrictMock<MockPrefetchedPagesTracker>*>( + prefetched_pages_tracker()); + WaitForSuggestionsProviderInitialization(provider.get()); + + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://prefetched.com/1") + .SetUrl("http://prefetched.com/1") + .SetAmpUrl("http://amp.prefetched.com/1") + .SetScore(1)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://prefetched.com/3") + .SetUrl("http://prefetched.com/3") + .SetAmpUrl("http://amp.prefetched.com/3") + .SetScore(3)) + .Build()); + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(2)); + + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://new.com/2") + .SetUrl("http://new.com/2") + .SetAmpUrl("http://amp.new.com/2") + .SetScore(2)) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://new.com/4") + .SetUrl("http://new.com/4") + .SetAmpUrl("http://amp.new.com/4") + .SetScore(4)) + .Build()); + + EXPECT_CALL(*mock_tracker, + PrefetchedOfflinePageExists(GURL("http://amp.prefetched.com/1"))) + .WillOnce(Return(true)); + EXPECT_CALL(*mock_tracker, + PrefetchedOfflinePageExists(GURL("http://amp.prefetched.com/3"))) + .WillOnce(Return(true)); + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + EXPECT_THAT( + observer().SuggestionsForCategory(articles_category()), + ElementsAre( + Property(&ContentSuggestion::id, MakeArticleID("http://new.com/4")), + Property(&ContentSuggestion::id, + MakeArticleID("http://prefetched.com/3")), + Property(&ContentSuggestion::id, MakeArticleID("http://new.com/2")), + Property(&ContentSuggestion::id, + MakeArticleID("http://prefetched.com/1")))); +} + +TEST_F( + RemoteSuggestionsProviderImplTest, + ShouldKeepMostRecentlyFetchedPrefetchedSuggestionsFirstAfterFetchWhenEnabled) { + EnableKeepingPrefetchedContentSuggestions( + kMaxAdditionalPrefetchedSuggestions, + kMaxAgeForAdditionalPrefetchedSuggestion); + + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/true); + auto* mock_tracker = static_cast<StrictMock<MockPrefetchedPagesTracker>*>( + prefetched_pages_tracker()); + WaitForSuggestionsProviderInitialization(provider.get()); + + std::vector<FetchedCategory> fetched_categories; + const int prefetched_suggestions_count = + 2 * kMaxAdditionalPrefetchedSuggestions + 1; + for (int i = 0; i < prefetched_suggestions_count; ++i) { + const std::string url = base::StringPrintf("http://prefetched.com/%d", i); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder().AddId(url).SetUrl(url).SetAmpUrl( + base::StringPrintf("http://amp.prefetched.com/%d", i))) + .Build()); + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + if (i != 0) { + EXPECT_CALL(*mock_tracker, + PrefetchedOfflinePageExists(GURL(base::StringPrintf( + "http://amp.prefetched.com/%d", i - 1)))) + .WillRepeatedly(Return(true)); + } + + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + } + + const std::vector<ContentSuggestion>& actual_suggestions = + observer().SuggestionsForCategory(articles_category()); + + ASSERT_THAT(actual_suggestions, + SizeIs(kMaxAdditionalPrefetchedSuggestions + 1)); + + int matched = 0; + for (int i = prefetched_suggestions_count - 1; i >= 0; --i) { + EXPECT_THAT(actual_suggestions, + Contains(Property(&ContentSuggestion::id, + MakeArticleID(base::StringPrintf( + "http://prefetched.com/%d", i))))); + ++matched; + if (matched == kMaxAdditionalPrefetchedSuggestions + 1) { + break; + } + } +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldNotKeepStalePrefetchedSuggestionsAfterFetchWhenEnabled) { + EnableKeepingPrefetchedContentSuggestions( + kMaxAdditionalPrefetchedSuggestions, + kMaxAgeForAdditionalPrefetchedSuggestion); + + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/true); + auto* mock_tracker = static_cast<StrictMock<MockPrefetchedPagesTracker>*>( + prefetched_pages_tracker()); + + auto wrapped_provider_clock = base::MakeUnique<base::SimpleTestClock>(); + base::SimpleTestClock* provider_clock = wrapped_provider_clock.get(); + provider->SetClockForTesting(std::move(wrapped_provider_clock)); + + provider_clock->SetNow(GetDefaultCreationTime() + + base::TimeDelta::FromHours(10)); + + WaitForSuggestionsProviderInitialization(provider.get()); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder() + .AddId("http://prefetched.com") + .SetUrl("http://prefetched.com") + .SetAmpUrl("http://amp.prefetched.com") + .SetFetchDate(provider_clock->Now()) + .SetPublishDate(GetDefaultCreationTime())) + .Build()); + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); + + provider_clock->Advance(kMaxAgeForAdditionalPrefetchedSuggestion - + base::TimeDelta::FromSeconds(1)); + + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder() + .AddId("http://other.com") + .SetUrl("http://other.com") + .SetAmpUrl("http://amp.other.com") + .SetFetchDate(provider_clock->Now()) + .SetPublishDate(GetDefaultCreationTime())) + .Build()); + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_tracker, + PrefetchedOfflinePageExists(GURL("http://amp.prefetched.com"))) + .WillOnce(Return(true)); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(2)); + + provider_clock->Advance(base::TimeDelta::FromSeconds(2)); + + fetched_categories.clear(); + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder( + RemoteSuggestionBuilder() + .AddId("http://other.com") + .SetUrl("http://other.com") + .SetAmpUrl("http://amp.other.com") + .SetFetchDate(provider_clock->Now()) + .SetPublishDate(GetDefaultCreationTime())) + .Build()); + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_tracker, + PrefetchedOfflinePageExists(GURL("http://amp.prefetched.com"))) + .WillOnce(Return(true)); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), + ElementsAre(Property(&ContentSuggestion::id, + MakeArticleID("http://other.com")))); +} + +TEST_F(RemoteSuggestionsProviderImplTest, + ShouldWaitForPrefetchedPagesTrackerInitialization) { + EnableKeepingPrefetchedContentSuggestions( + kMaxAdditionalPrefetchedSuggestions, + kMaxAgeForAdditionalPrefetchedSuggestion); + + auto provider = MakeSuggestionsProviderWithoutInitialization( + /*use_mock_prefetched_pages_tracker=*/true); + auto* mock_tracker = static_cast<StrictMock<MockPrefetchedPagesTracker>*>( + prefetched_pages_tracker()); + WaitForSuggestionsProviderInitialization(provider.get()); + + base::OnceCallback<void()> initialization_completed_callback; + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(false)); + EXPECT_CALL(*mock_tracker, AddInitializationCompletedCallback(_)) + .WillOnce(MoveFirstArgumentPointeeTo(&initialization_completed_callback)); + std::vector<FetchedCategory> fetched_categories; + fetched_categories.push_back( + FetchedCategoryBuilder() + .SetCategory(articles_category()) + .AddSuggestionViaBuilder(RemoteSuggestionBuilder() + .AddId("http://prefetched.com") + .SetUrl("http://prefetched.com") + .SetAmpUrl("http://amp.prefetched.com")) + .Build()); + FetchTheseSuggestions(provider.get(), /*interactive_request=*/true, + Status(StatusCode::SUCCESS, "message"), + std::move(fetched_categories)); + EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(0)); + + EXPECT_CALL(*mock_tracker, IsInitialized()).WillRepeatedly(Return(true)); + std::move(initialization_completed_callback).Run(); + EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); } } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler.h b/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler.h index 6af62879fc8..78d34176168 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler.h +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler.h @@ -56,15 +56,15 @@ class RemoteSuggestionsScheduler { // To keep start ups fast, defer any work possible. virtual void OnBrowserColdStart() = 0; - // Called whenever a new NTP is opened. This may be called on cold starts. - // So to keep start ups fast, defer heavy work for cold starts. - virtual void OnNTPOpened() = 0; + // Called whenever a new suggestions surface is opened. This may be called on + // cold starts. So to keep start ups fast, defer heavy work for cold starts. + virtual void OnSuggestionsSurfaceOpened() = 0; - // Fetch content suggestions. + // Called by PersistentScheduler implementation whenever it wakes up according + // to its schedule. Avoid heavy work, Chrome may be running in the background. virtual void OnPersistentSchedulerWakeUp() = 0; - // Force rescheduling of fetching. - virtual void RescheduleFetching() = 0; + virtual void OnBrowserUpgraded() = 0; }; } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl.cc b/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl.cc index 193628a55e4..064a30d3f46 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl.cc @@ -4,11 +4,13 @@ #include "components/ntp_snippets/remote/remote_suggestions_scheduler_impl.h" +#include <cfloat> #include <random> #include <string> #include <utility> #include "base/bind.h" +#include "base/feature_list.h" #include "base/memory/ptr_util.h" #include "base/metrics/field_trial_params.h" #include "base/metrics/histogram_macros.h" @@ -62,11 +64,23 @@ enum class FetchingInterval { // defined by the enum FetchingInterval. The default time intervals defined in // the arrays can be overridden using different variation parameters. const double kDefaultFetchingIntervalHoursRareNtpUser[] = {192.0, 96.0, 48.0, - 24.0, 12.0, 8.0}; -const double kDefaultFetchingIntervalHoursActiveNtpUser[] = {96.0, 48.0, 48.0, - 24.0, 12.0, 8.0}; + 48.0, 10.0, 10.0}; +const double kDefaultFetchingIntervalHoursActiveNtpUser[] = {96.0, 48.0, 24.0, + 24.0, 10.0, 10.0}; const double kDefaultFetchingIntervalHoursActiveSuggestionsConsumer[] = { - 48.0, 24.0, 24.0, 6.0, 2.0, 1.0}; + 48.0, 24.0, 12.0, 12.0, 1.0, 1.0}; + +// For a simple comparision: fetching intervals that emulate the state really +// rolled out to 100% M58 Stable. Used for evaluation of later changes. DBL_MAX +// values simulate this interval being disabled. +// TODO(jkrcal): Remove when not needed any more, incl. the feature. Probably +// after M62 when CH is launched. +const double kM58FetchingIntervalHoursRareNtpUser[] = {48.0, 24.0, DBL_MAX, + DBL_MAX, 4.0, 4.0}; +const double kM58FetchingIntervalHoursActiveNtpUser[] = {24.0, 8.0, DBL_MAX, + DBL_MAX, 10.0, 10.0}; +const double kM58FetchingIntervalHoursActiveSuggestionsConsumer[] = { + 24.0, 6.0, DBL_MAX, DBL_MAX, 1.0, 1.0}; // Variation parameters than can be used to override the default fetching // intervals. For backwards compatibility, we do not rename @@ -102,6 +116,12 @@ static_assert( static_cast<unsigned int>(FetchingInterval::COUNT) == arraysize(kDefaultFetchingIntervalHoursActiveSuggestionsConsumer) && static_cast<unsigned int>(FetchingInterval::COUNT) == + arraysize(kM58FetchingIntervalHoursRareNtpUser) && + static_cast<unsigned int>(FetchingInterval::COUNT) == + arraysize(kM58FetchingIntervalHoursActiveNtpUser) && + static_cast<unsigned int>(FetchingInterval::COUNT) == + arraysize(kM58FetchingIntervalHoursActiveSuggestionsConsumer) && + static_cast<unsigned int>(FetchingInterval::COUNT) == arraysize(kFetchingIntervalParamNameRareNtpUser) && static_cast<unsigned int>(FetchingInterval::COUNT) == arraysize(kFetchingIntervalParamNameActiveNtpUser) && @@ -109,6 +129,8 @@ static_assert( arraysize(kFetchingIntervalParamNameActiveSuggestionsConsumer), "Fill in all the info for fetching intervals."); +// For backward compatibility "ntp_opened" value is kept and denotes the +// SURFACE_OPENED trigger type. const char* kTriggerTypeNames[] = {"persistent_scheduler_wake_up", "ntp_opened", "browser_foregrounded", "browser_cold_start"}; @@ -132,20 +154,29 @@ base::TimeDelta GetDesiredFetchingInterval( const unsigned int index = static_cast<unsigned int>(interval); DCHECK(index < arraysize(kDefaultFetchingIntervalHoursRareNtpUser)); + bool emulateM58 = base::FeatureList::IsEnabled( + kRemoteSuggestionsEmulateM58FetchingSchedule); + double default_value_hours = 0.0; const char* param_name = nullptr; switch (user_class) { case UserClassifier::UserClass::RARE_NTP_USER: - default_value_hours = kDefaultFetchingIntervalHoursRareNtpUser[index]; + default_value_hours = + emulateM58 ? kM58FetchingIntervalHoursRareNtpUser[index] + : kDefaultFetchingIntervalHoursRareNtpUser[index]; param_name = kFetchingIntervalParamNameRareNtpUser[index]; break; case UserClassifier::UserClass::ACTIVE_NTP_USER: - default_value_hours = kDefaultFetchingIntervalHoursActiveNtpUser[index]; + default_value_hours = + emulateM58 ? kM58FetchingIntervalHoursActiveNtpUser[index] + : kDefaultFetchingIntervalHoursActiveNtpUser[index]; param_name = kFetchingIntervalParamNameActiveNtpUser[index]; break; case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER: default_value_hours = - kDefaultFetchingIntervalHoursActiveSuggestionsConsumer[index]; + emulateM58 + ? kM58FetchingIntervalHoursActiveSuggestionsConsumer[index] + : kDefaultFetchingIntervalHoursActiveSuggestionsConsumer[index]; param_name = kFetchingIntervalParamNameActiveSuggestionsConsumer[index]; break; } @@ -157,29 +188,62 @@ base::TimeDelta GetDesiredFetchingInterval( return base::TimeDelta::FromSecondsD(value_hours * 3600.0); } -void ReportTimeUntilFirstSoftTrigger(UserClassifier::UserClass user_class, - base::TimeDelta time_until_first_trigger) { +void ReportTimeUntilFirstShownTrigger( + UserClassifier::UserClass user_class, + base::TimeDelta time_until_first_shown_trigger) { + switch (user_class) { + case UserClassifier::UserClass::RARE_NTP_USER: + UMA_HISTOGRAM_CUSTOM_TIMES( + "NewTabPage.ContentSuggestions.TimeUntilFirstShownTrigger." + "RareNTPUser", + time_until_first_shown_trigger, base::TimeDelta::FromSeconds(1), + base::TimeDelta::FromDays(7), + /*bucket_count=*/50); + break; + case UserClassifier::UserClass::ACTIVE_NTP_USER: + UMA_HISTOGRAM_CUSTOM_TIMES( + "NewTabPage.ContentSuggestions.TimeUntilFirstShownTrigger." + "ActiveNTPUser", + time_until_first_shown_trigger, base::TimeDelta::FromSeconds(1), + base::TimeDelta::FromDays(7), + /*bucket_count=*/50); + break; + case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER: + UMA_HISTOGRAM_CUSTOM_TIMES( + "NewTabPage.ContentSuggestions.TimeUntilFirstShownTrigger." + "ActiveSuggestionsConsumer", + time_until_first_shown_trigger, base::TimeDelta::FromSeconds(1), + base::TimeDelta::FromDays(7), + /*bucket_count=*/50); + break; + } +} + +void ReportTimeUntilFirstStartupTrigger( + UserClassifier::UserClass user_class, + base::TimeDelta time_until_first_startup_trigger) { switch (user_class) { case UserClassifier::UserClass::RARE_NTP_USER: UMA_HISTOGRAM_CUSTOM_TIMES( - "NewTabPage.ContentSuggestions.TimeUntilFirstSoftTrigger.RareNTPUser", - time_until_first_trigger, base::TimeDelta::FromSeconds(1), + "NewTabPage.ContentSuggestions.TimeUntilFirstStartupTrigger." + "RareNTPUser", + time_until_first_startup_trigger, base::TimeDelta::FromSeconds(1), base::TimeDelta::FromDays(7), /*bucket_count=*/50); break; case UserClassifier::UserClass::ACTIVE_NTP_USER: UMA_HISTOGRAM_CUSTOM_TIMES( - "NewTabPage.ContentSuggestions.TimeUntilFirstSoftTrigger." + "NewTabPage.ContentSuggestions.TimeUntilFirstStartupTrigger." "ActiveNTPUser", - time_until_first_trigger, base::TimeDelta::FromSeconds(1), + time_until_first_startup_trigger, base::TimeDelta::FromSeconds(1), base::TimeDelta::FromDays(7), /*bucket_count=*/50); break; case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER: UMA_HISTOGRAM_CUSTOM_TIMES( - "NewTabPage.ContentSuggestions.TimeUntilFirstSoftTrigger." + "NewTabPage.ContentSuggestions.TimeUntilFirstStartupTrigger." "ActiveSuggestionsConsumer", - time_until_first_trigger, base::TimeDelta::FromSeconds(1), + time_until_first_startup_trigger, base::TimeDelta::FromSeconds(1), base::TimeDelta::FromDays(7), /*bucket_count=*/50); break; @@ -356,7 +420,7 @@ bool RemoteSuggestionsSchedulerImpl::FetchingSchedule::is_empty() const { // |kTriggerTypeNames| above. enum class RemoteSuggestionsSchedulerImpl::TriggerType { PERSISTENT_SCHEDULER_WAKE_UP = 0, - NTP_OPENED = 1, + SURFACE_OPENED = 1, BROWSER_FOREGROUNDED = 2, BROWSER_COLD_START = 3, COUNT @@ -384,7 +448,8 @@ RemoteSuggestionsSchedulerImpl::RemoteSuggestionsSchedulerImpl( profile_prefs, RequestThrottler::RequestType:: CONTENT_SUGGESTION_FETCHER_ACTIVE_SUGGESTIONS_CONSUMER), - time_until_first_trigger_reported_(false), + time_until_first_shown_trigger_reported_(false), + time_until_first_startup_trigger_reported_(false), eula_state_(base::MakeUnique<EulaState>(local_state_prefs, this)), profile_prefs_(profile_prefs), clock_(std::move(clock)), @@ -454,8 +519,9 @@ void RemoteSuggestionsSchedulerImpl::OnHistoryCleared() { ClearLastFetchAttemptTime(); } -void RemoteSuggestionsSchedulerImpl::RescheduleFetching() { - // Force the reschedule by stopping and starting it again. +void RemoteSuggestionsSchedulerImpl::OnBrowserUpgraded() { + // After browser upgrade, persistent schedule needs to get reset. Force the + // reschedule by stopping and starting it again. StopScheduling(); StartScheduling(); } @@ -486,10 +552,10 @@ void RemoteSuggestionsSchedulerImpl::OnBrowserColdStart() { RefetchInTheBackgroundIfAppropriate(TriggerType::BROWSER_COLD_START); } -void RemoteSuggestionsSchedulerImpl::OnNTPOpened() { +void RemoteSuggestionsSchedulerImpl::OnSuggestionsSurfaceOpened() { // TODO(jkrcal): Consider that this is called whenever we open an NTP. // Therefore, keep work light for fast start up calls. - RefetchInTheBackgroundIfAppropriate(TriggerType::NTP_OPENED); + RefetchInTheBackgroundIfAppropriate(TriggerType::SURFACE_OPENED); } void RemoteSuggestionsSchedulerImpl::StartScheduling() { @@ -589,21 +655,36 @@ void RemoteSuggestionsSchedulerImpl::RefetchInTheBackgroundIfAppropriate( return; } + if (net::NetworkChangeNotifier::IsOffline()) { + // Do not let a request fail due to lack of internet connection. Then, such + // a failure would get logged and further requests would be blocked for a + // while (even after becoming online). + return; + } + if (BackgroundFetchesDisabled(trigger)) { return; } - bool is_soft = trigger != TriggerType::PERSISTENT_SCHEDULER_WAKE_UP; const base::Time last_fetch_attempt_time = base::Time::FromInternalValue( profile_prefs_->GetInt64(prefs::kSnippetLastFetchAttempt)); - if (is_soft && !time_until_first_trigger_reported_) { - time_until_first_trigger_reported_ = true; - ReportTimeUntilFirstSoftTrigger(user_classifier_->GetUserClass(), - clock_->Now() - last_fetch_attempt_time); + if (trigger == TriggerType::SURFACE_OPENED && + !time_until_first_shown_trigger_reported_) { + time_until_first_shown_trigger_reported_ = true; + ReportTimeUntilFirstShownTrigger(user_classifier_->GetUserClass(), + clock_->Now() - last_fetch_attempt_time); + } + + if ((trigger == TriggerType::BROWSER_FOREGROUNDED || + trigger == TriggerType::BROWSER_COLD_START) && + !time_until_first_startup_trigger_reported_) { + time_until_first_startup_trigger_reported_ = true; + ReportTimeUntilFirstStartupTrigger(user_classifier_->GetUserClass(), + clock_->Now() - last_fetch_attempt_time); } - if (is_soft && + if (trigger != TriggerType::PERSISTENT_SCHEDULER_WAKE_UP && !ShouldRefetchInTheBackgroundNow(last_fetch_attempt_time, trigger)) { return; } @@ -617,7 +698,7 @@ void RemoteSuggestionsSchedulerImpl::RefetchInTheBackgroundIfAppropriate( case TriggerType::PERSISTENT_SCHEDULER_WAKE_UP: ReportTimeUntilPersistentFetch(user_classifier_->GetUserClass(), diff); break; - case TriggerType::NTP_OPENED: + case TriggerType::SURFACE_OPENED: ReportTimeUntilShownFetch(user_classifier_->GetUserClass(), diff); break; case TriggerType::BROWSER_FOREGROUNDED: @@ -648,7 +729,7 @@ bool RemoteSuggestionsSchedulerImpl::ShouldRefetchInTheBackgroundNow( } base::Time first_allowed_fetch_time = last_fetch_attempt_time; - if (trigger == TriggerType::NTP_OPENED) { + if (trigger == TriggerType::SURFACE_OPENED) { first_allowed_fetch_time += (wifi ? schedule_.interval_shown_wifi : schedule_.interval_shown_fallback); } else { @@ -707,7 +788,8 @@ void RemoteSuggestionsSchedulerImpl::RefetchInTheBackgroundFinished( void RemoteSuggestionsSchedulerImpl::OnFetchCompleted(Status fetch_status) { profile_prefs_->SetInt64(prefs::kSnippetLastFetchAttempt, clock_->Now().ToInternalValue()); - time_until_first_trigger_reported_ = false; + time_until_first_shown_trigger_reported_ = false; + time_until_first_startup_trigger_reported_ = false; // Reschedule after a fetch. The persistent schedule is applied only after a // successful fetch. After a failed fetch, we want to keep the previous @@ -764,8 +846,9 @@ RemoteSuggestionsSchedulerImpl::GetEnabledTriggerTypes() { std::set<RemoteSuggestionsSchedulerImpl::TriggerType> RemoteSuggestionsSchedulerImpl::GetDefaultEnabledTriggerTypes() { - return {TriggerType::PERSISTENT_SCHEDULER_WAKE_UP, TriggerType::NTP_OPENED, - TriggerType::BROWSER_COLD_START, TriggerType::BROWSER_FOREGROUNDED}; + return {TriggerType::PERSISTENT_SCHEDULER_WAKE_UP, + TriggerType::SURFACE_OPENED, TriggerType::BROWSER_COLD_START, + TriggerType::BROWSER_FOREGROUNDED}; } } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl.h b/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl.h index 3e103624bec..6c6e63df23b 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl.h +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl.h @@ -52,13 +52,13 @@ class RemoteSuggestionsSchedulerImpl : public RemoteSuggestionsScheduler { void OnProviderDeactivated() override; void OnSuggestionsCleared() override; void OnHistoryCleared() override; - void RescheduleFetching() override; + void OnBrowserUpgraded() override; bool AcquireQuotaForInteractiveFetch() override; void OnInteractiveFetchFinished(Status fetch_status) override; void OnPersistentSchedulerWakeUp() override; void OnBrowserForegrounded() override; void OnBrowserColdStart() override; - void OnNTPOpened() override; + void OnSuggestionsSurfaceOpened() override; private: // Abstract description of the fetching schedule. See the enum @@ -147,8 +147,10 @@ class RemoteSuggestionsSchedulerImpl : public RemoteSuggestionsScheduler { RequestThrottler request_throttler_active_ntp_user_; RequestThrottler request_throttler_active_suggestions_consumer_; - // To make sure we only report the first trigger to UMA. - bool time_until_first_trigger_reported_; + // Variables to make sure we only report the first trigger of each kind to + // UMA. + bool time_until_first_shown_trigger_reported_; + bool time_until_first_startup_trigger_reported_; // We should not fetch in background before EULA gets accepted. std::unique_ptr<EulaState> eula_state_; diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl_unittest.cc b/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl_unittest.cc index 0c22282ea0a..8567849469f 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl_unittest.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_scheduler_impl_unittest.cc @@ -15,6 +15,7 @@ #include "base/memory/ptr_util.h" #include "base/message_loop/message_loop.h" #include "base/run_loop.h" +#include "base/test/scoped_feature_list.h" #include "base/test/simple_test_clock.h" #include "base/threading/thread_task_runner_handle.h" #include "base/time/clock.h" @@ -28,8 +29,8 @@ #include "components/ntp_snippets/remote/test_utils.h" #include "components/ntp_snippets/status.h" #include "components/ntp_snippets/user_classifier.h" -#include "components/prefs/pref_registry_simple.h" #include "components/prefs/testing_pref_service.h" +#include "components/sync_preferences/testing_pref_service_syncable.h" #include "components/variations/variations_params_manager.h" #include "components/web_resource/web_resource_pref_names.h" #include "net/base/network_change_notifier.h" @@ -80,6 +81,7 @@ class MockRemoteSuggestionsProvider : public RemoteSuggestionsProvider { const RemoteSuggestionsFetcher*()); MOCK_CONST_METHOD1(GetUrlWithFavicon, GURL(const ContentSuggestion::ID& suggestion_id)); + MOCK_CONST_METHOD0(IsDisabled, bool()); MOCK_METHOD1(GetCategoryStatus, CategoryStatus(Category)); MOCK_METHOD1(GetCategoryInfo, CategoryInfo(Category)); MOCK_METHOD3(ClearHistory, @@ -101,6 +103,13 @@ class MockRemoteSuggestionsProvider : public RemoteSuggestionsProvider { MOCK_METHOD0(OnSignInStateChanged, void()); }; +class FakeOfflineNetworkChangeNotifier : public net::NetworkChangeNotifier { + public: + ConnectionType GetCurrentConnectionType() const override { + return NetworkChangeNotifier::CONNECTION_NONE; + } +}; + } // namespace class RemoteSuggestionsSchedulerImplTest : public ::testing::Test { @@ -205,7 +214,7 @@ class RemoteSuggestionsSchedulerImplTest : public ::testing::Test { TEST_F(RemoteSuggestionsSchedulerImplTest, ShouldIgnoreSignalsWhenNotEnabled) { scheduler()->OnPersistentSchedulerWakeUp(); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); scheduler()->OnBrowserForegrounded(); scheduler()->OnBrowserColdStart(); } @@ -238,7 +247,7 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, // All signals are ignored because of Eula not being accepted. scheduler()->OnPersistentSchedulerWakeUp(); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); scheduler()->OnBrowserForegrounded(); scheduler()->OnBrowserColdStart(); } @@ -271,7 +280,7 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, ActivateProvider(); scheduler()->OnPersistentSchedulerWakeUp(); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); scheduler()->OnBrowserForegrounded(); scheduler()->OnBrowserColdStart(); } @@ -359,7 +368,7 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, } TEST_F(RemoteSuggestionsSchedulerImplTest, - ShouldFetchOnNTPOpenedForTheFirstTime) { + ShouldFetchOnSuggestionsSurfaceOpenedForTheFirstTime) { // First set only this type to be allowed. SetVariationParameter("scheduler_trigger_types", "ntp_opened"); ResetProvider(); @@ -369,7 +378,7 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, ActivateProvider(); EXPECT_CALL(*provider(), RefetchInTheBackground(_)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); } TEST_F(RemoteSuggestionsSchedulerImplTest, @@ -401,7 +410,7 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, } TEST_F(RemoteSuggestionsSchedulerImplTest, - ShouldNotFetchOnNTPOpenedAfterSuccessfulSoftFetch) { + ShouldNotFetchOnSuggestionsSurfaceOpenedAfterSuccessfulSoftFetch) { // First enable the scheduler; the second Schedule is called after the // successful fetch. EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)).Times(2); @@ -411,14 +420,14 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, RemoteSuggestionsProvider::FetchStatusCallback signal_fetch_done; EXPECT_CALL(*provider(), RefetchInTheBackground(_)) .WillOnce(SaveArg<0>(&signal_fetch_done)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); signal_fetch_done.Run(Status::Success()); // The second call is ignored if it happens right after the first one. - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); } TEST_F(RemoteSuggestionsSchedulerImplTest, - ShouldNotFetchOnNTPOpenedAfterSuccessfulPersistentFetch) { + ShouldNotFetchOnSuggestionsSurfaceOpenedAfterSuccessfulPersistentFetch) { // First enable the scheduler; the second Schedule is called after the // successful fetch. EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)).Times(2); @@ -431,11 +440,11 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, scheduler()->OnPersistentSchedulerWakeUp(); signal_fetch_done.Run(Status::Success()); // The second call is ignored if it happens right after the first one. - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); } TEST_F(RemoteSuggestionsSchedulerImplTest, - ShouldNotFetchOnNTPOpenedAfterFailedSoftFetch) { + ShouldNotFetchOnSuggestionsSurfaceOpenedAfterFailedSoftFetch) { // First enable the scheduler. EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); ActivateProvider(); @@ -444,15 +453,15 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, RemoteSuggestionsProvider::FetchStatusCallback signal_fetch_done; EXPECT_CALL(*provider(), RefetchInTheBackground(_)) .WillOnce(SaveArg<0>(&signal_fetch_done)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); signal_fetch_done.Run(Status(StatusCode::PERMANENT_ERROR, "")); // The second call is ignored if it happens right after the first one. - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); } TEST_F(RemoteSuggestionsSchedulerImplTest, - ShouldNotFetchOnNTPOpenedAfterFailedPersistentFetch) { + ShouldNotFetchOnSuggestionsSurfaceOpenedAfterFailedPersistentFetch) { // First enable the scheduler. EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); ActivateProvider(); @@ -465,7 +474,7 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, signal_fetch_done.Run(Status(StatusCode::PERMANENT_ERROR, "")); // The second call is ignored if it happens right after the first one. - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); } TEST_F(RemoteSuggestionsSchedulerImplTest, @@ -494,10 +503,9 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, scheduler()->OnBrowserForegrounded(); } -TEST_F(RemoteSuggestionsSchedulerImplTest, - ShouldRescheduleOnRescheduleFetching) { +TEST_F(RemoteSuggestionsSchedulerImplTest, ShouldRescheduleOnBrowserUpgraded) { EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); - scheduler()->RescheduleFetching(); + scheduler()->OnBrowserUpgraded(); } TEST_F(RemoteSuggestionsSchedulerImplTest, ShouldScheduleOnActivation) { @@ -662,8 +670,6 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, FetchIntervalForShownTriggerOnWifi) { // Pretend we are on WiFi (already done in ctor, we make it explicit here). EXPECT_CALL(*persistent_scheduler(), IsOnUnmeteredConnection()) .WillRepeatedly(Return(true)); - // UserClassifier defaults to UserClass::ACTIVE_NTP_USER which uses a 8h time - // interval by default for shown trigger on WiFi. // Initial scheduling after being enabled. EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); @@ -673,19 +679,22 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, FetchIntervalForShownTriggerOnWifi) { RemoteSuggestionsProvider::FetchStatusCallback signal_fetch_done; EXPECT_CALL(*provider(), RefetchInTheBackground(_)) .WillOnce(SaveArg<0>(&signal_fetch_done)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); // Rescheduling after a succesful fetch. EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); signal_fetch_done.Run(Status::Success()); - // Open NTP again after too short delay. This time no fetch is executed. - test_clock()->Advance(base::TimeDelta::FromHours(1)); - scheduler()->OnNTPOpened(); + // Open NTP again after too short delay (one minute missing). UserClassifier + // defaults to UserClass::ACTIVE_NTP_USER - we work with the default interval + // for this class here. This time no fetch is executed. + test_clock()->Advance(base::TimeDelta::FromHours(10) - + base::TimeDelta::FromMinutes(1)); + scheduler()->OnSuggestionsSurfaceOpened(); // Open NTP after another delay, now together long enough to issue a fetch. - test_clock()->Advance(base::TimeDelta::FromHours(7)); + test_clock()->Advance(base::TimeDelta::FromMinutes(2)); EXPECT_CALL(*provider(), RefetchInTheBackground(_)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); } TEST_F(RemoteSuggestionsSchedulerImplTest, @@ -706,19 +715,19 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, RemoteSuggestionsProvider::FetchStatusCallback signal_fetch_done; EXPECT_CALL(*provider(), RefetchInTheBackground(_)) .WillOnce(SaveArg<0>(&signal_fetch_done)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); // Rescheduling after a succesful fetch. EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); signal_fetch_done.Run(Status::Success()); // Open NTP again after too short delay. This time no fetch is executed. test_clock()->Advance(base::TimeDelta::FromMinutes(20)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); // Open NTP after another delay, now together long enough to issue a fetch. test_clock()->Advance(base::TimeDelta::FromMinutes(10)); EXPECT_CALL(*provider(), RefetchInTheBackground(_)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); } TEST_F(RemoteSuggestionsSchedulerImplTest, @@ -737,19 +746,19 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, RemoteSuggestionsProvider::FetchStatusCallback signal_fetch_done; EXPECT_CALL(*provider(), RefetchInTheBackground(_)) .WillOnce(SaveArg<0>(&signal_fetch_done)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); // Rescheduling after a succesful fetch. EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); signal_fetch_done.Run(Status::Success()); // Open NTP again after too short delay. This time no fetch is executed. test_clock()->Advance(base::TimeDelta::FromHours(5)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); // Open NTP after another delay, now together long enough to issue a fetch. test_clock()->Advance(base::TimeDelta::FromHours(7)); EXPECT_CALL(*provider(), RefetchInTheBackground(_)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); } TEST_F(RemoteSuggestionsSchedulerImplTest, @@ -770,19 +779,19 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, RemoteSuggestionsProvider::FetchStatusCallback signal_fetch_done; EXPECT_CALL(*provider(), RefetchInTheBackground(_)) .WillOnce(SaveArg<0>(&signal_fetch_done)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); // Rescheduling after a succesful fetch. EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); signal_fetch_done.Run(Status::Success()); // Open NTP again after too short delay. This time no fetch is executed. test_clock()->Advance(base::TimeDelta::FromMinutes(20)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); // Open NTP after another delay, now together long enough to issue a fetch. test_clock()->Advance(base::TimeDelta::FromMinutes(10)); EXPECT_CALL(*provider(), RefetchInTheBackground(_)); - scheduler()->OnNTPOpened(); + scheduler()->OnSuggestionsSurfaceOpened(); } TEST_F(RemoteSuggestionsSchedulerImplTest, @@ -867,4 +876,45 @@ TEST_F(RemoteSuggestionsSchedulerImplTest, scheduler()->OnPersistentSchedulerWakeUp(); } +TEST_F(RemoteSuggestionsSchedulerImplTest, + ShouldIgnoreSubsequentStartupSignalsForM58) { + base::test::ScopedFeatureList feature_list; + feature_list.InitAndEnableFeature( + kRemoteSuggestionsEmulateM58FetchingSchedule); + RemoteSuggestionsProvider::FetchStatusCallback signal_fetch_done; + + // First enable the scheduler -- this will trigger the persistent scheduling. + EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); + ActivateProvider(); + + // The startup triggers are ignored. + EXPECT_CALL(*provider(), RefetchInTheBackground(_)).Times(0); + scheduler()->OnBrowserForegrounded(); + scheduler()->OnBrowserColdStart(); + + // Foreground the browser again after a very long delay. Again, no fetch is + // executed for neither Foregrounded, nor ColdStart. + test_clock()->Advance(base::TimeDelta::FromHours(100000)); + scheduler()->OnBrowserForegrounded(); + scheduler()->OnBrowserColdStart(); +} + +TEST_F(RemoteSuggestionsSchedulerImplTest, ShouldIgnoreSignalsWhenOffline) { + // Simulate being offline. NetworkChangeNotifier is a singleton, thus, this + // instance is actually globally accessible (from the static function + // NetworkChangeNotifier::IsOffline() that is called from the scheduler). + FakeOfflineNetworkChangeNotifier fake; + + // Activating the provider should schedule the persistent background fetches. + EXPECT_CALL(*persistent_scheduler(), Schedule(_, _)); + scheduler()->OnProviderActivated(); + + // All signals are ignored because of being offline. + EXPECT_CALL(*provider(), RefetchInTheBackground(_)).Times(0); + scheduler()->OnPersistentSchedulerWakeUp(); + scheduler()->OnSuggestionsSurfaceOpened(); + scheduler()->OnBrowserForegrounded(); + scheduler()->OnBrowserColdStart(); +} + } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/remote_suggestions_status_service_unittest.cc b/chromium/components/ntp_snippets/remote/remote_suggestions_status_service_unittest.cc index dcfc9b83e67..65ca301f231 100644 --- a/chromium/components/ntp_snippets/remote/remote_suggestions_status_service_unittest.cc +++ b/chromium/components/ntp_snippets/remote/remote_suggestions_status_service_unittest.cc @@ -7,16 +7,17 @@ #include <memory> #include "base/memory/ptr_util.h" +#include "build/build_config.h" #include "components/ntp_snippets/features.h" #include "components/ntp_snippets/ntp_snippets_constants.h" #include "components/ntp_snippets/pref_names.h" #include "components/ntp_snippets/remote/test_utils.h" -#include "components/prefs/pref_registry_simple.h" #include "components/prefs/testing_pref_service.h" #include "components/signin/core/browser/account_tracker_service.h" #include "components/signin/core/browser/fake_signin_manager.h" #include "components/signin/core/browser/test_signin_client.h" #include "components/signin/core/common/signin_pref_names.h" +#include "components/sync_preferences/testing_pref_service_syncable.h" #include "components/variations/variations_params_manager.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" @@ -48,7 +49,11 @@ TEST_F(RemoteSuggestionsStatusServiceTest, NoSigninNeeded) { service->GetStatusFromDeps()); // One can still sign in. +#if defined(OS_CHROMEOS) utils_.fake_signin_manager()->SignIn("foo@bar.com"); +#else + utils_.fake_signin_manager()->SignIn("foo@bar.com", "user", "pass"); +#endif EXPECT_EQ(RemoteSuggestionsStatus::ENABLED_AND_SIGNED_IN, service->GetStatusFromDeps()); } @@ -66,7 +71,11 @@ TEST_F(RemoteSuggestionsStatusServiceTest, DisabledViaPref) { service->GetStatusFromDeps()); // The other dependencies shouldn't matter anymore. +#if defined(OS_CHROMEOS) utils_.fake_signin_manager()->SignIn("foo@bar.com"); +#else + utils_.fake_signin_manager()->SignIn("foo@bar.com", "user", "pass"); +#endif EXPECT_EQ(RemoteSuggestionsStatus::EXPLICITLY_DISABLED, service->GetStatusFromDeps()); } diff --git a/chromium/components/ntp_snippets/remote/test_utils.cc b/chromium/components/ntp_snippets/remote/test_utils.cc index eb63bc40431..18809331a60 100644 --- a/chromium/components/ntp_snippets/remote/test_utils.cc +++ b/chromium/components/ntp_snippets/remote/test_utils.cc @@ -7,10 +7,9 @@ #include <memory> #include "base/memory/ptr_util.h" -#include "components/prefs/pref_registry_simple.h" #include "components/prefs/testing_pref_service.h" #include "components/signin/core/browser/account_tracker_service.h" -#include "components/signin/core/browser/fake_signin_manager.h" +#include "components/signin/core/browser/fake_profile_oauth2_token_service.h" #include "components/signin/core/browser/test_signin_client.h" #include "components/signin/core/common/signin_pref_names.h" #include "components/sync/driver/fake_sync_service.h" @@ -48,24 +47,37 @@ syncer::ModelTypeSet FakeSyncService::GetActiveDataTypes() const { } RemoteSuggestionsTestUtils::RemoteSuggestionsTestUtils() - : pref_service_(base::MakeUnique<TestingPrefServiceSimple>()) { - pref_service_->registry()->RegisterStringPref(prefs::kGoogleServicesAccountId, - std::string()); - pref_service_->registry()->RegisterStringPref( - prefs::kGoogleServicesLastAccountId, std::string()); - pref_service_->registry()->RegisterStringPref( - prefs::kGoogleServicesLastUsername, std::string()); + : pref_service_(base::MakeUnique<TestingPrefServiceSyncable>()) { + AccountTrackerService::RegisterPrefs(pref_service_->registry()); + +#if defined(OS_CHROMEOS) + SigninManagerBase::RegisterProfilePrefs(pref_service_->registry()); + SigninManagerBase::RegisterPrefs(pref_service_->registry()); +#else + SigninManager::RegisterProfilePrefs(pref_service_->registry()); + SigninManager::RegisterPrefs(pref_service_->registry()); +#endif // OS_CHROMEOS + + token_service_ = base::MakeUnique<FakeProfileOAuth2TokenService>(); signin_client_ = base::MakeUnique<TestSigninClient>(pref_service_.get()); account_tracker_ = base::MakeUnique<AccountTrackerService>(); + account_tracker_->Initialize(signin_client_.get()); fake_sync_service_ = base::MakeUnique<FakeSyncService>(); + ResetSigninManager(); } RemoteSuggestionsTestUtils::~RemoteSuggestionsTestUtils() = default; void RemoteSuggestionsTestUtils::ResetSigninManager() { +#if defined(OS_CHROMEOS) fake_signin_manager_ = base::MakeUnique<FakeSigninManagerBase>( signin_client_.get(), account_tracker_.get()); +#else + fake_signin_manager_ = base::MakeUnique<FakeSigninManager>( + signin_client_.get(), token_service_.get(), account_tracker_.get(), + /*cookie_manager_service=*/nullptr); +#endif } } // namespace test diff --git a/chromium/components/ntp_snippets/remote/test_utils.h b/chromium/components/ntp_snippets/remote/test_utils.h index d7cdfa1ba50..ed825a6196f 100644 --- a/chromium/components/ntp_snippets/remote/test_utils.h +++ b/chromium/components/ntp_snippets/remote/test_utils.h @@ -7,14 +7,25 @@ #include <memory> +#include "build/build_config.h" +#include "components/signin/core/browser/fake_signin_manager.h" #include "components/sync/driver/fake_sync_service.h" +#include "components/sync_preferences/testing_pref_service_syncable.h" #include "testing/gtest/include/gtest/gtest.h" class AccountTrackerService; -class FakeSigninManagerBase; -class TestingPrefServiceSimple; +class FakeProfileOAuth2TokenService; class TestSigninClient; +using sync_preferences::TestingPrefServiceSyncable; + +#if defined(OS_CHROMEOS) +// ChromeOS doesn't have SigninManager. +using SigninManagerForTest = FakeSigninManagerBase; +#else +using SigninManagerForTest = FakeSigninManager; +#endif // OS_CHROMEOS + namespace ntp_snippets { namespace test { @@ -46,15 +57,19 @@ class RemoteSuggestionsTestUtils { void ResetSigninManager(); FakeSyncService* fake_sync_service() { return fake_sync_service_.get(); } - FakeSigninManagerBase* fake_signin_manager() { + SigninManagerForTest* fake_signin_manager() { return fake_signin_manager_.get(); } - TestingPrefServiceSimple* pref_service() { return pref_service_.get(); } + TestingPrefServiceSyncable* pref_service() { return pref_service_.get(); } + FakeProfileOAuth2TokenService* token_service() { + return token_service_.get(); + } private: - std::unique_ptr<FakeSigninManagerBase> fake_signin_manager_; + std::unique_ptr<SigninManagerForTest> fake_signin_manager_; std::unique_ptr<FakeSyncService> fake_sync_service_; - std::unique_ptr<TestingPrefServiceSimple> pref_service_; + std::unique_ptr<TestingPrefServiceSyncable> pref_service_; + std::unique_ptr<FakeProfileOAuth2TokenService> token_service_; std::unique_ptr<TestSigninClient> signin_client_; std::unique_ptr<AccountTrackerService> account_tracker_; }; |