diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2017-01-04 14:17:57 +0100 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2017-01-05 10:05:06 +0000 |
commit | 39d357e3248f80abea0159765ff39554affb40db (patch) | |
tree | aba0e6bfb76de0244bba0f5fdbd64b830dd6e621 /chromium/components/ntp_snippets | |
parent | 87778abf5a1f89266f37d1321b92a21851d8244d (diff) | |
download | qtwebengine-chromium-39d357e3248f80abea0159765ff39554affb40db.tar.gz |
BASELINE: Update Chromium to 55.0.2883.105
And updates ninja to 1.7.2
Change-Id: I20d43c737f82764d857ada9a55586901b18b9243
Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/components/ntp_snippets')
89 files changed, 10712 insertions, 2837 deletions
diff --git a/chromium/components/ntp_snippets/BUILD.gn b/chromium/components/ntp_snippets/BUILD.gn index f520319d589..8e712af9eae 100644 --- a/chromium/components/ntp_snippets/BUILD.gn +++ b/chromium/components/ntp_snippets/BUILD.gn @@ -9,30 +9,63 @@ if (is_android) { import("//build/config/android/rules.gni") } -# GYP version: components/ntp_snippets.gypi:ntp_snippets -source_set("ntp_snippets") { +static_library("ntp_snippets") { sources = [ + "bookmarks/bookmark_last_visit_utils.cc", + "bookmarks/bookmark_last_visit_utils.h", + "bookmarks/bookmark_suggestions_provider.cc", + "bookmarks/bookmark_suggestions_provider.h", + "category.cc", + "category.h", + "category_factory.cc", + "category_factory.h", + "category_info.cc", + "category_info.h", + "category_status.cc", + "category_status.h", "content_suggestion.cc", "content_suggestion.h", - "content_suggestion_category.h", - "content_suggestions_provider_type.h", - "ntp_snippet.cc", - "ntp_snippet.h", + "content_suggestions_metrics.cc", + "content_suggestions_metrics.h", + "content_suggestions_provider.cc", + "content_suggestions_provider.h", + "content_suggestions_service.cc", + "content_suggestions_service.h", + "features.cc", + "features.h", "ntp_snippets_constants.cc", "ntp_snippets_constants.h", - "ntp_snippets_database.cc", - "ntp_snippets_database.h", - "ntp_snippets_fetcher.cc", - "ntp_snippets_fetcher.h", - "ntp_snippets_scheduler.h", - "ntp_snippets_service.cc", - "ntp_snippets_service.h", - "ntp_snippets_status_service.cc", - "ntp_snippets_status_service.h", + "offline_pages/offline_page_proxy.cc", + "offline_pages/offline_page_proxy.h", + "offline_pages/recent_tab_suggestions_provider.cc", + "offline_pages/recent_tab_suggestions_provider.h", + "physical_web_pages/physical_web_page_suggestions_provider.cc", + "physical_web_pages/physical_web_page_suggestions_provider.h", "pref_names.cc", "pref_names.h", + "pref_util.cc", + "pref_util.h", + "remote/ntp_snippet.cc", + "remote/ntp_snippet.h", + "remote/ntp_snippets_database.cc", + "remote/ntp_snippets_database.h", + "remote/ntp_snippets_fetcher.cc", + "remote/ntp_snippets_fetcher.h", + "remote/ntp_snippets_scheduler.h", + "remote/ntp_snippets_service.cc", + "remote/ntp_snippets_service.h", + "remote/ntp_snippets_status_service.cc", + "remote/ntp_snippets_status_service.h", + "remote/request_throttler.cc", + "remote/request_throttler.h", + "sessions/foreign_sessions_suggestions_provider.cc", + "sessions/foreign_sessions_suggestions_provider.h", + "sessions/tab_delegate_sync_adapter.cc", + "sessions/tab_delegate_sync_adapter.h", "switches.cc", "switches.h", + "user_classifier.cc", + "user_classifier.h", ] public_deps = [ @@ -41,18 +74,24 @@ source_set("ntp_snippets") { "//components/leveldb_proto", "//components/prefs", "//components/signin/core/browser", - "//components/suggestions", - "//components/sync_driver", + "//components/sync", "//google_apis", "//net", + "//ui/base", "//url", ] deps = [ + "//components/bookmarks/browser", "//components/data_use_measurement/core", + "//components/history/core/browser", "//components/image_fetcher", "//components/metrics", - "//components/ntp_snippets/proto", + "//components/ntp_snippets/remote/proto", + "//components/offline_pages", + "//components/sessions", + "//components/strings", + "//components/sync_sessions", "//components/variations", "//components/variations/net", "//third_party/icu/", @@ -63,7 +102,9 @@ source_set("ntp_snippets") { if (is_android) { java_cpp_enum("ntp_snippets_java_enums_srcjar") { sources = [ - "ntp_snippets_status_service.h", + "category.h", + "category_info.h", + "category_status.h", ] } } @@ -71,27 +112,45 @@ if (is_android) { source_set("unit_tests") { testonly = true sources = [ - "ntp_snippet_unittest.cc", - "ntp_snippets_database_unittest.cc", - "ntp_snippets_fetcher_unittest.cc", - "ntp_snippets_service_unittest.cc", - "ntp_snippets_status_service_unittest.cc", - "ntp_snippets_test_utils.cc", - "ntp_snippets_test_utils.h", + "bookmarks/bookmark_last_visit_utils_unittest.cc", + "category_factory_unittest.cc", + "content_suggestions_service_unittest.cc", + "mock_content_suggestions_provider_observer.cc", + "mock_content_suggestions_provider_observer.h", + "offline_pages/recent_tab_suggestions_provider_unittest.cc", + "physical_web_pages/physical_web_page_suggestions_provider_unittest.cc", + "remote/ntp_snippet_unittest.cc", + "remote/ntp_snippets_database_unittest.cc", + "remote/ntp_snippets_fetcher_unittest.cc", + "remote/ntp_snippets_service_unittest.cc", + "remote/ntp_snippets_status_service_unittest.cc", + "remote/ntp_snippets_test_utils.cc", + "remote/ntp_snippets_test_utils.h", + "remote/request_throttler_unittest.cc", + "sessions/foreign_sessions_suggestions_provider_unittest.cc", ] deps = [ ":ntp_snippets", "//base", "//base/test:test_support", + "//components/bookmarks/browser", + "//components/bookmarks/test", "//components/image_fetcher", "//components/leveldb_proto:test_support", + "//components/offline_pages", + "//components/offline_pages:test_support", + "//components/sessions", + "//components/sessions:test_support", "//components/signin/core/browser:test_support", "//components/signin/core/common", - "//components/sync_driver:test_support", + "//components/strings", + "//components/sync:test_support_sync_driver", + "//components/sync_sessions", "//components/variations", "//net:test_support", "//testing/gtest", "//third_party/icu/", + "//ui/gfx:test_support", ] } diff --git a/chromium/components/ntp_snippets/DEPS b/chromium/components/ntp_snippets/DEPS index 02c7f3b3a8c..51bcdffcecb 100644 --- a/chromium/components/ntp_snippets/DEPS +++ b/chromium/components/ntp_snippets/DEPS @@ -1,17 +1,19 @@ include_rules = [ "+components/data_use_measurement/core", + "+components/history/core", "+components/image_fetcher", "+components/keyed_service/core", "+components/leveldb_proto", "+components/metrics", "+components/prefs", "+components/signin", - "+components/suggestions", - "+components/sync_driver", + "+components/sync/driver", "+components/variations", + "+grit/components_strings.h", "+net/base", "+net/http", "+net/url_request", "+google_apis", + "+ui/base", "+ui/gfx", ] diff --git a/chromium/components/ntp_snippets/bookmarks/DEPS b/chromium/components/ntp_snippets/bookmarks/DEPS new file mode 100644 index 00000000000..fe6f7075507 --- /dev/null +++ b/chromium/components/ntp_snippets/bookmarks/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+components/bookmarks", +] diff --git a/chromium/components/ntp_snippets/bookmarks/bookmark_last_visit_utils.cc b/chromium/components/ntp_snippets/bookmarks/bookmark_last_visit_utils.cc new file mode 100644 index 00000000000..90aea5bf49b --- /dev/null +++ b/chromium/components/ntp_snippets/bookmarks/bookmark_last_visit_utils.cc @@ -0,0 +1,273 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/bookmarks/bookmark_last_visit_utils.h" + +#include <algorithm> +#include <set> +#include <string> +#include <utility> + +#include "base/bind.h" +#include "base/strings/string_number_conversions.h" +#include "base/time/time.h" +#include "components/bookmarks/browser/bookmark_model.h" +#include "components/bookmarks/browser/bookmark_node.h" +#include "url/gurl.h" + +using bookmarks::BookmarkModel; +using bookmarks::BookmarkNode; + +namespace ntp_snippets { + +namespace { + +struct RecentBookmark { + const bookmarks::BookmarkNode* node; + bool visited_recently; +}; + +const char* kBookmarksURLBlacklist[] = {"chrome://newtab/", + "chrome-native://newtab/", + "chrome://bookmarks/"}; + +const char kBookmarkLastVisitDateKey[] = "last_visited"; +const char kBookmarkDismissedFromNTP[] = "dismissed_from_ntp"; + +base::Time ParseLastVisitDate(const std::string& date_string) { + int64_t date = 0; + if (!base::StringToInt64(date_string, &date)) + return base::Time::UnixEpoch(); + return base::Time::FromInternalValue(date); +} + +std::string FormatLastVisitDate(const base::Time& date) { + return base::Int64ToString(date.ToInternalValue()); +} + +bool CompareBookmarksByLastVisitDate(const BookmarkNode* a, + const BookmarkNode* b, + bool creation_date_fallback) { + return GetLastVisitDateForBookmark(a, creation_date_fallback) > + GetLastVisitDateForBookmark(b, creation_date_fallback); +} + +bool IsBlacklisted(const GURL& url) { + for (const char* blacklisted : kBookmarksURLBlacklist) { + if (url.spec() == blacklisted) + return true; + } + return false; +} + +} // namespace + +void UpdateBookmarkOnURLVisitedInMainFrame(BookmarkModel* bookmark_model, + const GURL& url) { + // Skip URLs that are blacklisted. + if (IsBlacklisted(url)) + return; + + // Skip URLs that are not bookmarked. + std::vector<const BookmarkNode*> bookmarks_for_url; + bookmark_model->GetNodesByURL(url, &bookmarks_for_url); + if (bookmarks_for_url.empty()) + return; + + // If there are bookmarks for |url|, set their last visit date to now. + std::string now = FormatLastVisitDate(base::Time::Now()); + for (const BookmarkNode* node : bookmarks_for_url) { + bookmark_model->SetNodeMetaInfo(node, kBookmarkLastVisitDateKey, now); + // If the bookmark has been dismissed from NTP before, a new visit overrides + // such a dismissal. + bookmark_model->DeleteNodeMetaInfo(node, kBookmarkDismissedFromNTP); + } +} + +base::Time GetLastVisitDateForBookmark(const BookmarkNode* node, + bool creation_date_fallback) { + if (!node) + return base::Time::UnixEpoch(); + + std::string last_visit_date_string; + if (!node->GetMetaInfo(kBookmarkLastVisitDateKey, &last_visit_date_string) && + creation_date_fallback) { + return node->date_added(); + } + return ParseLastVisitDate(last_visit_date_string); +} + +base::Time GetLastVisitDateForBookmarkIfNotDismissed( + const BookmarkNode* node, + bool creation_date_fallback) { + if (IsDismissedFromNTPForBookmark(node)) + return base::Time::UnixEpoch(); + + return GetLastVisitDateForBookmark(node, creation_date_fallback); +} + +void MarkBookmarksDismissed(BookmarkModel* bookmark_model, const GURL& url) { + std::vector<const BookmarkNode*> nodes; + bookmark_model->GetNodesByURL(url, &nodes); + for (const BookmarkNode* node : nodes) + bookmark_model->SetNodeMetaInfo(node, kBookmarkDismissedFromNTP, "1"); +} + +bool IsDismissedFromNTPForBookmark(const BookmarkNode* node) { + if (!node) + return false; + + std::string dismissed_from_ntp; + bool result = + node->GetMetaInfo(kBookmarkDismissedFromNTP, &dismissed_from_ntp); + DCHECK(!result || dismissed_from_ntp == "1"); + return result; +} + +void MarkAllBookmarksUndismissed(BookmarkModel* bookmark_model) { + // Get all the bookmark URLs. + std::vector<BookmarkModel::URLAndTitle> bookmarks; + bookmark_model->GetBookmarks(&bookmarks); + + // Remove dismissed flag from all bookmarks + for (const BookmarkModel::URLAndTitle& bookmark : bookmarks) { + std::vector<const BookmarkNode*> nodes; + bookmark_model->GetNodesByURL(bookmark.url, &nodes); + for (const BookmarkNode* node : nodes) + bookmark_model->DeleteNodeMetaInfo(node, kBookmarkDismissedFromNTP); + } +} + +std::vector<const BookmarkNode*> GetRecentlyVisitedBookmarks( + BookmarkModel* bookmark_model, + int min_count, + int max_count, + const base::Time& min_visit_time, + bool creation_date_fallback) { + // Get all the bookmark URLs. + std::vector<BookmarkModel::URLAndTitle> bookmark_urls; + bookmark_model->GetBookmarks(&bookmark_urls); + + std::vector<RecentBookmark> bookmarks; + int recently_visited_count = 0; + // Find for each bookmark the most recently visited BookmarkNode and find out + // whether it is visited since |min_visit_time|. + for (const BookmarkModel::URLAndTitle& url_and_title : bookmark_urls) { + // Skip URLs that are blacklisted. + if (IsBlacklisted(url_and_title.url)) + continue; + + // Get all bookmarks for the given URL. + std::vector<const BookmarkNode*> bookmarks_for_url; + bookmark_model->GetNodesByURL(url_and_title.url, &bookmarks_for_url); + DCHECK(!bookmarks_for_url.empty()); + + // Find the most recent node (minimal w.r.t. + // CompareBookmarksByLastVisitDate). + const BookmarkNode* most_recent = *std::min_element( + bookmarks_for_url.begin(), bookmarks_for_url.end(), + [creation_date_fallback](const BookmarkNode* a, const BookmarkNode* b) { + return CompareBookmarksByLastVisitDate(a, b, creation_date_fallback); + }); + + // Find out if it has been _visited_ recently enough. + bool visited_recently = + min_visit_time < GetLastVisitDateForBookmark( + most_recent, /*creation_date_fallback=*/false); + if (visited_recently) + recently_visited_count++; + if (!IsDismissedFromNTPForBookmark(most_recent)) + bookmarks.push_back({most_recent, visited_recently}); + } + + if (recently_visited_count < min_count) { + // There aren't enough recently-visited bookmarks. Fill the list up to + // |min_count| with older bookmarks (in particular those with only a + // creation date, if creation_date_fallback is true). + max_count = min_count; + } else { + // Remove the bookmarks that are not recently visited; we do not need them. + // (We might end up with fewer than |min_count| bookmarks if all the recent + // ones are dismissed.) + bookmarks.erase( + std::remove_if(bookmarks.begin(), bookmarks.end(), + [](const RecentBookmark& bookmark) { + return !bookmark.visited_recently; + }), + bookmarks.end()); + } + + // Sort the remaining entries by date. + std::sort(bookmarks.begin(), bookmarks.end(), + [creation_date_fallback](const RecentBookmark& a, + const RecentBookmark& b) { + return CompareBookmarksByLastVisitDate(a.node, b.node, + creation_date_fallback); + }); + + // Insert the first |max_count| items from |bookmarks| into |result|. + std::vector<const BookmarkNode*> result; + for (const RecentBookmark& bookmark : bookmarks) { + if (!creation_date_fallback && + GetLastVisitDateForBookmark(bookmark.node, creation_date_fallback) < + min_visit_time) { + break; + } + + result.push_back(bookmark.node); + if (result.size() >= static_cast<size_t>(max_count)) + break; + } + return result; +} + +std::vector<const BookmarkNode*> GetDismissedBookmarksForDebugging( + BookmarkModel* bookmark_model) { + // Get all the bookmark URLs. + std::vector<BookmarkModel::URLAndTitle> bookmarks; + bookmark_model->GetBookmarks(&bookmarks); + + // Remove the bookmark URLs which have at least one non-dismissed bookmark. + bookmarks.erase( + std::remove_if( + bookmarks.begin(), bookmarks.end(), + [&bookmark_model](const BookmarkModel::URLAndTitle& bookmark) { + std::vector<const BookmarkNode*> bookmarks_for_url; + bookmark_model->GetNodesByURL(bookmark.url, &bookmarks_for_url); + DCHECK(!bookmarks_for_url.empty()); + + for (const BookmarkNode* node : bookmarks_for_url) { + if (!IsDismissedFromNTPForBookmark(node)) + return true; + } + return false; + }), + bookmarks.end()); + + // Insert into |result|. + std::vector<const BookmarkNode*> result; + for (const BookmarkModel::URLAndTitle& bookmark : bookmarks) { + result.push_back( + bookmark_model->GetMostRecentlyAddedUserNodeForURL(bookmark.url)); + } + return result; +} + +void RemoveAllLastVisitDates(bookmarks::BookmarkModel* bookmark_model) { + // Get all the bookmark URLs. + std::vector<BookmarkModel::URLAndTitle> bookmark_urls; + bookmark_model->GetBookmarks(&bookmark_urls); + + for (const BookmarkModel::URLAndTitle& url_and_title : bookmark_urls) { + // Get all bookmarks for the given URL. + std::vector<const BookmarkNode*> bookmarks_for_url; + bookmark_model->GetNodesByURL(url_and_title.url, &bookmarks_for_url); + + for (const BookmarkNode* bookmark : bookmarks_for_url) { + bookmark_model->DeleteNodeMetaInfo(bookmark, kBookmarkLastVisitDateKey); + } + } +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/bookmarks/bookmark_last_visit_utils.h b/chromium/components/ntp_snippets/bookmarks/bookmark_last_visit_utils.h new file mode 100644 index 00000000000..b056deaeb30 --- /dev/null +++ b/chromium/components/ntp_snippets/bookmarks/bookmark_last_visit_utils.h @@ -0,0 +1,74 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_BOOKMARKS_BOOKMARK_LAST_VISIT_UTILS_H_ +#define COMPONENTS_NTP_SNIPPETS_BOOKMARKS_BOOKMARK_LAST_VISIT_UTILS_H_ + +#include <vector> + +class GURL; + +namespace base { +class Time; +} // namespace base + +namespace bookmarks { +class BookmarkModel; +class BookmarkNode; +} // namespace bookmarks + +namespace ntp_snippets { + +// If there is a bookmark for |url|, this function updates its last visit date +// to now. If there are multiple bookmarks for a given URL, it updates all of +// them. +void UpdateBookmarkOnURLVisitedInMainFrame( + bookmarks::BookmarkModel* bookmark_model, + const GURL& url); + +// Gets the last visit date for a given bookmark |node|. The visit when the +// bookmark is created also counts. If no info about last visit date is present +// and |creation_date_fallback| is true, creation date is used. +base::Time GetLastVisitDateForBookmark(const bookmarks::BookmarkNode* node, + bool creation_date_fallback); + +// Like GetLastVisitDateForBookmark, but it returns the unix epoch if the +// bookmark is dismissed from NTP. +base::Time GetLastVisitDateForBookmarkIfNotDismissed( + const bookmarks::BookmarkNode* node, + bool creation_date_fallback); + +// Marks all bookmarks with the given URL as dismissed. +void MarkBookmarksDismissed(bookmarks::BookmarkModel* bookmark_model, + const GURL& url); + +// Gets the dismissed flag for a given bookmark |node|. Defaults to false. +bool IsDismissedFromNTPForBookmark(const bookmarks::BookmarkNode* node); + +// Removes the dismissed flag from all bookmarks (only for debugging). +void MarkAllBookmarksUndismissed(bookmarks::BookmarkModel* bookmark_model); + +// Returns the list of most recently visited, non-dismissed bookmarks. +// For each bookmarked URL, it returns the most recently created bookmark. +// The result is ordered by visit time (the most recent first). Only bookmarks +// visited after |min_visit_time| are considered, at most |max_count| bookmarks +// are returned. If this results into less than |min_count| bookmarks, the list +// is filled up with older bookmarks sorted by their last visit / creation date. +std::vector<const bookmarks::BookmarkNode*> GetRecentlyVisitedBookmarks( + bookmarks::BookmarkModel* bookmark_model, + int min_count, + int max_count, + const base::Time& min_visit_time, + bool creation_date_fallback); + +// Returns the list of all dismissed bookmarks. Only used for debugging. +std::vector<const bookmarks::BookmarkNode*> GetDismissedBookmarksForDebugging( + bookmarks::BookmarkModel* bookmark_model); + +// Removes last visited date metadata for all bookmarks. +void RemoveAllLastVisitDates(bookmarks::BookmarkModel* bookmark_model); + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_BOOKMARKS_BOOKMARK_LAST_VISIT_UTILS_H_ diff --git a/chromium/components/ntp_snippets/bookmarks/bookmark_last_visit_utils_unittest.cc b/chromium/components/ntp_snippets/bookmarks/bookmark_last_visit_utils_unittest.cc new file mode 100644 index 00000000000..45e595d56ea --- /dev/null +++ b/chromium/components/ntp_snippets/bookmarks/bookmark_last_visit_utils_unittest.cc @@ -0,0 +1,122 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/bookmarks/bookmark_last_visit_utils.h" + +#include <string> + +#include "base/strings/string_number_conversions.h" +#include "base/strings/stringprintf.h" +#include "base/strings/utf_string_conversions.h" +#include "components/bookmarks/browser/bookmark_model.h" +#include "components/bookmarks/browser/bookmark_node.h" +#include "components/bookmarks/test/test_bookmark_client.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "url/gurl.h" + +using bookmarks::BookmarkModel; +using bookmarks::BookmarkNode; + +using testing::IsEmpty; +using testing::SizeIs; + +namespace ntp_snippets { + +namespace { + +const char kBookmarkLastVisitDateKey[] = "last_visited"; + +std::unique_ptr<BookmarkModel> CreateModelWithRecentBookmarks( + int number_of_bookmarks, + int number_of_recent, + const base::Time& threshold_time) { + std::unique_ptr<BookmarkModel> model = + bookmarks::TestBookmarkClient::CreateModel(); + + base::TimeDelta week = base::TimeDelta::FromDays(7); + base::Time recent_time = threshold_time + week; + std::string recent_time_string = + base::Int64ToString(recent_time.ToInternalValue()); + base::Time nonrecent_time = threshold_time - week; + std::string nonrecent_time_string = + base::Int64ToString(nonrecent_time.ToInternalValue()); + + for (int index = 0; index < number_of_bookmarks; ++index) { + base::string16 title = + base::ASCIIToUTF16(base::StringPrintf("title%d", index)); + GURL url(base::StringPrintf("http://url%d.com", index)); + const BookmarkNode* node = + model->AddURL(model->bookmark_bar_node(), index, title, url); + + model->SetNodeMetaInfo( + node, kBookmarkLastVisitDateKey, + index < number_of_recent ? recent_time_string : nonrecent_time_string); + } + + return model; +} + +} // namespace + +class GetRecentlyVisitedBookmarksTest : public testing::Test { + public: + GetRecentlyVisitedBookmarksTest() { + base::TimeDelta week = base::TimeDelta::FromDays(7); + threshold_time_ = base::Time::UnixEpoch() + 52 * week; + } + + const base::Time& threshold_time() const { return threshold_time_; } + + private: + base::Time threshold_time_; + + DISALLOW_COPY_AND_ASSIGN(GetRecentlyVisitedBookmarksTest); +}; + +TEST_F(GetRecentlyVisitedBookmarksTest, + WithoutDateFallbackShouldNotReturnNonRecent) { + const int number_of_recent = 0; + const int number_of_bookmarks = 3; + std::unique_ptr<BookmarkModel> model = CreateModelWithRecentBookmarks( + number_of_bookmarks, number_of_recent, threshold_time()); + + std::vector<const bookmarks::BookmarkNode*> result = + GetRecentlyVisitedBookmarks(model.get(), 0, number_of_bookmarks, + threshold_time(), + /*creation_date_fallback=*/false); + EXPECT_THAT(result, IsEmpty()); +} + +TEST_F(GetRecentlyVisitedBookmarksTest, + WithDateFallbackShouldReturnNonRecentUpToMinCount) { + const int number_of_recent = 0; + const int number_of_bookmarks = 3; + std::unique_ptr<BookmarkModel> model = CreateModelWithRecentBookmarks( + number_of_bookmarks, number_of_recent, threshold_time()); + + const int min_count = number_of_bookmarks - 1; + const int max_count = min_count + 10; + std::vector<const bookmarks::BookmarkNode*> result = + GetRecentlyVisitedBookmarks(model.get(), min_count, max_count, + threshold_time(), + /*creation_date_fallback=*/true); + EXPECT_THAT(result, SizeIs(min_count)); +} + +TEST_F(GetRecentlyVisitedBookmarksTest, ShouldReturnNotMoreThanMaxCount) { + const int number_of_recent = 3; + const int number_of_bookmarks = number_of_recent; + std::unique_ptr<BookmarkModel> model = CreateModelWithRecentBookmarks( + number_of_bookmarks, number_of_recent, threshold_time()); + + const int max_count = number_of_recent - 1; + std::vector<const bookmarks::BookmarkNode*> result = + GetRecentlyVisitedBookmarks(model.get(), max_count, max_count, + threshold_time(), + /*creation_date_fallback=*/false); + EXPECT_THAT(result, SizeIs(max_count)); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/bookmarks/bookmark_suggestions_provider.cc b/chromium/components/ntp_snippets/bookmarks/bookmark_suggestions_provider.cc new file mode 100644 index 00000000000..b5020910f0c --- /dev/null +++ b/chromium/components/ntp_snippets/bookmarks/bookmark_suggestions_provider.cc @@ -0,0 +1,327 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/bookmarks/bookmark_suggestions_provider.h" + +#include <utility> +#include <vector> + +#include "base/bind.h" +#include "base/location.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/utf_string_conversions.h" +#include "base/threading/thread_task_runner_handle.h" +#include "base/time/time.h" +#include "components/bookmarks/browser/bookmark_model.h" +#include "components/ntp_snippets/bookmarks/bookmark_last_visit_utils.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/content_suggestion.h" +#include "components/ntp_snippets/features.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" +#include "components/variations/variations_associated_data.h" +#include "grit/components_strings.h" +#include "ui/base/l10n/l10n_util.h" +#include "ui/gfx/image/image.h" + +using bookmarks::BookmarkModel; +using bookmarks::BookmarkNode; + +namespace ntp_snippets { + +namespace { + +const int kMaxBookmarks = 10; +const int kMinBookmarks = 3; +const int kMaxBookmarkAgeInDays = 42; +const int kUseCreationDateFallbackForDays = 0; + +const char* kMaxBookmarksParamName = "bookmarks_max_count"; +const char* kMinBookmarksParamName = "bookmarks_min_count"; +const char* kMaxBookmarkAgeInDaysParamName = "bookmarks_max_age_in_days"; +const char* kUseCreationDateFallbackForDaysParamName = + "bookmarks_creation_date_fallback_days"; +const char* kShowIfEmptyParamName = "bookmarks_show_if_empty"; + +// Any bookmark created or visited after this time will be considered recent. +// Note that bookmarks can be shown that do not meet this threshold. +base::Time GetThresholdTime() { + return base::Time::Now() - + base::TimeDelta::FromDays(GetParamAsInt( + ntp_snippets::kBookmarkSuggestionsFeature, + kMaxBookmarkAgeInDaysParamName, kMaxBookmarkAgeInDays)); +} + +// The number of days used as a threshold where if this is larger than the time +// since M54 started, then the creation timestamp of a bookmark be used as a +// fallback if no last visited timestamp is present. +int UseCreationDateFallbackForDays() { + return GetParamAsInt(ntp_snippets::kBookmarkSuggestionsFeature, + kUseCreationDateFallbackForDaysParamName, + kUseCreationDateFallbackForDays); +} + +// The maximum number of suggestions ever provided. +int GetMaxCount() { + return GetParamAsInt(ntp_snippets::kBookmarkSuggestionsFeature, + kMaxBookmarksParamName, kMaxBookmarks); +} + +// The minimum number of suggestions to try to provide. Depending on other +// parameters this may or not be respected. Currently creation date fallback +// must be active in order for older bookmarks to be incorporated to meet this +// min. +int GetMinCount() { + return GetParamAsInt(ntp_snippets::kBookmarkSuggestionsFeature, + kMinBookmarksParamName, kMinBookmarks); +} + +bool ShouldShowIfEmpty() { + std::string show_if_empty = variations::GetVariationParamValueByFeature( + ntp_snippets::kBookmarkSuggestionsFeature, kShowIfEmptyParamName); + if (show_if_empty.empty() || show_if_empty == "false") + return false; + if (show_if_empty == "true") + return true; + + LOG(WARNING) << "Failed to parse show if empty " << show_if_empty; + return false; +} + +} // namespace + +BookmarkSuggestionsProvider::BookmarkSuggestionsProvider( + ContentSuggestionsProvider::Observer* observer, + CategoryFactory* category_factory, + bookmarks::BookmarkModel* bookmark_model, + PrefService* pref_service) + : ContentSuggestionsProvider(observer, category_factory), + category_status_(CategoryStatus::AVAILABLE_LOADING), + provided_category_( + category_factory->FromKnownCategory(KnownCategories::BOOKMARKS)), + bookmark_model_(bookmark_model), + fetch_requested_(false), + end_of_list_last_visit_date_(GetThresholdTime()) { + observer->OnCategoryStatusChanged(this, provided_category_, category_status_); + base::Time first_m54_start; + base::Time now = base::Time::Now(); + if (pref_service->HasPrefPath(prefs::kBookmarksFirstM54Start)) { + first_m54_start = base::Time::FromInternalValue( + pref_service->GetInt64(prefs::kBookmarksFirstM54Start)); + } else { + first_m54_start = now; + pref_service->SetInt64(prefs::kBookmarksFirstM54Start, + first_m54_start.ToInternalValue()); + } + base::TimeDelta time_since_first_m54_start = now - first_m54_start; + // Note: Setting the fallback timeout to zero effectively turns off the + // fallback entirely. + creation_date_fallback_ = + time_since_first_m54_start.InDays() < UseCreationDateFallbackForDays(); + bookmark_model_->AddObserver(this); + FetchBookmarks(); +} + +BookmarkSuggestionsProvider::~BookmarkSuggestionsProvider() { + bookmark_model_->RemoveObserver(this); +} + +// static +void BookmarkSuggestionsProvider::RegisterProfilePrefs( + PrefRegistrySimple* registry) { + registry->RegisterInt64Pref(prefs::kBookmarksFirstM54Start, 0); +} + +//////////////////////////////////////////////////////////////////////////////// +// Private methods + +CategoryStatus BookmarkSuggestionsProvider::GetCategoryStatus( + Category category) { + DCHECK_EQ(category, provided_category_); + return category_status_; +} + +CategoryInfo BookmarkSuggestionsProvider::GetCategoryInfo(Category category) { + return CategoryInfo( + l10n_util::GetStringUTF16(IDS_NTP_BOOKMARK_SUGGESTIONS_SECTION_HEADER), + ContentSuggestionsCardLayout::MINIMAL_CARD, + /*has_more_button=*/true, + /*show_if_empty=*/ShouldShowIfEmpty() && bookmark_model_->HasBookmarks()); + // TODO(treib): Setting show_if_empty to true is a temporary hack, see + // crbug.com/640568. +} + +void BookmarkSuggestionsProvider::DismissSuggestion( + const ContentSuggestion::ID& suggestion_id) { + DCHECK(bookmark_model_->loaded()); + GURL url(suggestion_id.id_within_category()); + MarkBookmarksDismissed(bookmark_model_, url); +} + +void BookmarkSuggestionsProvider::FetchSuggestionImage( + const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) { + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::Bind(callback, gfx::Image())); +} + +void BookmarkSuggestionsProvider::ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) { + // TODO(vitaliii): Do not remove all dates, but only the ones matched by the + // time range and the filter. + RemoveAllLastVisitDates(bookmark_model_); + ClearDismissedSuggestionsForDebugging(provided_category_); + FetchBookmarks(); + // Temporarily enter an "explicitly disabled" state, so that any open UIs + // will clear the suggestions too. + if (category_status_ != CategoryStatus::CATEGORY_EXPLICITLY_DISABLED) { + CategoryStatus old_category_status = category_status_; + NotifyStatusChanged(CategoryStatus::CATEGORY_EXPLICITLY_DISABLED); + NotifyStatusChanged(old_category_status); + } +} + +void BookmarkSuggestionsProvider::ClearCachedSuggestions(Category category) { + DCHECK_EQ(category, provided_category_); + // Ignored. +} + +void BookmarkSuggestionsProvider::GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) { + DCHECK_EQ(category, provided_category_); + std::vector<const BookmarkNode*> bookmarks = + GetDismissedBookmarksForDebugging(bookmark_model_); + + std::vector<ContentSuggestion> suggestions; + for (const BookmarkNode* bookmark : bookmarks) + suggestions.emplace_back(ConvertBookmark(bookmark)); + callback.Run(std::move(suggestions)); +} + +void BookmarkSuggestionsProvider::ClearDismissedSuggestionsForDebugging( + Category category) { + DCHECK_EQ(category, provided_category_); + if (!bookmark_model_->loaded()) + return; + MarkAllBookmarksUndismissed(bookmark_model_); +} + +void BookmarkSuggestionsProvider::BookmarkModelLoaded( + bookmarks::BookmarkModel* model, + bool ids_reassigned) { + DCHECK_EQ(bookmark_model_, model); + if (fetch_requested_) { + fetch_requested_ = false; + FetchBookmarksInternal(); + } +} + +void BookmarkSuggestionsProvider::OnWillChangeBookmarkMetaInfo( + BookmarkModel* model, + const BookmarkNode* node) { + // Store the last visit date of the node that is about to change. + node_to_change_last_visit_date_ = + GetLastVisitDateForBookmarkIfNotDismissed(node, creation_date_fallback_); +} + +void BookmarkSuggestionsProvider::BookmarkMetaInfoChanged( + BookmarkModel* model, + const BookmarkNode* node) { + base::Time time = + GetLastVisitDateForBookmarkIfNotDismissed(node, creation_date_fallback_); + if (time == node_to_change_last_visit_date_ || + time < end_of_list_last_visit_date_) + return; + + // Last visit date of a node has changed (and is relevant for the list), we + // should update the suggestions. + FetchBookmarks(); +} + +void BookmarkSuggestionsProvider::BookmarkNodeRemoved( + bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* parent, + int old_index, + const bookmarks::BookmarkNode* node, + const std::set<GURL>& no_longer_bookmarked) { + if (GetLastVisitDateForBookmarkIfNotDismissed(node, creation_date_fallback_) < + end_of_list_last_visit_date_) { + return; + } + + // Some node from our list got deleted, we should update the suggestions. + FetchBookmarks(); +} + +void BookmarkSuggestionsProvider::BookmarkNodeAdded( + bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* parent, + int index) { + if (GetLastVisitDateForBookmarkIfNotDismissed(parent->GetChild(index), + creation_date_fallback_) < + end_of_list_last_visit_date_) { + return; + } + + // Some node with last_visit info that is relevant for our list got created + // (e.g. by sync), we should update the suggestions. + FetchBookmarks(); +} + +ContentSuggestion BookmarkSuggestionsProvider::ConvertBookmark( + const BookmarkNode* bookmark) { + ContentSuggestion suggestion(provided_category_, bookmark->url().spec(), + bookmark->url()); + + suggestion.set_title(bookmark->GetTitle()); + suggestion.set_snippet_text(base::string16()); + suggestion.set_publish_date( + GetLastVisitDateForBookmark(bookmark, creation_date_fallback_)); + suggestion.set_publisher_name(base::UTF8ToUTF16(bookmark->url().host())); + return suggestion; +} + +void BookmarkSuggestionsProvider::FetchBookmarksInternal() { + DCHECK(bookmark_model_->loaded()); + + NotifyStatusChanged(CategoryStatus::AVAILABLE); + + base::Time threshold_time = GetThresholdTime(); + std::vector<const BookmarkNode*> bookmarks = + GetRecentlyVisitedBookmarks(bookmark_model_, GetMinCount(), GetMaxCount(), + threshold_time, creation_date_fallback_); + + std::vector<ContentSuggestion> suggestions; + for (const BookmarkNode* bookmark : bookmarks) + suggestions.emplace_back(ConvertBookmark(bookmark)); + + if (suggestions.empty()) + end_of_list_last_visit_date_ = threshold_time; + else + end_of_list_last_visit_date_ = suggestions.back().publish_date(); + + observer()->OnNewSuggestions(this, provided_category_, + std::move(suggestions)); +} + +void BookmarkSuggestionsProvider::FetchBookmarks() { + if (bookmark_model_->loaded()) + FetchBookmarksInternal(); + else + fetch_requested_ = true; +} + +void BookmarkSuggestionsProvider::NotifyStatusChanged( + CategoryStatus new_status) { + if (category_status_ == new_status) + return; + category_status_ = new_status; + observer()->OnCategoryStatusChanged(this, provided_category_, new_status); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/bookmarks/bookmark_suggestions_provider.h b/chromium/components/ntp_snippets/bookmarks/bookmark_suggestions_provider.h new file mode 100644 index 00000000000..2bf8ebb0fe2 --- /dev/null +++ b/chromium/components/ntp_snippets/bookmarks/bookmark_suggestions_provider.h @@ -0,0 +1,122 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_BOOKMARKS_BOOKMARK_SUGGESTIONS_PROVIDER_H_ +#define COMPONENTS_NTP_SNIPPETS_BOOKMARKS_BOOKMARK_SUGGESTIONS_PROVIDER_H_ + +#include <set> +#include <string> + +#include "components/bookmarks/browser/bookmark_model_observer.h" +#include "components/keyed_service/core/keyed_service.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_status.h" +#include "components/ntp_snippets/content_suggestions_provider.h" + +class PrefRegistrySimple; +class PrefService; + +namespace gfx { +class Image; +} // namespace gfx + +namespace ntp_snippets { + +// Provides content suggestions from the bookmarks model. +class BookmarkSuggestionsProvider : public ContentSuggestionsProvider, + public bookmarks::BookmarkModelObserver { + public: + BookmarkSuggestionsProvider(ContentSuggestionsProvider::Observer* observer, + CategoryFactory* category_factory, + bookmarks::BookmarkModel* bookmark_model, + PrefService* pref_service); + ~BookmarkSuggestionsProvider() override; + + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + private: + // ContentSuggestionsProvider implementation. + CategoryStatus GetCategoryStatus(Category category) override; + CategoryInfo GetCategoryInfo(Category category) override; + void DismissSuggestion(const ContentSuggestion::ID& suggestion_id) override; + void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) override; + void ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) override; + void ClearCachedSuggestions(Category category) override; + void GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) override; + void ClearDismissedSuggestionsForDebugging(Category category) override; + + // bookmarks::BookmarkModelObserver implementation. + void BookmarkModelLoaded(bookmarks::BookmarkModel* model, + bool ids_reassigned) override; + void OnWillChangeBookmarkMetaInfo( + bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* node) override; + void BookmarkMetaInfoChanged(bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* node) override; + + void BookmarkNodeMoved(bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* old_parent, + int old_index, + const bookmarks::BookmarkNode* new_parent, + int new_index) override {} + void BookmarkNodeAdded(bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* parent, + int index) override; + void BookmarkNodeRemoved( + bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* parent, + int old_index, + const bookmarks::BookmarkNode* node, + const std::set<GURL>& no_longer_bookmarked) override; + void BookmarkNodeChanged(bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* node) override {} + void BookmarkNodeFaviconChanged( + bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* node) override {} + void BookmarkNodeChildrenReordered( + bookmarks::BookmarkModel* model, + const bookmarks::BookmarkNode* node) override {} + void BookmarkAllUserNodesRemoved( + bookmarks::BookmarkModel* model, + const std::set<GURL>& removed_urls) override {} + + ContentSuggestion ConvertBookmark(const bookmarks::BookmarkNode* bookmark); + + // The actual method to fetch bookmarks - follows each call to FetchBookmarks + // but not sooner than the BookmarkModel gets loaded. + void FetchBookmarksInternal(); + + // Queries the BookmarkModel for recently visited bookmarks and pushes the + // results to the ContentSuggestionService. The actual fetching does not + // happen before the Bookmark model gets loaded. + void FetchBookmarks(); + + // Updates the |category_status_| and notifies the |observer_|, if necessary. + void NotifyStatusChanged(CategoryStatus new_status); + + CategoryStatus category_status_; + const Category provided_category_; + bookmarks::BookmarkModel* bookmark_model_; + bool fetch_requested_; + + base::Time node_to_change_last_visit_date_; + base::Time end_of_list_last_visit_date_; + + // TODO(jkrcal): Remove this field and the pref after M55. + // For six weeks after first installing M54, this is true and the + // fallback implemented in BookmarkLastVisitUtils is activated. + bool creation_date_fallback_; + + DISALLOW_COPY_AND_ASSIGN(BookmarkSuggestionsProvider); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_BOOKMARKS_BOOKMARK_SUGGESTIONS_PROVIDER_H_ diff --git a/chromium/components/ntp_snippets/category.cc b/chromium/components/ntp_snippets/category.cc new file mode 100644 index 00000000000..abc86ab9f2c --- /dev/null +++ b/chromium/components/ntp_snippets/category.cc @@ -0,0 +1,37 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/category.h" + +#include "base/logging.h" + +namespace ntp_snippets { + +Category::Category(int id) : id_(id) {} + +bool Category::IsKnownCategory(KnownCategories known_category) const { + DCHECK_NE(known_category, KnownCategories::LOCAL_CATEGORIES_COUNT); + DCHECK_NE(known_category, KnownCategories::REMOTE_CATEGORIES_OFFSET); + return id_ == static_cast<int>(known_category); +} + +bool operator==(const Category& left, const Category& right) { + return left.id() == right.id(); +} + +bool operator!=(const Category& left, const Category& right) { + return !(left == right); +} + +bool Category::CompareByID::operator()(const Category& left, + const Category& right) const { + return left.id() < right.id(); +} + +std::ostream& operator<<(std::ostream& os, const Category& obj) { + os << obj.id(); + return os; +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/category.h b/chromium/components/ntp_snippets/category.h new file mode 100644 index 00000000000..0220cb54e97 --- /dev/null +++ b/chromium/components/ntp_snippets/category.h @@ -0,0 +1,89 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_CATEGORY_H_ +#define COMPONENTS_NTP_SNIPPETS_CATEGORY_H_ + +#include <ostream> + +namespace ntp_snippets { + +class CategoryFactory; + +// These are the categories that the client knows about. +// The values before LOCAL_CATEGORIES_COUNT are the categories that are provided +// locally on the device. Categories provided by the server (IDs strictly larger +// than REMOTE_CATEGORIES_OFFSET) only need to be hard-coded here if they need +// to be recognized by the client implementation. +// NOTE: These are persisted, so don't reorder or remove values, and insert new +// values only in the appropriate places marked below. +// On Android builds, a Java counterpart will be generated for this enum. +// GENERATED_JAVA_ENUM_PACKAGE: org.chromium.chrome.browser.ntp.snippets +enum class KnownCategories { + // Pages recently downloaded during normal navigation. + RECENT_TABS, + + // Pages downloaded by the user for offline consumption. + DOWNLOADS, + + // Recently used bookmarks. + BOOKMARKS, + + // Physical Web page available in the vicinity. + PHYSICAL_WEB_PAGES, + + // Pages recently browsed to on other devices. + FOREIGN_TABS, + // INSERT NEW LOCAL CATEGORIES HERE! + + // Follows the last local category. + LOCAL_CATEGORIES_COUNT, + + // Remote categories come after this. + REMOTE_CATEGORIES_OFFSET = 10000, + + // Articles for you. + ARTICLES, + // INSERT NEW REMOTE CATEGORIES HERE! +}; + +// A category groups ContentSuggestions which belong together. Use the +// CategoryFactory to obtain instances. +class Category { + public: + // An arbitrary but consistent ordering. Can be used to look up categories in + // a std::map, but should not be used to order categories for other purposes. + struct CompareByID; + + // Returns a non-negative identifier that is unique for the category and can + // be converted back to a Category instance using + // |CategoryFactory::FromIDValue(id)|. + int id() const { return id_; } + + // Returns whether this category matches the given |known_category|. + bool IsKnownCategory(KnownCategories known_category) const; + + private: + friend class CategoryFactory; + + explicit Category(int id); + + int id_; + + // Allow copy and assignment. +}; + +bool operator==(const Category& left, const Category& right); + +bool operator!=(const Category& left, const Category& right); + +struct Category::CompareByID { + bool operator()(const Category& left, const Category& right) const; +}; + +std::ostream& operator<<(std::ostream& os, const Category& obj); + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_CATEGORY_H_ diff --git a/chromium/components/ntp_snippets/category_factory.cc b/chromium/components/ntp_snippets/category_factory.cc new file mode 100644 index 00000000000..4da984e0162 --- /dev/null +++ b/chromium/components/ntp_snippets/category_factory.cc @@ -0,0 +1,85 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/category_factory.h" + +#include <algorithm> + +#include "base/logging.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/stringprintf.h" + +namespace ntp_snippets { + +CategoryFactory::CategoryFactory() { + // Add all local categories in a fixed order. + AddKnownCategory(KnownCategories::DOWNLOADS); + AddKnownCategory(KnownCategories::RECENT_TABS); + AddKnownCategory(KnownCategories::FOREIGN_TABS); + AddKnownCategory(KnownCategories::BOOKMARKS); + AddKnownCategory(KnownCategories::PHYSICAL_WEB_PAGES); + + DCHECK_EQ(static_cast<size_t>(KnownCategories::LOCAL_CATEGORIES_COUNT), + ordered_categories_.size()); +} + +CategoryFactory::~CategoryFactory() = default; + +Category CategoryFactory::FromKnownCategory(KnownCategories known_category) { + if (known_category < KnownCategories::LOCAL_CATEGORIES_COUNT) { + // Local categories should have been added already. + DCHECK(CategoryExists(static_cast<int>(known_category))); + } else { + DCHECK_GT(known_category, KnownCategories::REMOTE_CATEGORIES_OFFSET); + } + return InternalFromID(static_cast<int>(known_category)); +} + +Category CategoryFactory::FromRemoteCategory(int remote_category) { + DCHECK_GT(remote_category, 0); + return InternalFromID( + static_cast<int>(KnownCategories::REMOTE_CATEGORIES_OFFSET) + + remote_category); +} + +Category CategoryFactory::FromIDValue(int id) { + DCHECK_GE(id, 0); + DCHECK(id < static_cast<int>(KnownCategories::LOCAL_CATEGORIES_COUNT) || + id > static_cast<int>(KnownCategories::REMOTE_CATEGORIES_OFFSET)); + return InternalFromID(id); +} + +bool CategoryFactory::CompareCategories(const Category& left, + const Category& right) const { + if (left == right) + return false; + return std::find(ordered_categories_.begin(), ordered_categories_.end(), + left) < std::find(ordered_categories_.begin(), + ordered_categories_.end(), right); +} + +//////////////////////////////////////////////////////////////////////////////// +// Private methods + +bool CategoryFactory::CategoryExists(int id) { + return std::find(ordered_categories_.begin(), ordered_categories_.end(), + Category(id)) != ordered_categories_.end(); +} + +void CategoryFactory::AddKnownCategory(KnownCategories known_category) { + InternalFromID(static_cast<int>(known_category)); +} + +Category CategoryFactory::InternalFromID(int id) { + auto it = std::find(ordered_categories_.begin(), ordered_categories_.end(), + Category(id)); + if (it != ordered_categories_.end()) + return *it; + + Category category(id); + ordered_categories_.push_back(category); + return category; +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/category_factory.h b/chromium/components/ntp_snippets/category_factory.h new file mode 100644 index 00000000000..c3941eaea2f --- /dev/null +++ b/chromium/components/ntp_snippets/category_factory.h @@ -0,0 +1,61 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_CATEGORY_FACTORY_H_ +#define COMPONENTS_NTP_SNIPPETS_CATEGORY_FACTORY_H_ + +#include <map> +#include <string> +#include <vector> + +#include "base/macros.h" +#include "components/ntp_snippets/category.h" + +namespace ntp_snippets { + +// Creates and orders Category instances. +class CategoryFactory { + public: + CategoryFactory(); + ~CategoryFactory(); + + // Creates a category from a KnownCategory value. The passed |known_category| + // must not be one of the special values (LOCAL_CATEGORIES_COUNT or + // REMOTE_CATEGORIES_OFFSET). + Category FromKnownCategory(KnownCategories known_category); + + // Creates a category from a category identifier delivered by the server. + // |remote_category| must be positive. + // Note that remote categories are ordered in the order in which they were + // first created by calling this method. + Category FromRemoteCategory(int remote_category); + + // Creates a category from an ID as returned by |Category::id()|. + // |id| must be a non-negative value. + Category FromIDValue(int id); + + // Compares the given categories according to a strict ordering, returning + // true if and only if |left| is strictly less than |right|. + // This method satisfies the "Compare" contract required by sort algorithms. + // The order is determined as follows: All local categories go first, in a + // specific order hard-coded in the |CategoryFactory| constructor. All remote + // categories follow in the order in which they were first created through + // |FromRemoteCategory|. + bool CompareCategories(const Category& left, const Category& right) const; + + private: + bool CategoryExists(int id); + void AddKnownCategory(KnownCategories known_category); + Category InternalFromID(int id); + + // Stores all known categories in the order which is also returned by + // |CompareCategories|. + std::vector<Category> ordered_categories_; + + DISALLOW_COPY_AND_ASSIGN(CategoryFactory); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_CATEGORY_FACTORY_H_ diff --git a/chromium/components/ntp_snippets/category_factory_unittest.cc b/chromium/components/ntp_snippets/category_factory_unittest.cc new file mode 100644 index 00000000000..6ffd149b79e --- /dev/null +++ b/chromium/components/ntp_snippets/category_factory_unittest.cc @@ -0,0 +1,126 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/category_factory.h" + +#include <algorithm> +#include <vector> + +#include "components/ntp_snippets/category.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace ntp_snippets { + +class CategoryFactoryTest : public testing::Test { + public: + CategoryFactoryTest() : unused_remote_category_id_(1) {} + + int GetUnusedRemoteCategoryID() { return unused_remote_category_id_++; } + + bool CompareCategories(const Category& left, const Category& right) { + return factory()->CompareCategories(left, right); + } + + void AddDummyRemoteCategories(int quantity) { + for (int i = 0; i < quantity; ++i) { + factory()->FromRemoteCategory(GetUnusedRemoteCategoryID()); + } + } + + CategoryFactory* factory() { return &factory_; } + + private: + CategoryFactory factory_; + int unused_remote_category_id_; + + DISALLOW_COPY_AND_ASSIGN(CategoryFactoryTest); +}; + +TEST_F(CategoryFactoryTest, + FromKnownCategoryShouldReturnSameIdForSameCategories) { + const KnownCategories known_category = KnownCategories::BOOKMARKS; + Category first = factory()->FromKnownCategory(known_category); + Category second = factory()->FromKnownCategory(known_category); + EXPECT_EQ(first, second); +} + +TEST_F(CategoryFactoryTest, + FromRemoteCategoryShouldReturnSameIdForSameCategories) { + const int remote_category_id = GetUnusedRemoteCategoryID(); + Category first = factory()->FromRemoteCategory(remote_category_id); + Category second = factory()->FromRemoteCategory(remote_category_id); + EXPECT_EQ(first, second); +} + +TEST_F(CategoryFactoryTest, FromRemoteCategoryOrder) { + const int small_id = GetUnusedRemoteCategoryID(); + const int large_id = GetUnusedRemoteCategoryID(); + // Categories are added in decreasing id order to test that they are not + // compared by id. + Category added_first = factory()->FromRemoteCategory(large_id); + Category added_second = factory()->FromRemoteCategory(small_id); + EXPECT_TRUE(CompareCategories(added_first, added_second)); + EXPECT_FALSE(CompareCategories(added_second, added_first)); +} + +TEST_F(CategoryFactoryTest, FromIDValueReturnsSameKnownCategory) { + Category known_category = + factory()->FromKnownCategory(KnownCategories::BOOKMARKS); + Category known_category_by_id = factory()->FromIDValue(known_category.id()); + EXPECT_EQ(known_category, known_category_by_id); +} + +TEST_F(CategoryFactoryTest, FromIDValueReturnsSameRemoteCategory) { + const int remote_category_id = GetUnusedRemoteCategoryID(); + Category remote_category = factory()->FromRemoteCategory(remote_category_id); + Category remote_category_by_id = factory()->FromIDValue(remote_category.id()); + EXPECT_EQ(remote_category, remote_category_by_id); +} + +TEST_F(CategoryFactoryTest, CompareCategoriesLocalBeforeRemote) { + const int remote_category_id = GetUnusedRemoteCategoryID(); + Category remote_category = factory()->FromRemoteCategory(remote_category_id); + Category local_category = + factory()->FromKnownCategory(KnownCategories::BOOKMARKS); + EXPECT_TRUE(CompareCategories(local_category, remote_category)); + EXPECT_FALSE(CompareCategories(remote_category, local_category)); +} + +TEST_F(CategoryFactoryTest, CompareCategoriesSame) { + const int remote_category_id = GetUnusedRemoteCategoryID(); + Category remote_category = factory()->FromRemoteCategory(remote_category_id); + EXPECT_FALSE(CompareCategories(remote_category, remote_category)); + + Category local_category = + factory()->FromKnownCategory(KnownCategories::BOOKMARKS); + EXPECT_FALSE(CompareCategories(local_category, local_category)); +} + +TEST_F(CategoryFactoryTest, CompareCategoriesAfterAddingNew) { + AddDummyRemoteCategories(3); + + Category consequtive_first = + factory()->FromRemoteCategory(GetUnusedRemoteCategoryID()); + Category consequtive_second = + factory()->FromRemoteCategory(GetUnusedRemoteCategoryID()); + + AddDummyRemoteCategories(3); + + Category nonconsequtive_first = + factory()->FromRemoteCategory(GetUnusedRemoteCategoryID()); + AddDummyRemoteCategories(3); + Category nonconsequtive_second = + factory()->FromRemoteCategory(GetUnusedRemoteCategoryID()); + + AddDummyRemoteCategories(3); + + EXPECT_TRUE(CompareCategories(consequtive_first, consequtive_second)); + EXPECT_FALSE(CompareCategories(consequtive_second, consequtive_first)); + + EXPECT_TRUE(CompareCategories(nonconsequtive_first, nonconsequtive_second)); + EXPECT_FALSE(CompareCategories(nonconsequtive_second, nonconsequtive_first)); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/category_info.cc b/chromium/components/ntp_snippets/category_info.cc new file mode 100644 index 00000000000..4037551f5e8 --- /dev/null +++ b/chromium/components/ntp_snippets/category_info.cc @@ -0,0 +1,20 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/category_info.h" + +namespace ntp_snippets { + +CategoryInfo::CategoryInfo(const base::string16& title, + ContentSuggestionsCardLayout card_layout, + bool has_more_button, + bool show_if_empty) + : title_(title), + card_layout_(card_layout), + has_more_button_(has_more_button), + show_if_empty_(show_if_empty) {} + +CategoryInfo::~CategoryInfo() = default; + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/category_info.h b/chromium/components/ntp_snippets/category_info.h new file mode 100644 index 00000000000..1a659ed232d --- /dev/null +++ b/chromium/components/ntp_snippets/category_info.h @@ -0,0 +1,60 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_CATEGORY_INFO_H_ +#define COMPONENTS_NTP_SNIPPETS_CATEGORY_INFO_H_ + +#include "base/macros.h" +#include "base/strings/string16.h" + +namespace ntp_snippets { + +// On Android builds, a Java counterpart will be generated for this enum. +// GENERATED_JAVA_ENUM_PACKAGE: org.chromium.chrome.browser.ntp.snippets +enum class ContentSuggestionsCardLayout { + // Uses all fields. + FULL_CARD, + + // No snippet_text and no thumbnail image. + MINIMAL_CARD +}; + +// Contains static meta information about a Category. +class CategoryInfo { + public: + CategoryInfo(const base::string16& title, + ContentSuggestionsCardLayout card_layout, + bool has_more_button, + bool show_if_empty); + CategoryInfo(CategoryInfo&&) = default; + CategoryInfo& operator=(CategoryInfo&&) = default; + + ~CategoryInfo(); + + // Localized title of the category. + const base::string16& title() const { return title_; } + + // Layout of the cards to be used to display suggestions in this category. + ContentSuggestionsCardLayout card_layout() const { return card_layout_; } + + // Whether the category supports a "More" button. The button either triggers + // a fixed action (like opening a native page) or, if there is no such fixed + // action, it queries the provider for more suggestions. + bool has_more_button() const { return has_more_button_; } + + // Whether this category should be shown if it offers no suggestions. + bool show_if_empty() const { return show_if_empty_; } + + private: + base::string16 title_; + ContentSuggestionsCardLayout card_layout_; + bool has_more_button_; + bool show_if_empty_; + + DISALLOW_COPY_AND_ASSIGN(CategoryInfo); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_CATEGORY_INFO_H_ diff --git a/chromium/components/ntp_snippets/category_status.cc b/chromium/components/ntp_snippets/category_status.cc new file mode 100644 index 00000000000..5afa5100d5e --- /dev/null +++ b/chromium/components/ntp_snippets/category_status.cc @@ -0,0 +1,21 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/category_status.h" + +namespace ntp_snippets { + +bool IsCategoryStatusAvailable(CategoryStatus status) { + // Note: This code is duplicated in SnippetsBridge.java. + return status == CategoryStatus::AVAILABLE_LOADING || + status == CategoryStatus::AVAILABLE; +} + +bool IsCategoryStatusInitOrAvailable(CategoryStatus status) { + // Note: This code is duplicated in SnippetsBridge.java. + return status == CategoryStatus::INITIALIZING || + IsCategoryStatusAvailable(status); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/category_status.h b/chromium/components/ntp_snippets/category_status.h new file mode 100644 index 00000000000..2f6c7cdfbd9 --- /dev/null +++ b/chromium/components/ntp_snippets/category_status.h @@ -0,0 +1,52 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_CATEGORY_STATUS_H_ +#define COMPONENTS_NTP_SNIPPETS_CATEGORY_STATUS_H_ + +namespace ntp_snippets { + +// Represents the status of a category of content suggestions. +// On Android builds, a Java counterpart will be generated for this enum. +// GENERATED_JAVA_ENUM_PACKAGE: org.chromium.chrome.browser.ntp.snippets +enum class CategoryStatus { + // The provider is still initializing and it is not yet determined whether + // content suggestions will be available or not. + INITIALIZING, + + // Content suggestions are available (though the list of available suggestions + // may be empty simply because there are no reasonable suggestions to be made + // at the moment). + AVAILABLE, + // Content suggestions are provided but not yet loaded. + AVAILABLE_LOADING, + + // There is no provider that provides suggestions for this category. + NOT_PROVIDED, + // The entire content suggestions feature has explicitly been disabled as part + // of the service configuration. + ALL_SUGGESTIONS_EXPLICITLY_DISABLED, + // Content suggestions from a specific category have been disabled as part of + // the service configuration. + CATEGORY_EXPLICITLY_DISABLED, + + // Content suggestions are not available because the user is not signed in. + SIGNED_OUT, + + // Content suggestions are not available because an error occurred when + // loading or updating them. + LOADING_ERROR +}; + +// Determines whether the given status is one of the AVAILABLE statuses. +bool IsCategoryStatusAvailable(CategoryStatus status); + +// Determines whether the given status is INITIALIZING or one of the AVAILABLE +// statuses. All of these statuses have in common that there is or will soon be +// content. +bool IsCategoryStatusInitOrAvailable(CategoryStatus status); + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_CATEGORY_STATUS_H_ diff --git a/chromium/components/ntp_snippets/content_suggestion.cc b/chromium/components/ntp_snippets/content_suggestion.cc index 0fa4f0cd9dd..44524cb8bf9 100644 --- a/chromium/components/ntp_snippets/content_suggestion.cc +++ b/chromium/components/ntp_snippets/content_suggestion.cc @@ -6,13 +6,32 @@ namespace ntp_snippets { -ContentSuggestion::ContentSuggestion( - const std::string& id, - const ContentSuggestionsProviderType provider, - const ContentSuggestionCategory category, - const GURL& url) - : id_(id), provider_(provider), category_(category), url_(url), score_(0) {} - -ContentSuggestion::~ContentSuggestion() {} +bool ContentSuggestion::ID::operator==(const ID& rhs) const { + return category_ == rhs.category_ && + id_within_category_ == rhs.id_within_category_; +} + +bool ContentSuggestion::ID::operator!=(const ID& rhs) const { + return !(*this == rhs); +} + +ContentSuggestion::ContentSuggestion(ID id, const GURL& url) + : id_(id), url_(url), score_(0) {} + +ContentSuggestion::ContentSuggestion(Category category, + const std::string& id_within_category, + const GURL& url) + : id_(category, id_within_category), url_(url), score_(0) {} + +ContentSuggestion::ContentSuggestion(ContentSuggestion&&) = default; + +ContentSuggestion& ContentSuggestion::operator=(ContentSuggestion&&) = default; + +ContentSuggestion::~ContentSuggestion() = default; + +std::ostream& operator<<(std::ostream& os, ContentSuggestion::ID id) { + os << id.category() << "|" << id.id_within_category(); + return os; +} } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/content_suggestion.h b/chromium/components/ntp_snippets/content_suggestion.h index 60ab27285d3..bf8442cfa10 100644 --- a/chromium/components/ntp_snippets/content_suggestion.h +++ b/chromium/components/ntp_snippets/content_suggestion.h @@ -5,41 +5,54 @@ #ifndef COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTION_H_ #define COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTION_H_ -#include <memory> #include <string> -#include <vector> #include "base/macros.h" +#include "base/strings/string16.h" #include "base/time/time.h" -#include "components/ntp_snippets/content_suggestion_category.h" -#include "components/ntp_snippets/content_suggestions_provider_type.h" +#include "components/ntp_snippets/category.h" #include "url/gurl.h" namespace ntp_snippets { // A content suggestion for the new tab page, which can be an article or an // offline page, for example. -// NOTE: This class is not yet in use, please use NTPSnippet for now -// (see ntp_snippet.h). class ContentSuggestion { public: - ContentSuggestion(const std::string& id, - const ContentSuggestionsProviderType provider, - const ContentSuggestionCategory category, - const GURL& url); + class ID { + public: + ID(Category category, const std::string& id_within_category) + : category_(category), id_within_category_(id_within_category) {} - ~ContentSuggestion(); + Category category() const { return category_; } + + const std::string& id_within_category() const { + return id_within_category_; + } - // An ID for identifying the suggestion. The ID is unique among all - // suggestions from the same provider, so to determine a globally unique - // identifier, combine this ID with the provider type. - const std::string& id() const { return id_; } + bool operator==(const ID& rhs) const; + bool operator!=(const ID& rhs) const; - // The provider that created this suggestion. - ContentSuggestionsProviderType provider() const { return provider_; } + private: + Category category_; + std::string id_within_category_; - // The category that this suggestion belongs to. - ContentSuggestionCategory category() const { return category_; } + // Allow copy and assignment. + }; + + // Creates a new ContentSuggestion. The caller must ensure that the |id| + // passed in here is unique application-wide. + ContentSuggestion(ID id, const GURL& url); + ContentSuggestion(Category category, + const std::string& id_within_category, + const GURL& url); + ContentSuggestion(ContentSuggestion&&); + ContentSuggestion& operator=(ContentSuggestion&&); + + ~ContentSuggestion(); + + // An ID for identifying the suggestion. The ID is unique application-wide. + const ID& id() const { return id_; } // The normal content URL where the content referenced by the suggestion can // be accessed. @@ -51,12 +64,12 @@ class ContentSuggestion { void set_amp_url(const GURL& amp_url) { amp_url_ = amp_url; } // Title of the suggestion. - const std::string& title() const { return title_; } - void set_title(const std::string& title) { title_ = title; } + const base::string16& title() const { return title_; } + void set_title(const base::string16& title) { title_ = title; } // Summary or relevant textual extract from the content. - const std::string& snippet_text() const { return snippet_text_; } - void set_snippet_text(const std::string& snippet_text) { + const base::string16& snippet_text() const { return snippet_text_; } + void set_snippet_text(const base::string16& snippet_text) { snippet_text_ = snippet_text; } @@ -67,12 +80,12 @@ class ContentSuggestion { } // The name of the source/publisher of this suggestion. - const std::string& publisher_name() const { return publisher_name_; } - void set_publisher_name(const std::string& publisher_name) { + const base::string16& publisher_name() const { return publisher_name_; } + void set_publisher_name(const base::string16& publisher_name) { publisher_name_ = publisher_name; } - // TODO(pke) Remove the score from the ContentSuggestion class. The UI only + // TODO(pke): Remove the score from the ContentSuggestion class. The UI only // uses it to track user clicks (histogram data). Instead, the providers // should be informed about clicks and do appropriate logging themselves. // IMPORTANT: The score may simply be 0 for suggestions from providers which @@ -81,21 +94,20 @@ class ContentSuggestion { void set_score(float score) { score_ = score; } private: - std::string id_; - ContentSuggestionsProviderType provider_; - ContentSuggestionCategory category_; + ID id_; GURL url_; GURL amp_url_; - std::string title_; - std::string snippet_text_; - GURL salient_image_url_; + base::string16 title_; + base::string16 snippet_text_; base::Time publish_date_; - std::string publisher_name_; + base::string16 publisher_name_; float score_; DISALLOW_COPY_AND_ASSIGN(ContentSuggestion); }; +std::ostream& operator<<(std::ostream& os, ContentSuggestion::ID id); + } // namespace ntp_snippets #endif // COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTION_H_ diff --git a/chromium/components/ntp_snippets/content_suggestion_category.h b/chromium/components/ntp_snippets/content_suggestion_category.h deleted file mode 100644 index 80268fbe140..00000000000 --- a/chromium/components/ntp_snippets/content_suggestion_category.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTION_CATEGORY_H_ -#define COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTION_CATEGORY_H_ - -namespace ntp_snippets { - -// The category of a content suggestion. Note that even though these categories -// currently match the provider types, a provider type is not limited to provide -// suggestions of a single (fixed) category only. The category is used to -// determine where to display the suggestion. -enum class ContentSuggestionCategory : int { ARTICLE }; - -} // namespace ntp_snippets - -#endif // COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTION_CATEGORY_H_ diff --git a/chromium/components/ntp_snippets/content_suggestions_metrics.cc b/chromium/components/ntp_snippets/content_suggestions_metrics.cc new file mode 100644 index 00000000000..4117fae096b --- /dev/null +++ b/chromium/components/ntp_snippets/content_suggestions_metrics.cc @@ -0,0 +1,272 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/content_suggestions_metrics.h" + +#include <string> +#include <type_traits> + +#include "base/metrics/histogram.h" +#include "base/metrics/histogram_macros.h" +#include "base/strings/stringprintf.h" +#include "base/template_util.h" + +namespace ntp_snippets { +namespace metrics { + +namespace { + +const int kMaxSuggestionsPerCategory = 10; +const int kMaxSuggestionsTotal = 50; + +const char kHistogramCountOnNtpOpened[] = + "NewTabPage.ContentSuggestions.CountOnNtpOpened"; +const char kHistogramShown[] = "NewTabPage.ContentSuggestions.Shown"; +const char kHistogramShownAge[] = "NewTabPage.ContentSuggestions.ShownAge"; +const char kHistogramShownScore[] = "NewTabPage.ContentSuggestions.ShownScore"; +const char kHistogramOpened[] = "NewTabPage.ContentSuggestions.Opened"; +const char kHistogramOpenedAge[] = "NewTabPage.ContentSuggestions.OpenedAge"; +const char kHistogramOpenedScore[] = + "NewTabPage.ContentSuggestions.OpenedScore"; +const char kHistogramOpenDisposition[] = + "NewTabPage.ContentSuggestions.OpenDisposition"; +const char kHistogramMenuOpened[] = "NewTabPage.ContentSuggestions.MenuOpened"; +const char kHistogramMenuOpenedAge[] = + "NewTabPage.ContentSuggestions.MenuOpenedAge"; +const char kHistogramMenuOpenedScore[] = + "NewTabPage.ContentSuggestions.MenuOpenedScore"; +const char kHistogramDismissedUnvisited[] = + "NewTabPage.ContentSuggestions.DismissedUnvisited"; +const char kHistogramDismissedVisited[] = + "NewTabPage.ContentSuggestions.DismissedVisited"; +const char kHistogramVisitDuration[] = + "NewTabPage.ContentSuggestions.VisitDuration"; +const char kHistogramMoreButtonShown[] = + "NewTabPage.ContentSuggestions.MoreButtonShown"; +const char kHistogramMoreButtonClicked[] = + "NewTabPage.ContentSuggestions.MoreButtonClicked"; + +const char kPerCategoryHistogramFormat[] = "%s.%s"; + +// Each suffix here should correspond to an entry under histogram suffix +// ContentSuggestionCategory in histograms.xml. +std::string GetCategorySuffix(Category category) { + static_assert( + std::is_same<decltype(category.id()), typename base::underlying_type< + KnownCategories>::type>::value, + "KnownCategories must have the same underlying type as category.id()"); + // Note: Since the underlying type of KnownCategories is int, it's legal to + // cast from int to KnownCategories, even if the given value isn't listed in + // the enumeration. The switch still makes sure that all known values are + // listed here. + KnownCategories known_category = static_cast<KnownCategories>(category.id()); + switch (known_category) { + case KnownCategories::RECENT_TABS: + return "RecentTabs"; + case KnownCategories::DOWNLOADS: + return "Downloads"; + case KnownCategories::BOOKMARKS: + return "Bookmarks"; + case KnownCategories::PHYSICAL_WEB_PAGES: + return "PhysicalWeb"; + case KnownCategories::FOREIGN_TABS: + return "ForeignTabs"; + case KnownCategories::ARTICLES: + return "Articles"; + case KnownCategories::LOCAL_CATEGORIES_COUNT: + case KnownCategories::REMOTE_CATEGORIES_OFFSET: + NOTREACHED(); + break; + } + // All other (unknown) categories go into a single "Experimental" bucket. + return "Experimental"; +} + +std::string GetCategoryHistogramName(const char* base_name, Category category) { + return base::StringPrintf(kPerCategoryHistogramFormat, base_name, + GetCategorySuffix(category).c_str()); +} + +// This corresponds to UMA_HISTOGRAM_ENUMERATION, for use with dynamic histogram +// names. +void UmaHistogramEnumeration(const std::string& name, + int value, + int boundary_value) { + base::LinearHistogram::FactoryGet( + name, 1, boundary_value, boundary_value + 1, + base::HistogramBase::kUmaTargetedHistogramFlag) + ->Add(value); +} + +// This corresponds to UMA_HISTOGRAM_LONG_TIMES for use with dynamic histogram +// names. +void UmaHistogramLongTimes(const std::string& name, + const base::TimeDelta& value) { + base::Histogram::FactoryTimeGet( + name, base::TimeDelta::FromMilliseconds(1), base::TimeDelta::FromHours(1), + 50, base::HistogramBase::kUmaTargetedHistogramFlag) + ->AddTime(value); +} + +// This corresponds to UMA_HISTOGRAM_CUSTOM_TIMES (with min/max appropriate +// for the age of suggestions) for use with dynamic histogram names. +void UmaHistogramAge(const std::string& name, const base::TimeDelta& value) { + base::Histogram::FactoryTimeGet( + name, base::TimeDelta::FromSeconds(1), base::TimeDelta::FromDays(7), 100, + base::HistogramBase::kUmaTargetedHistogramFlag) + ->AddTime(value); +} + +// This corresponds to UMA_HISTOGRAM_CUSTOM_COUNTS (with min/max appropriate +// for the score of suggestions) for use with dynamic histogram names. +void UmaHistogramScore(const std::string& name, float value) { + base::Histogram::FactoryGet(name, 1, 100000, 50, + base::HistogramBase::kUmaTargetedHistogramFlag) + ->Add(value); +} + +void LogCategoryHistogramEnumeration(const char* base_name, + Category category, + int value, + int boundary_value) { + std::string name = GetCategoryHistogramName(base_name, category); + // Since the histogram name is dynamic, we can't use the regular macro. + UmaHistogramEnumeration(name, value, boundary_value); +} + +void LogCategoryHistogramLongTimes(const char* base_name, + Category category, + const base::TimeDelta& value) { + std::string name = GetCategoryHistogramName(base_name, category); + // Since the histogram name is dynamic, we can't use the regular macro. + UmaHistogramLongTimes(name, value); +} + +void LogCategoryHistogramAge(const char* base_name, + Category category, + const base::TimeDelta& value) { + std::string name = GetCategoryHistogramName(base_name, category); + // Since the histogram name is dynamic, we can't use the regular macro. + UmaHistogramAge(name, value); +} + +void LogCategoryHistogramScore(const char* base_name, + Category category, + float score) { + std::string name = GetCategoryHistogramName(base_name, category); + // Since the histogram name is dynamic, we can't use the regular macro. + UmaHistogramScore(name, score); +} + +} // namespace + +void OnPageShown( + const std::vector<std::pair<Category, int>>& suggestions_per_category) { + int suggestions_total = 0; + for (const std::pair<Category, int>& item : suggestions_per_category) { + LogCategoryHistogramEnumeration(kHistogramCountOnNtpOpened, item.first, + item.second, kMaxSuggestionsPerCategory); + suggestions_total += item.second; + } + + UMA_HISTOGRAM_ENUMERATION(kHistogramCountOnNtpOpened, suggestions_total, + kMaxSuggestionsTotal); +} + +void OnSuggestionShown(int global_position, + Category category, + int category_position, + base::Time publish_date, + float score) { + UMA_HISTOGRAM_ENUMERATION(kHistogramShown, global_position, + kMaxSuggestionsTotal); + LogCategoryHistogramEnumeration(kHistogramShown, category, category_position, + kMaxSuggestionsPerCategory); + + base::TimeDelta age = base::Time::Now() - publish_date; + LogCategoryHistogramAge(kHistogramShownAge, category, age); + + LogCategoryHistogramScore(kHistogramShownScore, category, score); +} + +void OnSuggestionOpened(int global_position, + Category category, + int category_position, + base::Time publish_date, + float score, + WindowOpenDisposition disposition) { + UMA_HISTOGRAM_ENUMERATION(kHistogramOpened, global_position, + kMaxSuggestionsTotal); + LogCategoryHistogramEnumeration(kHistogramOpened, category, category_position, + kMaxSuggestionsPerCategory); + + base::TimeDelta age = base::Time::Now() - publish_date; + LogCategoryHistogramAge(kHistogramOpenedAge, category, age); + + LogCategoryHistogramScore(kHistogramOpenedScore, category, score); + + UMA_HISTOGRAM_ENUMERATION( + kHistogramOpenDisposition, static_cast<int>(disposition), + static_cast<int>(WindowOpenDisposition::MAX_VALUE) + 1); + LogCategoryHistogramEnumeration( + kHistogramOpenDisposition, category, static_cast<int>(disposition), + static_cast<int>(WindowOpenDisposition::MAX_VALUE) + 1); +} + +void OnSuggestionMenuOpened(int global_position, + Category category, + int category_position, + base::Time publish_date, + float score) { + UMA_HISTOGRAM_ENUMERATION(kHistogramMenuOpened, global_position, + kMaxSuggestionsTotal); + LogCategoryHistogramEnumeration(kHistogramMenuOpened, category, + category_position, + kMaxSuggestionsPerCategory); + + base::TimeDelta age = base::Time::Now() - publish_date; + LogCategoryHistogramAge(kHistogramMenuOpenedAge, category, age); + + LogCategoryHistogramScore(kHistogramMenuOpenedScore, category, score); +} + +void OnSuggestionDismissed(int global_position, + Category category, + int category_position, + bool visited) { + if (visited) { + UMA_HISTOGRAM_ENUMERATION(kHistogramDismissedVisited, global_position, + kMaxSuggestionsTotal); + LogCategoryHistogramEnumeration(kHistogramDismissedVisited, category, + category_position, + kMaxSuggestionsPerCategory); + } else { + UMA_HISTOGRAM_ENUMERATION(kHistogramDismissedUnvisited, global_position, + kMaxSuggestionsTotal); + LogCategoryHistogramEnumeration(kHistogramDismissedUnvisited, category, + category_position, + kMaxSuggestionsPerCategory); + } +} + +void OnSuggestionTargetVisited(Category category, base::TimeDelta visit_time) { + LogCategoryHistogramLongTimes(kHistogramVisitDuration, category, visit_time); +} + +void OnMoreButtonShown(Category category, int position) { + // The "more" card can appear in addition to the actual suggestions, so add + // one extra bucket to this histogram. + LogCategoryHistogramEnumeration(kHistogramMoreButtonShown, category, position, + kMaxSuggestionsPerCategory + 1); +} + +void OnMoreButtonClicked(Category category, int position) { + // The "more" card can appear in addition to the actual suggestions, so add + // one extra bucket to this histogram. + LogCategoryHistogramEnumeration(kHistogramMoreButtonClicked, category, + position, kMaxSuggestionsPerCategory + 1); +} + +} // namespace metrics +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/content_suggestions_metrics.h b/chromium/components/ntp_snippets/content_suggestions_metrics.h new file mode 100644 index 00000000000..cba4846e19c --- /dev/null +++ b/chromium/components/ntp_snippets/content_suggestions_metrics.h @@ -0,0 +1,56 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_METRICS_H_ +#define COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_METRICS_H_ + +#include <utility> +#include <vector> + +#include "base/time/time.h" +#include "components/ntp_snippets/category.h" +#include "ui/base/window_open_disposition.h" + +namespace ntp_snippets { +namespace metrics { + +void OnPageShown( + const std::vector<std::pair<Category, int>>& suggestions_per_category); + +// Should only be called once per NTP for each suggestion. +void OnSuggestionShown(int global_position, + Category category, + int category_position, + base::Time publish_date, + float score); + +void OnSuggestionOpened(int global_position, + Category category, + int category_position, + base::Time publish_date, + float score, + WindowOpenDisposition disposition); + +void OnSuggestionMenuOpened(int global_position, + Category category, + int category_position, + base::Time publish_date, + float score); + +void OnSuggestionDismissed(int global_position, + Category category, + int category_position, + bool visited); + +void OnSuggestionTargetVisited(Category category, base::TimeDelta visit_time); + +// Should only be called once per NTP for each "more" button. +void OnMoreButtonShown(Category category, int position); + +void OnMoreButtonClicked(Category category, int position); + +} // namespace metrics +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_SERVICE_H_ diff --git a/chromium/components/ntp_snippets/content_suggestions_provider.cc b/chromium/components/ntp_snippets/content_suggestions_provider.cc new file mode 100644 index 00000000000..34896577b98 --- /dev/null +++ b/chromium/components/ntp_snippets/content_suggestions_provider.cc @@ -0,0 +1,18 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/content_suggestions_provider.h" + +#include "components/ntp_snippets/category_factory.h" + +namespace ntp_snippets { + +ContentSuggestionsProvider::ContentSuggestionsProvider( + Observer* observer, + CategoryFactory* category_factory) + : observer_(observer), category_factory_(category_factory) {} + +ContentSuggestionsProvider::~ContentSuggestionsProvider() = default; + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/content_suggestions_provider.h b/chromium/components/ntp_snippets/content_suggestions_provider.h new file mode 100644 index 00000000000..f6741356d66 --- /dev/null +++ b/chromium/components/ntp_snippets/content_suggestions_provider.h @@ -0,0 +1,148 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_PROVIDER_H_ +#define COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_PROVIDER_H_ + +#include <string> +#include <vector> + +#include "base/callback_forward.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_info.h" +#include "components/ntp_snippets/category_status.h" +#include "components/ntp_snippets/content_suggestion.h" + +namespace gfx { +class Image; +} // namespace gfx + +namespace ntp_snippets { + +// Provides content suggestions from one particular source. +// A provider can provide suggestions for multiple ContentSuggestionCategories, +// but for every category that it provides, it will be the only provider in the +// system which provides suggestions for that category. +// Providers are created by the ContentSuggestionsServiceFactory and owned and +// shut down by the ContentSuggestionsService. +class ContentSuggestionsProvider { + public: + using ImageFetchedCallback = base::Callback<void(const gfx::Image&)>; + using DismissedSuggestionsCallback = base::Callback<void( + std::vector<ContentSuggestion> dismissed_suggestions)>; + + // The observer of a provider is notified when new data is available. + class Observer { + public: + // Called when the available content changed. + // If a provider provides suggestions for multiple categories, this callback + // is called once per category. The |suggestions| parameter always contains + // the full list of currently available suggestions for that category, i.e., + // an empty list will remove all suggestions from the given category. Note + // that to clear them from the UI immediately, the provider needs to change + // the status of the respective category. If the given |category| is not + // known yet, the calling |provider| will be registered as its provider. + virtual void OnNewSuggestions( + ContentSuggestionsProvider* provider, + Category category, + std::vector<ContentSuggestion> suggestions) = 0; + + // Called when the status of a category changed, including when it is added. + // If |new_status| is NOT_PROVIDED, the calling provider must be the one + // that currently provides the |category|, and the category is unregistered + // without clearing the UI. + // If |new_status| is any other value, it must match the value that is + // currently returned from the provider's |GetCategoryStatus(category)|. In + // case the given |category| is not known yet, the calling |provider| will + // be registered as its provider. Whenever the status changes to an + // unavailable status, all suggestions in that category must immediately be + // removed from all caches and from the UI, but the provider remains + // registered. + virtual void OnCategoryStatusChanged(ContentSuggestionsProvider* provider, + Category category, + CategoryStatus new_status) = 0; + + // Called when a suggestion has been invalidated. It will not be provided + // through |OnNewSuggestions| anymore, is not supported by + // |FetchSuggestionImage| or |DismissSuggestion| anymore, and should + // immediately be cleared from the UI and caches. This happens, for example, + // when the content that the suggestion refers to is gone. + // Note that this event may be fired even if the corresponding category is + // not currently AVAILABLE, because open UIs may still be showing the + // suggestion that is to be removed. This event may also be fired for + // |suggestion_id|s that never existed and should be ignored in that case. + virtual void OnSuggestionInvalidated( + ContentSuggestionsProvider* provider, + const ContentSuggestion::ID& suggestion_id) = 0; + }; + + virtual ~ContentSuggestionsProvider(); + + // Determines the status of the given |category|, see CategoryStatus. + virtual CategoryStatus GetCategoryStatus(Category category) = 0; + + // Returns the meta information for the given |category|. + virtual CategoryInfo GetCategoryInfo(Category category) = 0; + + // Dismisses the suggestion with the given ID. A provider needs to ensure that + // a once-dismissed suggestion is never delivered again (through the + // Observer). The provider must not call Observer::OnSuggestionsChanged if the + // removal of the dismissed suggestion is the only change. + virtual void DismissSuggestion( + const ContentSuggestion::ID& suggestion_id) = 0; + + // Fetches the image for the suggestion with the given ID and returns it + // through the callback. This fetch may occur locally or from the internet. + // If that suggestion doesn't exist, doesn't have an image or if the fetch + // fails, the callback gets a null image. The callback will not be called + // synchronously. + virtual void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) = 0; + + // Removes history from the specified time range where the URL matches the + // |filter|. The data removed depends on the provider. Note that the + // data outside the time range may be deleted, for example suggestions, which + // are based on history from that time range. Providers should immediately + // clear any data related to history from the specified time range where the + // URL matches the |filter|. + virtual void ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) = 0; + + // Clears all caches for the given category, so that the next fetch starts + // from scratch. + virtual void ClearCachedSuggestions(Category category) = 0; + + // Used only for debugging purposes. Retrieves suggestions for the given + // |category| that have previously been dismissed and are still stored in the + // provider. If the provider doesn't store dismissed suggestions for the given + // |category|, it always calls the callback with an empty vector. The callback + // may be called synchronously. + virtual void GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) = 0; + + // Used only for debugging purposes. Clears the cache of dismissed + // suggestions for the given |category|, if present, so that no suggestions + // are suppressed. This does not necessarily make previously dismissed + // suggestions reappear, as they may have been permanently deleted, depending + // on the provider implementation. + virtual void ClearDismissedSuggestionsForDebugging(Category category) = 0; + + protected: + ContentSuggestionsProvider(Observer* observer, + CategoryFactory* category_factory); + + Observer* observer() const { return observer_; } + CategoryFactory* category_factory() const { return category_factory_; } + + private: + Observer* observer_; + CategoryFactory* category_factory_; +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_PROVIDER_H_ diff --git a/chromium/components/ntp_snippets/content_suggestions_provider_type.h b/chromium/components/ntp_snippets/content_suggestions_provider_type.h deleted file mode 100644 index 9825de8bcd7..00000000000 --- a/chromium/components/ntp_snippets/content_suggestions_provider_type.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_PROVIDER_TYPE_H_ -#define COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_PROVIDER_TYPE_H_ - -namespace ntp_snippets { - -// A type of content suggestion provider. For each of these types, there will be -// at most one provider instance registered and running. Note that these -// provider types do not necessarily match the suggestion categories. The -// provider type is used to identify the source of a suggestion and to direct -// calls from the UI like discarding back to the right provider. -enum class ContentSuggestionsProviderType : int { ARTICLES }; - -} // namespace ntp_snippets - -#endif // COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_PROVIDER_TYPE_H_ diff --git a/chromium/components/ntp_snippets/content_suggestions_service.cc b/chromium/components/ntp_snippets/content_suggestions_service.cc new file mode 100644 index 00000000000..75b4f7db182 --- /dev/null +++ b/chromium/components/ntp_snippets/content_suggestions_service.cc @@ -0,0 +1,337 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/content_suggestions_service.h" + +#include <algorithm> +#include <iterator> +#include <set> +#include <utility> + +#include "base/bind.h" +#include "base/location.h" +#include "base/strings/string_number_conversions.h" +#include "base/threading/thread_task_runner_handle.h" +#include "ui/gfx/image/image.h" + +namespace ntp_snippets { + +ContentSuggestionsService::ContentSuggestionsService( + State state, + history::HistoryService* history_service, + PrefService* pref_service) + : state_(state), + history_service_observer_(this), + ntp_snippets_service_(nullptr), + user_classifier_(pref_service) { + // Can be null in tests. + if (history_service) + history_service_observer_.Add(history_service); +} + +ContentSuggestionsService::~ContentSuggestionsService() = default; + +void ContentSuggestionsService::Shutdown() { + ntp_snippets_service_ = nullptr; + suggestions_by_category_.clear(); + providers_by_category_.clear(); + categories_.clear(); + providers_.clear(); + state_ = State::DISABLED; + FOR_EACH_OBSERVER(Observer, observers_, ContentSuggestionsServiceShutdown()); +} + +CategoryStatus ContentSuggestionsService::GetCategoryStatus( + Category category) const { + if (state_ == State::DISABLED) { + return CategoryStatus::ALL_SUGGESTIONS_EXPLICITLY_DISABLED; + } + + auto iterator = providers_by_category_.find(category); + if (iterator == providers_by_category_.end()) + return CategoryStatus::NOT_PROVIDED; + + return iterator->second->GetCategoryStatus(category); +} + +base::Optional<CategoryInfo> ContentSuggestionsService::GetCategoryInfo( + Category category) const { + auto iterator = providers_by_category_.find(category); + if (iterator == providers_by_category_.end()) + return base::Optional<CategoryInfo>(); + return iterator->second->GetCategoryInfo(category); +} + +const std::vector<ContentSuggestion>& +ContentSuggestionsService::GetSuggestionsForCategory(Category category) const { + auto iterator = suggestions_by_category_.find(category); + if (iterator == suggestions_by_category_.end()) + return no_suggestions_; + return iterator->second; +} + +void ContentSuggestionsService::FetchSuggestionImage( + const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) { + if (!providers_by_category_.count(suggestion_id.category())) { + LOG(WARNING) << "Requested image for suggestion " << suggestion_id + << " for unavailable category " << suggestion_id.category(); + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::Bind(callback, gfx::Image())); + return; + } + providers_by_category_[suggestion_id.category()]->FetchSuggestionImage( + suggestion_id, callback); +} + +void ContentSuggestionsService::ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) { + for (const auto& provider : providers_) { + provider->ClearHistory(begin, end, filter); + } +} + +void ContentSuggestionsService::ClearAllCachedSuggestions() { + suggestions_by_category_.clear(); + for (const auto& category_provider_pair : providers_by_category_) { + category_provider_pair.second->ClearCachedSuggestions( + category_provider_pair.first); + FOR_EACH_OBSERVER(Observer, observers_, + OnNewSuggestions(category_provider_pair.first)); + } +} + +void ContentSuggestionsService::ClearCachedSuggestions(Category category) { + suggestions_by_category_[category].clear(); + auto iterator = providers_by_category_.find(category); + if (iterator != providers_by_category_.end()) + iterator->second->ClearCachedSuggestions(category); +} + +void ContentSuggestionsService::GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) { + auto iterator = providers_by_category_.find(category); + if (iterator != providers_by_category_.end()) + iterator->second->GetDismissedSuggestionsForDebugging(category, callback); + else + callback.Run(std::vector<ContentSuggestion>()); +} + +void ContentSuggestionsService::ClearDismissedSuggestionsForDebugging( + Category category) { + auto iterator = providers_by_category_.find(category); + if (iterator != providers_by_category_.end()) + iterator->second->ClearDismissedSuggestionsForDebugging(category); +} + +void ContentSuggestionsService::DismissSuggestion( + const ContentSuggestion::ID& suggestion_id) { + if (!providers_by_category_.count(suggestion_id.category())) { + LOG(WARNING) << "Dismissed suggestion " << suggestion_id + << " for unavailable category " << suggestion_id.category(); + return; + } + providers_by_category_[suggestion_id.category()]->DismissSuggestion( + suggestion_id); + + // Remove the suggestion locally. + bool removed = RemoveSuggestionByID(suggestion_id); + DCHECK(removed) << "The dismissed suggestion " << suggestion_id + << " has already been removed. Providers must not call" + << " OnNewSuggestions in response to DismissSuggestion."; +} + +void ContentSuggestionsService::DismissCategory(Category category) { + auto providers_it = providers_by_category_.find(category); + if (providers_it == providers_by_category_.end()) + return; + + providers_by_category_.erase(providers_it); + categories_.erase( + std::find(categories_.begin(), categories_.end(), category)); +} + +void ContentSuggestionsService::AddObserver(Observer* observer) { + observers_.AddObserver(observer); +} + +void ContentSuggestionsService::RemoveObserver(Observer* observer) { + observers_.RemoveObserver(observer); +} + +void ContentSuggestionsService::RegisterProvider( + std::unique_ptr<ContentSuggestionsProvider> provider) { + DCHECK(state_ == State::ENABLED); + providers_.push_back(std::move(provider)); +} + +//////////////////////////////////////////////////////////////////////////////// +// Private methods + +void ContentSuggestionsService::OnNewSuggestions( + ContentSuggestionsProvider* provider, + Category category, + std::vector<ContentSuggestion> suggestions) { + if (RegisterCategoryIfRequired(provider, category)) + NotifyCategoryStatusChanged(category); + + if (!IsCategoryStatusAvailable(provider->GetCategoryStatus(category))) { + // A provider shouldn't send us suggestions while it's not available. + DCHECK(suggestions.empty()); + return; + } + + suggestions_by_category_[category] = std::move(suggestions); + + // The positioning of the bookmarks category depends on whether it's empty. + // TODO(treib): Remove this temporary hack, crbug.com/640568. + if (category.IsKnownCategory(KnownCategories::BOOKMARKS)) + SortCategories(); + + FOR_EACH_OBSERVER(Observer, observers_, OnNewSuggestions(category)); +} + +void ContentSuggestionsService::OnCategoryStatusChanged( + ContentSuggestionsProvider* provider, + Category category, + CategoryStatus new_status) { + if (!IsCategoryStatusAvailable(new_status)) { + suggestions_by_category_.erase(category); + } + if (new_status == CategoryStatus::NOT_PROVIDED) { + DCHECK(providers_by_category_.find(category) != + providers_by_category_.end()); + DCHECK_EQ(provider, providers_by_category_.find(category)->second); + DismissCategory(category); + } else { + RegisterCategoryIfRequired(provider, category); + DCHECK_EQ(new_status, provider->GetCategoryStatus(category)); + } + NotifyCategoryStatusChanged(category); +} + +void ContentSuggestionsService::OnSuggestionInvalidated( + ContentSuggestionsProvider* provider, + const ContentSuggestion::ID& suggestion_id) { + RemoveSuggestionByID(suggestion_id); + FOR_EACH_OBSERVER(Observer, observers_, + OnSuggestionInvalidated(suggestion_id)); +} + +// history::HistoryServiceObserver implementation. +void ContentSuggestionsService::OnURLsDeleted( + history::HistoryService* history_service, + bool all_history, + bool expired, + const history::URLRows& deleted_rows, + const std::set<GURL>& favicon_urls) { + // We don't care about expired entries. + if (expired) + return; + + // Redirect to ClearHistory(). + if (all_history) { + base::Time begin = base::Time(); + base::Time end = base::Time::Max(); + base::Callback<bool(const GURL& url)> filter = + base::Bind([](const GURL& url) { return true; }); + ClearHistory(begin, end, filter); + } else { + if (deleted_rows.empty()) + return; + + base::Time begin = deleted_rows[0].last_visit(); + base::Time end = deleted_rows[0].last_visit(); + std::set<GURL> deleted_urls; + for (const history::URLRow& row : deleted_rows) { + if (row.last_visit() < begin) + begin = row.last_visit(); + if (row.last_visit() > end) + end = row.last_visit(); + deleted_urls.insert(row.url()); + } + base::Callback<bool(const GURL& url)> filter = base::Bind( + [](const std::set<GURL>& set, const GURL& url) { + return set.count(url) != 0; + }, + deleted_urls); + ClearHistory(begin, end, filter); + } +} + +void ContentSuggestionsService::HistoryServiceBeingDeleted( + history::HistoryService* history_service) { + history_service_observer_.RemoveAll(); +} + +bool ContentSuggestionsService::RegisterCategoryIfRequired( + ContentSuggestionsProvider* provider, + Category category) { + auto it = providers_by_category_.find(category); + if (it != providers_by_category_.end()) { + DCHECK_EQ(it->second, provider); + return false; + } + + providers_by_category_[category] = provider; + categories_.push_back(category); + SortCategories(); + if (IsCategoryStatusAvailable(provider->GetCategoryStatus(category))) { + suggestions_by_category_.insert( + std::make_pair(category, std::vector<ContentSuggestion>())); + } + return true; +} + +bool ContentSuggestionsService::RemoveSuggestionByID( + const ContentSuggestion::ID& suggestion_id) { + std::vector<ContentSuggestion>* suggestions = + &suggestions_by_category_[suggestion_id.category()]; + auto position = + std::find_if(suggestions->begin(), suggestions->end(), + [&suggestion_id](const ContentSuggestion& suggestion) { + return suggestion_id == suggestion.id(); + }); + if (position == suggestions->end()) + return false; + suggestions->erase(position); + + // The positioning of the bookmarks category depends on whether it's empty. + // TODO(treib): Remove this temporary hack, crbug.com/640568. + if (suggestion_id.category().IsKnownCategory(KnownCategories::BOOKMARKS)) + SortCategories(); + + return true; +} + +void ContentSuggestionsService::NotifyCategoryStatusChanged(Category category) { + FOR_EACH_OBSERVER( + Observer, observers_, + OnCategoryStatusChanged(category, GetCategoryStatus(category))); +} + +void ContentSuggestionsService::SortCategories() { + auto it = suggestions_by_category_.find( + category_factory_.FromKnownCategory(KnownCategories::BOOKMARKS)); + bool bookmarks_empty = + (it == suggestions_by_category_.end() || it->second.empty()); + std::sort( + categories_.begin(), categories_.end(), + [this, bookmarks_empty](const Category& left, const Category& right) { + // If the bookmarks section is empty, put it at the end. + // TODO(treib): This is a temporary hack, see crbug.com/640568. + if (bookmarks_empty) { + if (left.IsKnownCategory(KnownCategories::BOOKMARKS)) + return false; + if (right.IsKnownCategory(KnownCategories::BOOKMARKS)) + return true; + } + return category_factory_.CompareCategories(left, right); + }); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/content_suggestions_service.h b/chromium/components/ntp_snippets/content_suggestions_service.h new file mode 100644 index 00000000000..987a9fb4acc --- /dev/null +++ b/chromium/components/ntp_snippets/content_suggestions_service.h @@ -0,0 +1,273 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_SERVICE_H_ +#define COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_SERVICE_H_ + +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "base/callback_forward.h" +#include "base/observer_list.h" +#include "base/optional.h" +#include "base/scoped_observer.h" +#include "base/time/time.h" +#include "components/history/core/browser/history_service.h" +#include "components/history/core/browser/history_service_observer.h" +#include "components/keyed_service/core/keyed_service.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/category_status.h" +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "components/ntp_snippets/user_classifier.h" + +class PrefService; + +namespace gfx { +class Image; +} // namespace gfx + +namespace ntp_snippets { + +class NTPSnippetsService; + +// Retrieves suggestions from a number of ContentSuggestionsProviders and serves +// them grouped into categories. There can be at most one provider per category. +class ContentSuggestionsService : public KeyedService, + public ContentSuggestionsProvider::Observer, + public history::HistoryServiceObserver { + public: + using ImageFetchedCallback = base::Callback<void(const gfx::Image&)>; + using DismissedSuggestionsCallback = base::Callback<void( + std::vector<ContentSuggestion> dismissed_suggestions)>; + + class Observer { + public: + // Fired every time the service receives a new set of data for the given + // |category|, replacing any previously available data (though in most cases + // there will be an overlap and only a few changes within the data). The new + // data is then available through |GetSuggestionsForCategory(category)|. + virtual void OnNewSuggestions(Category category) = 0; + + // Fired when the status of a suggestions category changed. When the status + // changes to an unavailable status, the suggestions of the respective + // category have been invalidated, which means that they must no longer be + // displayed to the user. The UI must immediately clear any suggestions of + // that category. + virtual void OnCategoryStatusChanged(Category category, + CategoryStatus new_status) = 0; + + // Fired when a suggestion has been invalidated. The UI must immediately + // clear the suggestion even from open NTPs. Invalidation happens, for + // example, when the content that the suggestion refers to is gone. + // Note that this event may be fired even if the corresponding category is + // not currently AVAILABLE, because open UIs may still be showing the + // suggestion that is to be removed. This event may also be fired for + // |suggestion_id|s that never existed and should be ignored in that case. + virtual void OnSuggestionInvalidated( + const ContentSuggestion::ID& suggestion_id) = 0; + + // Sent when the service is shutting down. After the service has shut down, + // it will not provide any data anymore, though calling the getters is still + // safe. + virtual void ContentSuggestionsServiceShutdown() = 0; + + protected: + virtual ~Observer() = default; + }; + + enum State { + ENABLED, + DISABLED, + }; + + ContentSuggestionsService(State state, + history::HistoryService* history_service, + PrefService* pref_service); + ~ContentSuggestionsService() override; + + // Inherited from KeyedService. + void Shutdown() override; + + State state() { return state_; } + + // Gets all categories for which a provider is registered. The categories + // may or may not be available, see |GetCategoryStatus()|. + const std::vector<Category>& GetCategories() const { return categories_; } + + // Gets the status of a category. + CategoryStatus GetCategoryStatus(Category category) const; + + // Gets the meta information of a category. + base::Optional<CategoryInfo> GetCategoryInfo(Category category) const; + + // Gets the available suggestions for a category. The result is empty if the + // category is available and empty, but also if the category is unavailable + // for any reason, see |GetCategoryStatus()|. + const std::vector<ContentSuggestion>& GetSuggestionsForCategory( + Category category) const; + + // Fetches the image for the suggestion with the given |suggestion_id| and + // runs the |callback|. If that suggestion doesn't exist or the fetch fails, + // the callback gets an empty image. The callback will not be called + // synchronously. + void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback); + + // Dismisses the suggestion with the given |suggestion_id|, if it exists. + // This will not trigger an update through the observers. + void DismissSuggestion(const ContentSuggestion::ID& suggestion_id); + + // Dismisses the given |category|, if it exists. + // This will not trigger an update through the observers. + void DismissCategory(Category category); + + // Observer accessors. + void AddObserver(Observer* observer); + void RemoveObserver(Observer* observer); + + // Registers a new ContentSuggestionsProvider. It must be ensured that at most + // one provider is registered for every category and that this method is + // called only once per provider. + void RegisterProvider(std::unique_ptr<ContentSuggestionsProvider> provider); + + // Removes history from the specified time range where the URL matches the + // |filter| from all providers. The data removed depends on the provider. Note + // that the data outside the time range may be deleted, for example + // suggestions, which are based on history from that time range. Providers + // should immediately clear any data related to history from the specified + // time range where the URL matches the |filter|. + void ClearHistory(base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter); + + // Removes all suggestions from all caches or internal stores in all + // providers. See |ClearCachedSuggestions|. + void ClearAllCachedSuggestions(); + + // Removes all suggestions of the given |category| from all caches or internal + // stores in the service and the corresponding provider. It does, however, not + // remove any suggestions from the provider's sources, so if its configuration + // hasn't changed, it might return the same results when it fetches the next + // time. In particular, calling this method will not mark any suggestions as + // dismissed. + void ClearCachedSuggestions(Category category); + + // Only for debugging use through the internals page. + // Retrieves suggestions of the given |category| that have previously been + // dismissed and are still stored in the respective provider. If the + // provider doesn't store dismissed suggestions, the callback receives an + // empty vector. The callback may be called synchronously. + void GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback); + + // Only for debugging use through the internals page. Some providers + // internally store a list of dismissed suggestions to prevent them from + // reappearing. This function clears all suggestions of the given |category| + // from such lists, making dismissed suggestions reappear (if the provider + // supports it). + void ClearDismissedSuggestionsForDebugging(Category category); + + CategoryFactory* category_factory() { return &category_factory_; } + + // The reference to the NTPSnippetsService provider should only be set by the + // factory and only be used for scheduling, periodic fetching and debugging. + NTPSnippetsService* ntp_snippets_service() { return ntp_snippets_service_; } + void set_ntp_snippets_service(NTPSnippetsService* ntp_snippets_service) { + ntp_snippets_service_ = ntp_snippets_service; + } + + UserClassifier* user_classifier() { return &user_classifier_; } + + private: + friend class ContentSuggestionsServiceTest; + + // Implementation of ContentSuggestionsProvider::Observer. + void OnNewSuggestions(ContentSuggestionsProvider* provider, + Category category, + std::vector<ContentSuggestion> suggestions) override; + void OnCategoryStatusChanged(ContentSuggestionsProvider* provider, + Category category, + CategoryStatus new_status) override; + void OnSuggestionInvalidated( + ContentSuggestionsProvider* provider, + const ContentSuggestion::ID& suggestion_id) override; + + // history::HistoryServiceObserver implementation. + void OnURLsDeleted(history::HistoryService* history_service, + bool all_history, + bool expired, + const history::URLRows& deleted_rows, + const std::set<GURL>& favicon_urls) override; + void HistoryServiceBeingDeleted( + history::HistoryService* history_service) override; + + // Registers the given |provider| for the given |category|, unless it is + // already registered. Returns true if the category was newly registered or + // false if it was present before. + bool RegisterCategoryIfRequired(ContentSuggestionsProvider* provider, + Category category); + + // Removes a suggestion from the local store |suggestions_by_category_|, if it + // exists. Returns true if a suggestion was removed. + bool RemoveSuggestionByID(const ContentSuggestion::ID& suggestion_id); + + // Fires the OnCategoryStatusChanged event for the given |category|. + void NotifyCategoryStatusChanged(Category category); + + void SortCategories(); + + // Whether the content suggestions feature is enabled. + State state_; + + // Provides new and existing categories and an order for them. + CategoryFactory category_factory_; + + // All registered providers, owned by the service. + std::vector<std::unique_ptr<ContentSuggestionsProvider>> providers_; + + // All registered categories and their providers. A provider may be contained + // multiple times, if it provides multiple categories. The keys of this map + // are exactly the entries of |categories_| and the values are a subset of + // |providers_|. + std::map<Category, ContentSuggestionsProvider*, Category::CompareByID> + providers_by_category_; + + // All current suggestion categories, in an order determined by the + // |category_factory_|. This vector contains exactly the same categories as + // |providers_by_category_|. + std::vector<Category> categories_; + + // All current suggestions grouped by category. This contains an entry for + // every category in |categories_| whose status is an available status. It may + // contain an empty vector if the category is available but empty (or still + // loading). + std::map<Category, std::vector<ContentSuggestion>, Category::CompareByID> + suggestions_by_category_; + + // Observer for the HistoryService. All providers are notified when history is + // deleted. + ScopedObserver<history::HistoryService, history::HistoryServiceObserver> + history_service_observer_; + + base::ObserverList<Observer> observers_; + + const std::vector<ContentSuggestion> no_suggestions_; + + // Keep a direct reference to this special provider to redirect scheduling, + // background fetching and debugging calls to it. If the NTPSnippetsService is + // loaded, it is also present in |providers_|, otherwise this is a nullptr. + NTPSnippetsService* ntp_snippets_service_; + + UserClassifier user_classifier_; + + DISALLOW_COPY_AND_ASSIGN(ContentSuggestionsService); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_CONTENT_SUGGESTIONS_SERVICE_H_ diff --git a/chromium/components/ntp_snippets/content_suggestions_service_unittest.cc b/chromium/components/ntp_snippets/content_suggestions_service_unittest.cc new file mode 100644 index 00000000000..a2245bfd6cb --- /dev/null +++ b/chromium/components/ntp_snippets/content_suggestions_service_unittest.cc @@ -0,0 +1,604 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/content_suggestions_service.h" + +#include <memory> +#include <utility> +#include <vector> + +#include "base/bind.h" +#include "base/macros.h" +#include "base/memory/ptr_util.h" +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/utf_string_conversions.h" +#include "components/ntp_snippets/category_info.h" +#include "components/ntp_snippets/category_status.h" +#include "components/ntp_snippets/content_suggestion.h" +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "ui/gfx/image/image.h" + +using testing::ElementsAre; +using testing::Eq; +using testing::InvokeWithoutArgs; +using testing::IsEmpty; +using testing::Mock; +using testing::Property; +using testing::_; + +namespace ntp_snippets { + +namespace { + +class MockProvider : public ContentSuggestionsProvider { + public: + MockProvider(Observer* observer, + CategoryFactory* category_factory, + const std::vector<Category>& provided_categories) + : ContentSuggestionsProvider(observer, category_factory) { + SetProvidedCategories(provided_categories); + } + + void SetProvidedCategories(const std::vector<Category>& provided_categories) { + statuses_.clear(); + provided_categories_ = provided_categories; + for (Category category : provided_categories) { + statuses_[category.id()] = CategoryStatus::AVAILABLE; + } + } + + CategoryStatus GetCategoryStatus(Category category) override { + return statuses_[category.id()]; + } + + CategoryInfo GetCategoryInfo(Category category) override { + return CategoryInfo(base::ASCIIToUTF16("Section title"), + ContentSuggestionsCardLayout::FULL_CARD, true, true); + } + + void FireSuggestionsChanged( + Category category, + std::vector<ContentSuggestion> suggestions) { + observer()->OnNewSuggestions(this, category, std::move(suggestions)); + } + + void FireCategoryStatusChanged(Category category, CategoryStatus new_status) { + statuses_[category.id()] = new_status; + observer()->OnCategoryStatusChanged(this, category, new_status); + } + + void FireCategoryStatusChangedWithCurrentStatus(Category category) { + observer()->OnCategoryStatusChanged(this, category, + statuses_[category.id()]); + } + + void FireSuggestionInvalidated(const ContentSuggestion::ID& suggestion_id) { + observer()->OnSuggestionInvalidated(this, suggestion_id); + } + + MOCK_METHOD3(ClearHistory, + void(base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter)); + MOCK_METHOD1(ClearCachedSuggestions, void(Category category)); + MOCK_METHOD2(GetDismissedSuggestionsForDebugging, + void(Category category, + const DismissedSuggestionsCallback& callback)); + MOCK_METHOD1(ClearDismissedSuggestionsForDebugging, void(Category category)); + MOCK_METHOD1(DismissSuggestion, + void(const ContentSuggestion::ID& suggestion_id)); + MOCK_METHOD2(FetchSuggestionImage, + void(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback)); + + private: + std::vector<Category> provided_categories_; + std::map<int, CategoryStatus> statuses_; +}; + +class MockServiceObserver : public ContentSuggestionsService::Observer { + public: + MockServiceObserver() = default; + ~MockServiceObserver() override = default; + + MOCK_METHOD1(OnNewSuggestions, void(Category category)); + MOCK_METHOD2(OnCategoryStatusChanged, + void(Category changed_category, CategoryStatus new_status)); + MOCK_METHOD1(OnSuggestionInvalidated, + void(const ContentSuggestion::ID& suggestion_id)); + MOCK_METHOD0(ContentSuggestionsServiceShutdown, void()); + + private: + DISALLOW_COPY_AND_ASSIGN(MockServiceObserver); +}; + +} // namespace + +class ContentSuggestionsServiceTest : public testing::Test { + public: + ContentSuggestionsServiceTest() {} + + void SetUp() override { + CreateContentSuggestionsService(ContentSuggestionsService::State::ENABLED); + } + + void TearDown() override { + service_->Shutdown(); + service_.reset(); + } + + // Verifies that exactly the suggestions with the given |numbers| are + // returned by the service for the given |category|. + void ExpectThatSuggestionsAre(Category category, std::vector<int> numbers) { + std::vector<Category> categories = service()->GetCategories(); + auto position = std::find(categories.begin(), categories.end(), category); + if (!numbers.empty()) { + EXPECT_NE(categories.end(), position); + } + + for (const auto& suggestion : + service()->GetSuggestionsForCategory(category)) { + std::string id_within_category = suggestion.id().id_within_category(); + int id; + ASSERT_TRUE(base::StringToInt(id_within_category, &id)); + auto position = std::find(numbers.begin(), numbers.end(), id); + if (position == numbers.end()) { + ADD_FAILURE() << "Unexpected suggestion with ID " << id; + } else { + numbers.erase(position); + } + } + for (int number : numbers) { + ADD_FAILURE() << "Suggestion number " << number + << " not present, though expected"; + } + } + + const std::map<Category, ContentSuggestionsProvider*, Category::CompareByID>& + providers() { + return service()->providers_by_category_; + } + + CategoryFactory* category_factory() { return service()->category_factory(); } + + Category FromKnownCategory(KnownCategories known_category) { + return service()->category_factory()->FromKnownCategory(known_category); + } + + Category FromRemoteCategory(int remote_category) { + return service()->category_factory()->FromRemoteCategory(remote_category); + } + + MockProvider* RegisterProvider(Category provided_category) { + return RegisterProvider(std::vector<Category>({provided_category})); + } + + MockProvider* RegisterProvider( + const std::vector<Category>& provided_categories) { + std::unique_ptr<MockProvider> provider = base::MakeUnique<MockProvider>( + service(), category_factory(), provided_categories); + MockProvider* result = provider.get(); + service()->RegisterProvider(std::move(provider)); + return result; + } + + MOCK_METHOD1(OnImageFetched, void(const gfx::Image&)); + + protected: + void CreateContentSuggestionsService( + ContentSuggestionsService::State enabled) { + ASSERT_FALSE(service_); + service_.reset(new ContentSuggestionsService(enabled, + nullptr /* history_service */, + nullptr /* pref_service */)); + } + + ContentSuggestionsService* service() { return service_.get(); } + + // Returns a suggestion instance for testing. + ContentSuggestion CreateSuggestion(Category category, int number) { + return ContentSuggestion( + category, base::IntToString(number), + GURL("http://testsuggestion/" + base::IntToString(number))); + } + + std::vector<ContentSuggestion> CreateSuggestions( + Category category, + const std::vector<int>& numbers) { + std::vector<ContentSuggestion> result; + for (int number : numbers) { + result.push_back(CreateSuggestion(category, number)); + } + return result; + } + + private: + std::unique_ptr<ContentSuggestionsService> service_; + + DISALLOW_COPY_AND_ASSIGN(ContentSuggestionsServiceTest); +}; + +class ContentSuggestionsServiceDisabledTest + : public ContentSuggestionsServiceTest { + public: + void SetUp() override { + CreateContentSuggestionsService(ContentSuggestionsService::State::DISABLED); + } +}; + +TEST_F(ContentSuggestionsServiceTest, ShouldRegisterProviders) { + EXPECT_THAT(service()->state(), + Eq(ContentSuggestionsService::State::ENABLED)); + Category articles_category = FromKnownCategory(KnownCategories::ARTICLES); + Category offline_pages_category = + FromKnownCategory(KnownCategories::DOWNLOADS); + ASSERT_THAT(providers(), IsEmpty()); + EXPECT_THAT(service()->GetCategories(), IsEmpty()); + EXPECT_THAT(service()->GetCategoryStatus(articles_category), + Eq(CategoryStatus::NOT_PROVIDED)); + EXPECT_THAT(service()->GetCategoryStatus(offline_pages_category), + Eq(CategoryStatus::NOT_PROVIDED)); + + MockProvider* provider1 = RegisterProvider(articles_category); + provider1->FireCategoryStatusChangedWithCurrentStatus(articles_category); + EXPECT_THAT(providers().count(offline_pages_category), Eq(0ul)); + ASSERT_THAT(providers().count(articles_category), Eq(1ul)); + EXPECT_THAT(providers().at(articles_category), Eq(provider1)); + EXPECT_THAT(providers().size(), Eq(1ul)); + EXPECT_THAT(service()->GetCategories(), ElementsAre(articles_category)); + EXPECT_THAT(service()->GetCategoryStatus(articles_category), + Eq(CategoryStatus::AVAILABLE)); + EXPECT_THAT(service()->GetCategoryStatus(offline_pages_category), + Eq(CategoryStatus::NOT_PROVIDED)); + + MockProvider* provider2 = RegisterProvider(offline_pages_category); + provider2->FireCategoryStatusChangedWithCurrentStatus(offline_pages_category); + ASSERT_THAT(providers().count(offline_pages_category), Eq(1ul)); + EXPECT_THAT(providers().at(articles_category), Eq(provider1)); + ASSERT_THAT(providers().count(articles_category), Eq(1ul)); + EXPECT_THAT(providers().at(offline_pages_category), Eq(provider2)); + EXPECT_THAT(providers().size(), Eq(2ul)); + EXPECT_THAT(service()->GetCategories(), + ElementsAre(offline_pages_category, articles_category)); + EXPECT_THAT(service()->GetCategoryStatus(articles_category), + Eq(CategoryStatus::AVAILABLE)); + EXPECT_THAT(service()->GetCategoryStatus(offline_pages_category), + Eq(CategoryStatus::AVAILABLE)); +} + +TEST_F(ContentSuggestionsServiceDisabledTest, ShouldDoNothingWhenDisabled) { + Category articles_category = FromKnownCategory(KnownCategories::ARTICLES); + Category offline_pages_category = + FromKnownCategory(KnownCategories::DOWNLOADS); + EXPECT_THAT(service()->state(), + Eq(ContentSuggestionsService::State::DISABLED)); + EXPECT_THAT(providers(), IsEmpty()); + EXPECT_THAT(service()->GetCategoryStatus(articles_category), + Eq(CategoryStatus::ALL_SUGGESTIONS_EXPLICITLY_DISABLED)); + EXPECT_THAT(service()->GetCategoryStatus(offline_pages_category), + Eq(CategoryStatus::ALL_SUGGESTIONS_EXPLICITLY_DISABLED)); + EXPECT_THAT(service()->GetCategories(), IsEmpty()); + EXPECT_THAT(service()->GetSuggestionsForCategory(articles_category), + IsEmpty()); +} + +TEST_F(ContentSuggestionsServiceTest, ShouldRedirectFetchSuggestionImage) { + Category articles_category = FromKnownCategory(KnownCategories::ARTICLES); + Category offline_pages_category = + FromKnownCategory(KnownCategories::DOWNLOADS); + MockProvider* provider1 = RegisterProvider(articles_category); + MockProvider* provider2 = RegisterProvider(offline_pages_category); + + provider1->FireSuggestionsChanged(articles_category, + CreateSuggestions(articles_category, {1})); + ContentSuggestion::ID suggestion_id(articles_category, "1"); + + EXPECT_CALL(*provider1, FetchSuggestionImage(suggestion_id, _)); + EXPECT_CALL(*provider2, FetchSuggestionImage(_, _)).Times(0); + service()->FetchSuggestionImage( + suggestion_id, base::Bind(&ContentSuggestionsServiceTest::OnImageFetched, + base::Unretained(this))); +} + +TEST_F(ContentSuggestionsServiceTest, + ShouldCallbackEmptyImageForUnavailableProvider) { + // Setup the current thread's MessageLoop. + base::MessageLoop message_loop; + + base::RunLoop run_loop; + // Assuming there will never be a category with the id below. + ContentSuggestion::ID suggestion_id(category_factory()->FromIDValue(21563), + "TestID"); + EXPECT_CALL(*this, OnImageFetched(Property(&gfx::Image::IsEmpty, Eq(true)))) + .WillOnce(InvokeWithoutArgs(&run_loop, &base::RunLoop::Quit)); + service()->FetchSuggestionImage( + suggestion_id, base::Bind(&ContentSuggestionsServiceTest::OnImageFetched, + base::Unretained(this))); + run_loop.Run(); +} + +TEST_F(ContentSuggestionsServiceTest, ShouldRedirectDismissSuggestion) { + Category articles_category = FromKnownCategory(KnownCategories::ARTICLES); + Category offline_pages_category = + FromKnownCategory(KnownCategories::DOWNLOADS); + MockProvider* provider1 = RegisterProvider(articles_category); + MockProvider* provider2 = RegisterProvider(offline_pages_category); + + provider2->FireSuggestionsChanged( + offline_pages_category, CreateSuggestions(offline_pages_category, {11})); + ContentSuggestion::ID suggestion_id(offline_pages_category, "11"); + + EXPECT_CALL(*provider1, DismissSuggestion(_)).Times(0); + EXPECT_CALL(*provider2, DismissSuggestion(suggestion_id)); + service()->DismissSuggestion(suggestion_id); +} + +TEST_F(ContentSuggestionsServiceTest, ShouldRedirectSuggestionInvalidated) { + Category articles_category = FromKnownCategory(KnownCategories::ARTICLES); + + MockProvider* provider = RegisterProvider(articles_category); + MockServiceObserver observer; + service()->AddObserver(&observer); + + provider->FireSuggestionsChanged( + articles_category, CreateSuggestions(articles_category, {11, 12, 13})); + ExpectThatSuggestionsAre(articles_category, {11, 12, 13}); + + ContentSuggestion::ID suggestion_id(articles_category, "12"); + EXPECT_CALL(observer, OnSuggestionInvalidated(suggestion_id)); + provider->FireSuggestionInvalidated(suggestion_id); + ExpectThatSuggestionsAre(articles_category, {11, 13}); + Mock::VerifyAndClearExpectations(&observer); + + // Unknown IDs must be forwarded (though no change happens to the service's + // internal data structures) because previously opened UIs, which can still + // show the invalidated suggestion, must be notified. + ContentSuggestion::ID unknown_id(articles_category, "1234"); + EXPECT_CALL(observer, OnSuggestionInvalidated(unknown_id)); + provider->FireSuggestionInvalidated(unknown_id); + ExpectThatSuggestionsAre(articles_category, {11, 13}); + Mock::VerifyAndClearExpectations(&observer); + + service()->RemoveObserver(&observer); +} + +TEST_F(ContentSuggestionsServiceTest, ShouldForwardSuggestions) { + Category articles_category = FromKnownCategory(KnownCategories::ARTICLES); + Category offline_pages_category = + FromKnownCategory(KnownCategories::DOWNLOADS); + + // Create and register providers + MockProvider* provider1 = RegisterProvider(articles_category); + provider1->FireCategoryStatusChangedWithCurrentStatus(articles_category); + MockProvider* provider2 = RegisterProvider(offline_pages_category); + provider2->FireCategoryStatusChangedWithCurrentStatus(offline_pages_category); + ASSERT_THAT(providers().count(articles_category), Eq(1ul)); + EXPECT_THAT(providers().at(articles_category), Eq(provider1)); + ASSERT_THAT(providers().count(offline_pages_category), Eq(1ul)); + EXPECT_THAT(providers().at(offline_pages_category), Eq(provider2)); + + // Create and register observer + MockServiceObserver observer; + service()->AddObserver(&observer); + + // Send suggestions 1 and 2 + EXPECT_CALL(observer, OnNewSuggestions(articles_category)); + provider1->FireSuggestionsChanged( + articles_category, CreateSuggestions(articles_category, {1, 2})); + ExpectThatSuggestionsAre(articles_category, {1, 2}); + Mock::VerifyAndClearExpectations(&observer); + + // Send them again, make sure they're not reported twice + EXPECT_CALL(observer, OnNewSuggestions(articles_category)); + provider1->FireSuggestionsChanged( + articles_category, CreateSuggestions(articles_category, {1, 2})); + ExpectThatSuggestionsAre(articles_category, {1, 2}); + ExpectThatSuggestionsAre(offline_pages_category, std::vector<int>()); + Mock::VerifyAndClearExpectations(&observer); + + // Send suggestions 13 and 14 + EXPECT_CALL(observer, OnNewSuggestions(offline_pages_category)); + provider2->FireSuggestionsChanged( + offline_pages_category, CreateSuggestions(articles_category, {13, 14})); + ExpectThatSuggestionsAre(articles_category, {1, 2}); + ExpectThatSuggestionsAre(offline_pages_category, {13, 14}); + Mock::VerifyAndClearExpectations(&observer); + + // Send suggestion 1 only + EXPECT_CALL(observer, OnNewSuggestions(articles_category)); + provider1->FireSuggestionsChanged(articles_category, + CreateSuggestions(articles_category, {1})); + ExpectThatSuggestionsAre(articles_category, {1}); + ExpectThatSuggestionsAre(offline_pages_category, {13, 14}); + Mock::VerifyAndClearExpectations(&observer); + + // provider2 reports BOOKMARKS as unavailable + EXPECT_CALL(observer, OnCategoryStatusChanged( + offline_pages_category, + CategoryStatus::CATEGORY_EXPLICITLY_DISABLED)); + provider2->FireCategoryStatusChanged( + offline_pages_category, CategoryStatus::CATEGORY_EXPLICITLY_DISABLED); + EXPECT_THAT(service()->GetCategoryStatus(articles_category), + Eq(CategoryStatus::AVAILABLE)); + EXPECT_THAT(service()->GetCategoryStatus(offline_pages_category), + Eq(CategoryStatus::CATEGORY_EXPLICITLY_DISABLED)); + ExpectThatSuggestionsAre(articles_category, {1}); + ExpectThatSuggestionsAre(offline_pages_category, std::vector<int>()); + Mock::VerifyAndClearExpectations(&observer); + + // Shutdown the service + EXPECT_CALL(observer, ContentSuggestionsServiceShutdown()); + service()->Shutdown(); + service()->RemoveObserver(&observer); + // The service will receive two Shutdown() calls. +} + +TEST_F(ContentSuggestionsServiceTest, + ShouldNotReturnCategoryInfoForNonexistentCategory) { + Category category = FromKnownCategory(KnownCategories::DOWNLOADS); + base::Optional<CategoryInfo> result = service()->GetCategoryInfo(category); + EXPECT_FALSE(result.has_value()); +} + +TEST_F(ContentSuggestionsServiceTest, ShouldReturnCategoryInfo) { + Category category = FromKnownCategory(KnownCategories::DOWNLOADS); + MockProvider* provider = RegisterProvider(category); + provider->FireCategoryStatusChangedWithCurrentStatus(category); + base::Optional<CategoryInfo> result = service()->GetCategoryInfo(category); + ASSERT_TRUE(result.has_value()); + CategoryInfo expected = provider->GetCategoryInfo(category); + const CategoryInfo& actual = result.value(); + EXPECT_THAT(expected.title(), Eq(actual.title())); + EXPECT_THAT(expected.card_layout(), Eq(actual.card_layout())); + EXPECT_THAT(expected.has_more_button(), Eq(actual.has_more_button())); +} + +TEST_F(ContentSuggestionsServiceTest, + ShouldRegisterNewCategoryOnNewSuggestions) { + Category category = FromKnownCategory(KnownCategories::DOWNLOADS); + MockProvider* provider = RegisterProvider(category); + provider->FireCategoryStatusChangedWithCurrentStatus(category); + MockServiceObserver observer; + service()->AddObserver(&observer); + + // Provider starts providing |new_category| without calling + // |OnCategoryStatusChanged|. This is supported for now until further + // reconsideration. + Category new_category = FromKnownCategory(KnownCategories::ARTICLES); + provider->SetProvidedCategories( + std::vector<Category>({category, new_category})); + + EXPECT_CALL(observer, OnNewSuggestions(new_category)); + EXPECT_CALL(observer, + OnCategoryStatusChanged(new_category, CategoryStatus::AVAILABLE)); + provider->FireSuggestionsChanged(new_category, + CreateSuggestions(new_category, {1, 2})); + + ExpectThatSuggestionsAre(new_category, {1, 2}); + ASSERT_THAT(providers().count(category), Eq(1ul)); + EXPECT_THAT(providers().at(category), Eq(provider)); + EXPECT_THAT(service()->GetCategoryStatus(category), + Eq(CategoryStatus::AVAILABLE)); + ASSERT_THAT(providers().count(new_category), Eq(1ul)); + EXPECT_THAT(providers().at(new_category), Eq(provider)); + EXPECT_THAT(service()->GetCategoryStatus(new_category), + Eq(CategoryStatus::AVAILABLE)); + + service()->RemoveObserver(&observer); +} + +TEST_F(ContentSuggestionsServiceTest, + ShouldRegisterNewCategoryOnCategoryStatusChanged) { + Category category = FromKnownCategory(KnownCategories::DOWNLOADS); + MockProvider* provider = RegisterProvider(category); + provider->FireCategoryStatusChangedWithCurrentStatus(category); + MockServiceObserver observer; + service()->AddObserver(&observer); + + // Provider starts providing |new_category| and calls + // |OnCategoryStatusChanged|, but the category is not yet available. + Category new_category = FromKnownCategory(KnownCategories::ARTICLES); + provider->SetProvidedCategories( + std::vector<Category>({category, new_category})); + EXPECT_CALL(observer, OnCategoryStatusChanged(new_category, + CategoryStatus::INITIALIZING)); + provider->FireCategoryStatusChanged(new_category, + CategoryStatus::INITIALIZING); + + ASSERT_THAT(providers().count(new_category), Eq(1ul)); + EXPECT_THAT(providers().at(new_category), Eq(provider)); + ExpectThatSuggestionsAre(new_category, std::vector<int>()); + EXPECT_THAT(service()->GetCategoryStatus(new_category), + Eq(CategoryStatus::INITIALIZING)); + EXPECT_THAT(service()->GetCategories(), + Eq(std::vector<Category>({category, new_category}))); + + service()->RemoveObserver(&observer); +} + +TEST_F(ContentSuggestionsServiceTest, ShouldRemoveCategoryWhenNotProvided) { + Category category = FromKnownCategory(KnownCategories::DOWNLOADS); + MockProvider* provider = RegisterProvider(category); + MockServiceObserver observer; + service()->AddObserver(&observer); + + provider->FireSuggestionsChanged(category, + CreateSuggestions(category, {1, 2})); + ExpectThatSuggestionsAre(category, {1, 2}); + + EXPECT_CALL(observer, + OnCategoryStatusChanged(category, CategoryStatus::NOT_PROVIDED)); + provider->FireCategoryStatusChanged(category, CategoryStatus::NOT_PROVIDED); + + EXPECT_THAT(service()->GetCategoryStatus(category), + Eq(CategoryStatus::NOT_PROVIDED)); + EXPECT_TRUE(service()->GetCategories().empty()); + ExpectThatSuggestionsAre(category, std::vector<int>()); + + service()->RemoveObserver(&observer); +} + +// This tests the temporary special-casing of the bookmarks section: If it is +// empty, it should appear at the end; see crbug.com/640568. +TEST_F(ContentSuggestionsServiceTest, ShouldPutBookmarksAtEndIfEmpty) { + // Register a bookmarks provider and an arbitrary remote provider. + Category bookmarks = FromKnownCategory(KnownCategories::BOOKMARKS); + MockProvider* bookmarks_provider = RegisterProvider(bookmarks); + bookmarks_provider->FireCategoryStatusChangedWithCurrentStatus(bookmarks); + Category remote = FromRemoteCategory(123); + MockProvider* remote_provider = RegisterProvider(remote); + remote_provider->FireCategoryStatusChangedWithCurrentStatus(remote); + + // By default, the bookmarks category is empty, so it should be at the end. + EXPECT_THAT(service()->GetCategories(), ElementsAre(remote, bookmarks)); + + // Add two bookmark suggestions; now bookmarks should be in the front. + bookmarks_provider->FireSuggestionsChanged( + bookmarks, CreateSuggestions(bookmarks, {1, 2})); + EXPECT_THAT(service()->GetCategories(), ElementsAre(bookmarks, remote)); + // Dismiss the first suggestion; bookmarks should stay in the front. + service()->DismissSuggestion(CreateSuggestion(bookmarks, 1).id()); + EXPECT_THAT(service()->GetCategories(), ElementsAre(bookmarks, remote)); + // Dismiss the second suggestion; now bookmarks should go back to the end. + service()->DismissSuggestion(CreateSuggestion(bookmarks, 2).id()); + EXPECT_THAT(service()->GetCategories(), ElementsAre(remote, bookmarks)); + + // Same thing, but invalidate instead of dismissing. + bookmarks_provider->FireSuggestionsChanged( + bookmarks, CreateSuggestions(bookmarks, {1, 2})); + EXPECT_THAT(service()->GetCategories(), ElementsAre(bookmarks, remote)); + bookmarks_provider->FireSuggestionInvalidated( + ContentSuggestion::ID(bookmarks, "1")); + EXPECT_THAT(service()->GetCategories(), ElementsAre(bookmarks, remote)); + bookmarks_provider->FireSuggestionInvalidated( + ContentSuggestion::ID(bookmarks, "2")); + EXPECT_THAT(service()->GetCategories(), ElementsAre(remote, bookmarks)); + + // Same thing, but now the bookmarks category updates "naturally". + bookmarks_provider->FireSuggestionsChanged( + bookmarks, CreateSuggestions(bookmarks, {1, 2})); + EXPECT_THAT(service()->GetCategories(), ElementsAre(bookmarks, remote)); + bookmarks_provider->FireSuggestionsChanged(bookmarks, + CreateSuggestions(bookmarks, {1})); + EXPECT_THAT(service()->GetCategories(), ElementsAre(bookmarks, remote)); + bookmarks_provider->FireSuggestionsChanged( + bookmarks, CreateSuggestions(bookmarks, std::vector<int>())); + EXPECT_THAT(service()->GetCategories(), ElementsAre(remote, bookmarks)); +} + +TEST_F(ContentSuggestionsServiceTest, ShouldForwardClearHistory) { + Category category = FromKnownCategory(KnownCategories::DOWNLOADS); + MockProvider* provider = RegisterProvider(category); + base::Time begin = base::Time::FromTimeT(123), + end = base::Time::FromTimeT(456); + EXPECT_CALL(*provider, ClearHistory(begin, end, _)); + base::Callback<bool(const GURL& url)> filter; + service()->ClearHistory(begin, end, filter); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/features.cc b/chromium/components/ntp_snippets/features.cc new file mode 100644 index 00000000000..7baf3d7bc0a --- /dev/null +++ b/chromium/components/ntp_snippets/features.cc @@ -0,0 +1,55 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/features.h" + +#include "base/strings/string_number_conversions.h" +#include "components/variations/variations_associated_data.h" + +namespace ntp_snippets { + +const base::Feature kArticleSuggestionsFeature{ + "NTPArticleSuggestions", base::FEATURE_ENABLED_BY_DEFAULT}; + +const base::Feature kBookmarkSuggestionsFeature{ + "NTPBookmarkSuggestions", base::FEATURE_ENABLED_BY_DEFAULT}; + +const base::Feature kRecentOfflineTabSuggestionsFeature{ + "NTPOfflinePageSuggestions", base::FEATURE_DISABLED_BY_DEFAULT}; + +const base::Feature kSaveToOfflineFeature{ + "NTPSaveToOffline", base::FEATURE_DISABLED_BY_DEFAULT}; + +const base::Feature kDownloadSuggestionsFeature{ + "NTPDownloadSuggestions", base::FEATURE_DISABLED_BY_DEFAULT}; + +const base::Feature kPhysicalWebPageSuggestionsFeature{ + "NTPPhysicalWebPageSuggestions", base::FEATURE_DISABLED_BY_DEFAULT}; + +const base::Feature kContentSuggestionsFeature{ + "NTPSnippets", base::FEATURE_ENABLED_BY_DEFAULT}; + +const base::Feature kForeignSessionsSuggestionsFeature{ + "NTPForeignSessionsSuggestions", base::FEATURE_DISABLED_BY_DEFAULT}; + +int GetParamAsInt(const base::Feature& feature, + const std::string& param_name, + const int default_value) { + std::string value_as_string = + variations::GetVariationParamValueByFeature(feature, param_name); + int value_as_int = 0; + if (!base::StringToInt(value_as_string, &value_as_int)) { + if (!value_as_string.empty()) { + LOG(WARNING) << "Failed to parse variation param " << param_name + << " with string value " << value_as_string + << " under feature " << feature.name + << " into an int. Falling back to default value of " + << default_value; + } + value_as_int = default_value; + } + return value_as_int; +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/features.h b/chromium/components/ntp_snippets/features.h new file mode 100644 index 00000000000..a3b16089253 --- /dev/null +++ b/chromium/components/ntp_snippets/features.h @@ -0,0 +1,37 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_FEATURES_H_ +#define COMPONENTS_NTP_SNIPPETS_FEATURES_H_ + +#include <string> + +#include "base/feature_list.h" + +namespace ntp_snippets { + +// Features to turn individual providers/categories on/off. +extern const base::Feature kArticleSuggestionsFeature; +extern const base::Feature kBookmarkSuggestionsFeature; +extern const base::Feature kRecentOfflineTabSuggestionsFeature; +extern const base::Feature kDownloadSuggestionsFeature; +extern const base::Feature kPhysicalWebPageSuggestionsFeature; +extern const base::Feature kForeignSessionsSuggestionsFeature; + +// Feature to allow the 'save to offline' option to appear in the snippets +// context menu. +extern const base::Feature kSaveToOfflineFeature; + +// Global toggle for the whole content suggestions feature. If this is set to +// false, all the per-provider features are ignored. +extern const base::Feature kContentSuggestionsFeature; + +// Returns a feature param as an int instead of a string. +int GetParamAsInt(const base::Feature& feature, + const std::string& param_name, + int default_value); + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_FEATURES_H_ diff --git a/chromium/components/ntp_snippets/mock_content_suggestions_provider_observer.cc b/chromium/components/ntp_snippets/mock_content_suggestions_provider_observer.cc new file mode 100644 index 00000000000..38f97191106 --- /dev/null +++ b/chromium/components/ntp_snippets/mock_content_suggestions_provider_observer.cc @@ -0,0 +1,26 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/mock_content_suggestions_provider_observer.h" + +namespace ntp_snippets { + +MockContentSuggestionsProviderObserver:: + MockContentSuggestionsProviderObserver() = default; + +MockContentSuggestionsProviderObserver:: + ~MockContentSuggestionsProviderObserver() = default; + +void MockContentSuggestionsProviderObserver::OnNewSuggestions( + ContentSuggestionsProvider* provider, + Category category, + std::vector<ContentSuggestion> suggestions) { + std::list<ContentSuggestion> suggestions_list; + for (ContentSuggestion& suggestion : suggestions) { + suggestions_list.push_back(std::move(suggestion)); + } + OnNewSuggestions(provider, category, suggestions_list); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/mock_content_suggestions_provider_observer.h b/chromium/components/ntp_snippets/mock_content_suggestions_provider_observer.h new file mode 100644 index 00000000000..12bb1fdc55b --- /dev/null +++ b/chromium/components/ntp_snippets/mock_content_suggestions_provider_observer.h @@ -0,0 +1,47 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_MOCK_CONTENT_SUGGESTIONS_PROVIDER_OBSERVER_H_ +#define COMPONENTS_NTP_SNIPPETS_MOCK_CONTENT_SUGGESTIONS_PROVIDER_OBSERVER_H_ + +#include <list> +#include <vector> + +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "testing/gmock/include/gmock/gmock.h" + +namespace ntp_snippets { + +class MockContentSuggestionsProviderObserver + : public ContentSuggestionsProvider::Observer { + public: + MockContentSuggestionsProviderObserver(); + ~MockContentSuggestionsProviderObserver(); + + // Call of this function is redirected to the mock function OnNewSuggestions + // which takes const list of suggestions. We do this trick so that the + // MOCK_METHOD behaves the same way in tests as the actual method and we can + // keep this gMock issue limited to the mock class. MOCK_METHOD cannot be + // applied here directly, since gMock does not support movable-only types + // such as ContentSuggestion. + void OnNewSuggestions(ContentSuggestionsProvider* provider, + Category category, + std::vector<ContentSuggestion> suggestions) override; + + MOCK_METHOD3(OnNewSuggestions, + void(ContentSuggestionsProvider* provider, + Category category, + const std::list<ContentSuggestion>& suggestions)); + MOCK_METHOD3(OnCategoryStatusChanged, + void(ContentSuggestionsProvider* provider, + Category category, + CategoryStatus new_status)); + MOCK_METHOD2(OnSuggestionInvalidated, + void(ContentSuggestionsProvider* provider, + const ContentSuggestion::ID& suggestion_id)); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_MOCK_CONTENT_SUGGESTIONS_PROVIDER_OBSERVER_H_ diff --git a/chromium/components/ntp_snippets/ntp_snippet_unittest.cc b/chromium/components/ntp_snippets/ntp_snippet_unittest.cc deleted file mode 100644 index 6e254c66929..00000000000 --- a/chromium/components/ntp_snippets/ntp_snippet_unittest.cc +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/ntp_snippets/ntp_snippet.h" - -#include "base/json/json_reader.h" -#include "base/values.h" -#include "testing/gmock/include/gmock/gmock.h" -#include "testing/gtest/include/gtest/gtest.h" - -namespace ntp_snippets { -namespace { - -TEST(NTPSnippetTest, FromChromeContentSuggestionsDictionary) { - const std::string kJsonStr = - "{" - " \"id\" : [\"http://localhost/foobar\"]," - " \"title\" : \"Foo Barred from Baz\"," - " \"summaryText\" : \"...\"," - " \"fullPageUrl\" : \"http://localhost/foobar\"," - " \"publishTime\" : \"2016-06-30T11:01:37.000Z\"," - " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," - " \"publisherName\" : \"Foo News\"," - " \"imageUrl\" : \"http://localhost/foobar.jpg\"," - " \"ampUrl\" : \"http://localhost/amp\"," - " \"faviconUrl\" : \"http://localhost/favicon.ico\" " - "}"; - auto json_value = base::JSONReader::Read(kJsonStr); - base::DictionaryValue* json_dict; - ASSERT_TRUE(json_value->GetAsDictionary(&json_dict)); - - auto snippet = NTPSnippet::CreateFromContentSuggestionsDictionary(*json_dict); - ASSERT_THAT(snippet, testing::NotNull()); - - EXPECT_EQ(snippet->id(), "http://localhost/foobar"); - EXPECT_EQ(snippet->title(), "Foo Barred from Baz"); - EXPECT_EQ(snippet->snippet(), "..."); - EXPECT_EQ(snippet->salient_image_url(), GURL("http://localhost/foobar.jpg")); - auto unix_publish_date = snippet->publish_date() - base::Time::UnixEpoch(); - auto expiry_duration = snippet->expiry_date() - snippet->publish_date(); - EXPECT_FLOAT_EQ(unix_publish_date.InSecondsF(), 1467284497.000000f); - EXPECT_FLOAT_EQ(expiry_duration.InSecondsF(), 86400.000000f); - - EXPECT_EQ(snippet->best_source().publisher_name, "Foo News"); - EXPECT_EQ(snippet->best_source().url, GURL("http://localhost/foobar")); - EXPECT_EQ(snippet->best_source().amp_url, GURL("http://localhost/amp")); -} - -} // namespace -} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_constants.cc b/chromium/components/ntp_snippets/ntp_snippets_constants.cc index 5ee43fbb473..4b92ddd8f22 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_constants.cc +++ b/chromium/components/ntp_snippets/ntp_snippets_constants.cc @@ -12,4 +12,13 @@ const char kStudyName[] = "NTPSnippets"; const base::FilePath::CharType kDatabaseFolder[] = FILE_PATH_LITERAL("NTPSnippets"); +const char kChromeReaderServer[] = + "https://chromereader-pa.googleapis.com/v1/fetch"; +const char kContentSuggestionsServer[] = + "https://chromecontentsuggestions-pa.googleapis.com/v1/suggestions/fetch"; +const char kContentSuggestionsDevServer[] = + "https://dev-chromecontentsuggestions-pa.googleapis.com/v1/suggestions/fetch"; +const char kContentSuggestionsAlphaServer[] = + "https://alpha-chromecontentsuggestions-pa.sandbox.googleapis.com/v1/suggestions/fetch"; + } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_constants.h b/chromium/components/ntp_snippets/ntp_snippets_constants.h index 0bb403cba5d..9c39975ff43 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_constants.h +++ b/chromium/components/ntp_snippets/ntp_snippets_constants.h @@ -17,6 +17,12 @@ extern const char kStudyName[]; // profile path. extern const base::FilePath::CharType kDatabaseFolder[]; +// Server endpoints for fetching snippets. +extern const char kChromeReaderServer[]; // old endpoint +extern const char kContentSuggestionsServer[]; // new, used on stable/beta +extern const char kContentSuggestionsDevServer[]; // new, used on dev/canary +extern const char kContentSuggestionsAlphaServer[]; // new, for testing + } // namespace ntp_snippets #endif diff --git a/chromium/components/ntp_snippets/ntp_snippets_scheduler.h b/chromium/components/ntp_snippets/ntp_snippets_scheduler.h deleted file mode 100644 index d46e03b182e..00000000000 --- a/chromium/components/ntp_snippets/ntp_snippets_scheduler.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_SCHEDULER_H_ -#define COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_SCHEDULER_H_ - -#include "base/macros.h" -#include "base/time/time.h" - -namespace ntp_snippets { - -// Interface to schedule the periodic fetching of snippets. -class NTPSnippetsScheduler { - public: - // Schedule periodic fetching of snippets, with different period depending on - // network and charging state, and also set up a delay after which the periods - // may change. The concrete implementation should call - // NTPSnippetsService::FetchSnippets once per period, and - // NTPSnippetsService::RescheduleFetching at |reschedule_time|. - // Any of the values can be zero to indicate that the corresponding task - // should not be scheduled. - virtual bool Schedule(base::TimeDelta period_wifi_charging, - base::TimeDelta period_wifi, - base::TimeDelta period_fallback, - base::Time reschedule_time) = 0; - - // Cancel any scheduled tasks. - virtual bool Unschedule() = 0; - - protected: - NTPSnippetsScheduler() = default; - - private: - DISALLOW_COPY_AND_ASSIGN(NTPSnippetsScheduler); -}; - -} // namespace ntp_snippets - -#endif // COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_SCHEDULER_H_ diff --git a/chromium/components/ntp_snippets/ntp_snippets_service.cc b/chromium/components/ntp_snippets/ntp_snippets_service.cc deleted file mode 100644 index 08f4203ee24..00000000000 --- a/chromium/components/ntp_snippets/ntp_snippets_service.cc +++ /dev/null @@ -1,766 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/ntp_snippets/ntp_snippets_service.h" - -#include <algorithm> -#include <iterator> -#include <utility> - -#include "base/command_line.h" -#include "base/files/file_path.h" -#include "base/files/file_util.h" -#include "base/location.h" -#include "base/metrics/histogram_macros.h" -#include "base/metrics/sparse_histogram.h" -#include "base/path_service.h" -#include "base/strings/string_number_conversions.h" -#include "base/task_runner_util.h" -#include "base/time/time.h" -#include "base/values.h" -#include "components/image_fetcher/image_decoder.h" -#include "components/image_fetcher/image_fetcher.h" -#include "components/ntp_snippets/ntp_snippets_constants.h" -#include "components/ntp_snippets/ntp_snippets_database.h" -#include "components/ntp_snippets/pref_names.h" -#include "components/ntp_snippets/switches.h" -#include "components/prefs/pref_registry_simple.h" -#include "components/prefs/pref_service.h" -#include "components/suggestions/proto/suggestions.pb.h" -#include "components/variations/variations_associated_data.h" -#include "ui/gfx/image/image.h" - -using image_fetcher::ImageDecoder; -using image_fetcher::ImageFetcher; -using suggestions::ChromeSuggestion; -using suggestions::SuggestionsProfile; -using suggestions::SuggestionsService; - -namespace ntp_snippets { - -namespace { - -// Number of snippets requested to the server. Consider replacing sparse UMA -// histograms with COUNTS() if this number increases beyond 50. -const int kMaxSnippetCount = 10; - -// Default values for snippets fetching intervals. -const int kDefaultFetchingIntervalWifiChargingSeconds = 30 * 60; -const int kDefaultFetchingIntervalWifiSeconds = 2 * 60 * 60; -const int kDefaultFetchingIntervalFallbackSeconds = 24 * 60 * 60; - -// Variation parameters than can override the default fetching intervals. -const char kFetchingIntervalWifiChargingParamName[] = - "fetching_interval_wifi_charging_seconds"; -const char kFetchingIntervalWifiParamName[] = - "fetching_interval_wifi_seconds"; -const char kFetchingIntervalFallbackParamName[] = - "fetching_interval_fallback_seconds"; - -// These define the times of day during which we will fetch via Wifi (without -// charging) - 6 AM to 10 PM. -const int kWifiFetchingHourMin = 6; -const int kWifiFetchingHourMax = 22; - -const int kDefaultExpiryTimeMins = 24 * 60; - -base::TimeDelta GetFetchingInterval(const char* switch_name, - const char* param_name, - int default_value_seconds) { - int value_seconds = default_value_seconds; - - // The default value can be overridden by a variation parameter. - // TODO(treib,jkrcal): Use GetVariationParamValueByFeature and get rid of - // kStudyName, also in NTPSnippetsFetcher. - std::string param_value_str = variations::GetVariationParamValue( - ntp_snippets::kStudyName, param_name); - if (!param_value_str.empty()) { - int param_value_seconds = 0; - if (base::StringToInt(param_value_str, ¶m_value_seconds)) - value_seconds = param_value_seconds; - else - LOG(WARNING) << "Invalid value for variation parameter " << param_name; - } - - // A value from the command line parameter overrides anything else. - const base::CommandLine& cmdline = *base::CommandLine::ForCurrentProcess(); - if (cmdline.HasSwitch(switch_name)) { - std::string str = cmdline.GetSwitchValueASCII(switch_name); - int switch_value_seconds = 0; - if (base::StringToInt(str, &switch_value_seconds)) - value_seconds = switch_value_seconds; - else - LOG(WARNING) << "Invalid value for switch " << switch_name; - } - return base::TimeDelta::FromSeconds(value_seconds); -} - -base::TimeDelta GetFetchingIntervalWifiCharging() { - return GetFetchingInterval(switches::kFetchingIntervalWifiChargingSeconds, - kFetchingIntervalWifiChargingParamName, - kDefaultFetchingIntervalWifiChargingSeconds); -} - -base::TimeDelta GetFetchingIntervalWifi(const base::Time& now) { - // Only fetch via Wifi (without charging) during the proper times of day. - base::Time::Exploded exploded; - now.LocalExplode(&exploded); - if (kWifiFetchingHourMin <= exploded.hour && - exploded.hour < kWifiFetchingHourMax) { - return GetFetchingInterval(switches::kFetchingIntervalWifiSeconds, - kFetchingIntervalWifiParamName, - kDefaultFetchingIntervalWifiSeconds); - } - return base::TimeDelta(); -} - -base::TimeDelta GetFetchingIntervalFallback() { - return GetFetchingInterval(switches::kFetchingIntervalFallbackSeconds, - kFetchingIntervalFallbackParamName, - kDefaultFetchingIntervalFallbackSeconds); -} - -base::Time GetRescheduleTime(const base::Time& now) { - base::Time::Exploded exploded; - now.LocalExplode(&exploded); - // The scheduling changes at both |kWifiFetchingHourMin| and - // |kWifiFetchingHourMax|. Find the time of the next one that we'll hit. - bool next_day = false; - if (exploded.hour < kWifiFetchingHourMin) { - exploded.hour = kWifiFetchingHourMin; - } else if (exploded.hour < kWifiFetchingHourMax) { - exploded.hour = kWifiFetchingHourMax; - } else { - next_day = true; - exploded.hour = kWifiFetchingHourMin; - } - // In any case, reschedule at the full hour. - exploded.minute = 0; - exploded.second = 0; - exploded.millisecond = 0; - base::Time reschedule = base::Time::FromLocalExploded(exploded); - if (next_day) - reschedule += base::TimeDelta::FromDays(1); - - return reschedule; -} - -// Extracts the hosts from |suggestions| and returns them in a set. -std::set<std::string> GetSuggestionsHostsImpl( - const SuggestionsProfile& suggestions) { - std::set<std::string> hosts; - for (int i = 0; i < suggestions.suggestions_size(); ++i) { - const ChromeSuggestion& suggestion = suggestions.suggestions(i); - GURL url(suggestion.url()); - if (url.is_valid()) - hosts.insert(url.host()); - } - return hosts; -} - -void InsertAllIDs(const NTPSnippet::PtrVector& snippets, - std::set<std::string>* ids) { - for (const std::unique_ptr<NTPSnippet>& snippet : snippets) { - ids->insert(snippet->id()); - for (const SnippetSource& source : snippet->sources()) - ids->insert(source.url.spec()); - } -} - -void Compact(NTPSnippet::PtrVector* snippets) { - snippets->erase( - std::remove_if( - snippets->begin(), snippets->end(), - [](const std::unique_ptr<NTPSnippet>& snippet) { return !snippet; }), - snippets->end()); -} - -} // namespace - -NTPSnippetsService::NTPSnippetsService( - bool enabled, - PrefService* pref_service, - SuggestionsService* suggestions_service, - const std::string& application_language_code, - NTPSnippetsScheduler* scheduler, - std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher, - std::unique_ptr<ImageFetcher> image_fetcher, - std::unique_ptr<ImageDecoder> image_decoder, - std::unique_ptr<NTPSnippetsDatabase> database, - std::unique_ptr<NTPSnippetsStatusService> status_service) - : state_(State::NOT_INITED), - pref_service_(pref_service), - suggestions_service_(suggestions_service), - application_language_code_(application_language_code), - scheduler_(scheduler), - snippets_fetcher_(std::move(snippets_fetcher)), - image_fetcher_(std::move(image_fetcher)), - image_decoder_(std::move(image_decoder)), - database_(std::move(database)), - snippets_status_service_(std::move(status_service)), - fetch_after_load_(false) { - // TODO(dgn) should be removed after branch point (https://crbug.com/617585). - ClearDeprecatedPrefs(); - - if (!enabled || database_->IsErrorState()) { - // Don't even bother loading the database. - EnterState(State::SHUT_DOWN); - return; - } - - database_->SetErrorCallback(base::Bind(&NTPSnippetsService::OnDatabaseError, - base::Unretained(this))); - - // We transition to other states while finalizing the initialization, when the - // database is done loading. - database_->LoadSnippets(base::Bind(&NTPSnippetsService::OnDatabaseLoaded, - base::Unretained(this))); -} - -NTPSnippetsService::~NTPSnippetsService() { - DCHECK(state_ == State::SHUT_DOWN); -} - -// static -void NTPSnippetsService::RegisterProfilePrefs(PrefRegistrySimple* registry) { - registry->RegisterListPref(prefs::kDeprecatedSnippets); - registry->RegisterListPref(prefs::kDeprecatedDiscardedSnippets); - registry->RegisterListPref(prefs::kSnippetHosts); -} - -// Inherited from KeyedService. -void NTPSnippetsService::Shutdown() { - EnterState(State::SHUT_DOWN); -} - -void NTPSnippetsService::FetchSnippets() { - if (ready()) - FetchSnippetsFromHosts(GetSuggestionsHosts()); - else - fetch_after_load_ = true; -} - -void NTPSnippetsService::FetchSnippetsFromHosts( - const std::set<std::string>& hosts) { - if (!ready()) - return; - snippets_fetcher_->FetchSnippetsFromHosts(hosts, application_language_code_, - kMaxSnippetCount); -} - -void NTPSnippetsService::RescheduleFetching() { - // The scheduler only exists on Android so far, it's null on other platforms. - if (!scheduler_) - return; - - if (ready()) { - base::Time now = base::Time::Now(); - scheduler_->Schedule( - GetFetchingIntervalWifiCharging(), GetFetchingIntervalWifi(now), - GetFetchingIntervalFallback(), GetRescheduleTime(now)); - } else { - scheduler_->Unschedule(); - } -} - -void NTPSnippetsService::FetchSnippetImage( - const std::string& snippet_id, - const ImageFetchedCallback& callback) { - database_->LoadImage( - snippet_id, - base::Bind(&NTPSnippetsService::OnSnippetImageFetchedFromDatabase, - base::Unretained(this), snippet_id, callback)); -} - -void NTPSnippetsService::ClearSnippets() { - if (!initialized()) - return; - - if (snippets_.empty()) - return; - - database_->DeleteSnippets(snippets_); - snippets_.clear(); - - FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_, - NTPSnippetsServiceLoaded()); -} - -std::set<std::string> NTPSnippetsService::GetSuggestionsHosts() const { - // |suggestions_service_| can be null in tests. - if (!suggestions_service_) - return std::set<std::string>(); - - // TODO(treib): This should just call GetSnippetHostsFromPrefs. - return GetSuggestionsHostsImpl( - suggestions_service_->GetSuggestionsDataFromCache()); -} - -bool NTPSnippetsService::DiscardSnippet(const std::string& snippet_id) { - if (!ready()) - return false; - - auto it = - std::find_if(snippets_.begin(), snippets_.end(), - [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) { - return snippet->id() == snippet_id; - }); - if (it == snippets_.end()) - return false; - - (*it)->set_discarded(true); - - database_->SaveSnippet(**it); - database_->DeleteImage((*it)->id()); - - discarded_snippets_.push_back(std::move(*it)); - snippets_.erase(it); - - FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_, - NTPSnippetsServiceLoaded()); - return true; -} - -void NTPSnippetsService::ClearDiscardedSnippets() { - if (!initialized()) - return; - - if (discarded_snippets_.empty()) - return; - - database_->DeleteSnippets(discarded_snippets_); - discarded_snippets_.clear(); -} - -void NTPSnippetsService::AddObserver(NTPSnippetsServiceObserver* observer) { - observers_.AddObserver(observer); -} - -void NTPSnippetsService::RemoveObserver(NTPSnippetsServiceObserver* observer) { - observers_.RemoveObserver(observer); -} - -// static -int NTPSnippetsService::GetMaxSnippetCountForTesting() { - return kMaxSnippetCount; -} - -//////////////////////////////////////////////////////////////////////////////// -// Private methods - -// image_fetcher::ImageFetcherDelegate implementation. -void NTPSnippetsService::OnImageDataFetched(const std::string& snippet_id, - const std::string& image_data) { - if (image_data.empty()) - return; - - // Only save the image if the corresponding snippet still exists. - auto it = - std::find_if(snippets_.begin(), snippets_.end(), - [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) { - return snippet->id() == snippet_id; - }); - if (it == snippets_.end()) - return; - - database_->SaveImage(snippet_id, image_data); -} - -void NTPSnippetsService::OnDatabaseLoaded(NTPSnippet::PtrVector snippets) { - DCHECK(state_ == State::NOT_INITED || state_ == State::SHUT_DOWN); - if (state_ == State::SHUT_DOWN) - return; - - DCHECK(snippets_.empty()); - DCHECK(discarded_snippets_.empty()); - for (std::unique_ptr<NTPSnippet>& snippet : snippets) { - if (snippet->is_discarded()) - discarded_snippets_.emplace_back(std::move(snippet)); - else - snippets_.emplace_back(std::move(snippet)); - } - std::sort(snippets_.begin(), snippets_.end(), - [](const std::unique_ptr<NTPSnippet>& lhs, - const std::unique_ptr<NTPSnippet>& rhs) { - return lhs->score() > rhs->score(); - }); - - ClearExpiredSnippets(); - FinishInitialization(); -} - -void NTPSnippetsService::OnDatabaseError() { - EnterState(State::SHUT_DOWN); -} - -void NTPSnippetsService::OnSuggestionsChanged( - const SuggestionsProfile& suggestions) { - DCHECK(initialized()); - - std::set<std::string> hosts = GetSuggestionsHostsImpl(suggestions); - if (hosts == GetSnippetHostsFromPrefs()) - return; - - // Remove existing snippets that aren't in the suggestions anymore. - // TODO(treib,maybelle): If there is another source with an allowed host, - // then we should fall back to that. - // First, move them over into |to_delete|. - NTPSnippet::PtrVector to_delete; - for (std::unique_ptr<NTPSnippet>& snippet : snippets_) { - if (!hosts.count(snippet->best_source().url.host())) - to_delete.emplace_back(std::move(snippet)); - } - Compact(&snippets_); - // Then delete the removed snippets from the database. - database_->DeleteSnippets(to_delete); - - StoreSnippetHostsToPrefs(hosts); - - FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_, - NTPSnippetsServiceLoaded()); - - FetchSnippetsFromHosts(hosts); -} - -void NTPSnippetsService::OnFetchFinished( - NTPSnippetsFetcher::OptionalSnippets snippets) { - if (!ready()) - return; - - if (snippets) { - // Sparse histogram used because the number of snippets is small (bound by - // kMaxSnippetCount). - DCHECK_LE(snippets->size(), static_cast<size_t>(kMaxSnippetCount)); - UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticlesFetched", - snippets->size()); - MergeSnippets(std::move(*snippets)); - } - - ClearExpiredSnippets(); - - // If there are more snippets than we want to show, delete the extra ones. - if (snippets_.size() > kMaxSnippetCount) { - NTPSnippet::PtrVector to_delete( - std::make_move_iterator(snippets_.begin() + kMaxSnippetCount), - std::make_move_iterator(snippets_.end())); - snippets_.resize(kMaxSnippetCount); - database_->DeleteSnippets(to_delete); - } - - UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticles", - snippets_.size()); - if (snippets_.empty() && !discarded_snippets_.empty()) { - UMA_HISTOGRAM_COUNTS("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded", - discarded_snippets_.size()); - } - - FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_, - NTPSnippetsServiceLoaded()); -} - -void NTPSnippetsService::MergeSnippets(NTPSnippet::PtrVector new_snippets) { - DCHECK(ready()); - - // Remove new snippets that we already have, or that have been discarded. - std::set<std::string> old_snippet_ids; - InsertAllIDs(discarded_snippets_, &old_snippet_ids); - InsertAllIDs(snippets_, &old_snippet_ids); - new_snippets.erase( - std::remove_if( - new_snippets.begin(), new_snippets.end(), - [&old_snippet_ids](const std::unique_ptr<NTPSnippet>& snippet) { - if (old_snippet_ids.count(snippet->id())) - return true; - for (const SnippetSource& source : snippet->sources()) { - if (old_snippet_ids.count(source.url.spec())) - return true; - } - return false; - }), - new_snippets.end()); - - // Fill in default publish/expiry dates where required. - for (std::unique_ptr<NTPSnippet>& snippet : new_snippets) { - if (snippet->publish_date().is_null()) - snippet->set_publish_date(base::Time::Now()); - if (snippet->expiry_date().is_null()) { - snippet->set_expiry_date( - snippet->publish_date() + - base::TimeDelta::FromMinutes(kDefaultExpiryTimeMins)); - } - - // TODO(treib): Prefetch and cache the snippet image. crbug.com/605870 - } - - if (!base::CommandLine::ForCurrentProcess()->HasSwitch( - switches::kAddIncompleteSnippets)) { - int num_new_snippets = new_snippets.size(); - // Remove snippets that do not have all the info we need to display it to - // the user. - new_snippets.erase( - std::remove_if(new_snippets.begin(), new_snippets.end(), - [](const std::unique_ptr<NTPSnippet>& snippet) { - return !snippet->is_complete(); - }), - new_snippets.end()); - int num_snippets_discarded = num_new_snippets - new_snippets.size(); - UMA_HISTOGRAM_BOOLEAN("NewTabPage.Snippets.IncompleteSnippetsAfterFetch", - num_snippets_discarded > 0); - if (num_snippets_discarded > 0) { - UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumIncompleteSnippets", - num_snippets_discarded); - } - } - - // Save the new snippets to the DB. - database_->SaveSnippets(new_snippets); - - // Insert the new snippets at the front. - snippets_.insert(snippets_.begin(), - std::make_move_iterator(new_snippets.begin()), - std::make_move_iterator(new_snippets.end())); -} - -std::set<std::string> NTPSnippetsService::GetSnippetHostsFromPrefs() const { - std::set<std::string> hosts; - const base::ListValue* list = pref_service_->GetList(prefs::kSnippetHosts); - for (const auto& value : *list) { - std::string str; - bool success = value->GetAsString(&str); - DCHECK(success) << "Failed to parse snippet host from prefs"; - hosts.insert(std::move(str)); - } - return hosts; -} - -void NTPSnippetsService::StoreSnippetHostsToPrefs( - const std::set<std::string>& hosts) { - base::ListValue list; - for (const std::string& host : hosts) - list.AppendString(host); - pref_service_->Set(prefs::kSnippetHosts, list); -} - -void NTPSnippetsService::ClearExpiredSnippets() { - base::Time expiry = base::Time::Now(); - - // Move expired snippets over into |to_delete|. - NTPSnippet::PtrVector to_delete; - for (std::unique_ptr<NTPSnippet>& snippet : snippets_) { - if (snippet->expiry_date() <= expiry) - to_delete.emplace_back(std::move(snippet)); - } - Compact(&snippets_); - - // Move expired discarded snippets over into |to_delete| as well. - for (std::unique_ptr<NTPSnippet>& snippet : discarded_snippets_) { - if (snippet->expiry_date() <= expiry) - to_delete.emplace_back(std::move(snippet)); - } - Compact(&discarded_snippets_); - - // Finally, actually delete the removed snippets from the DB. - database_->DeleteSnippets(to_delete); - - // If there are any snippets left, schedule a timer for the next expiry. - if (snippets_.empty() && discarded_snippets_.empty()) - return; - - base::Time next_expiry = base::Time::Max(); - for (const auto& snippet : snippets_) { - if (snippet->expiry_date() < next_expiry) - next_expiry = snippet->expiry_date(); - } - for (const auto& snippet : discarded_snippets_) { - if (snippet->expiry_date() < next_expiry) - next_expiry = snippet->expiry_date(); - } - DCHECK_GT(next_expiry, expiry); - expiry_timer_.Start(FROM_HERE, next_expiry - expiry, - base::Bind(&NTPSnippetsService::ClearExpiredSnippets, - base::Unretained(this))); -} - -void NTPSnippetsService::OnSnippetImageFetchedFromDatabase( - const std::string& snippet_id, - const ImageFetchedCallback& callback, - std::string data) { - // |image_decoder_| is null in tests. - if (image_decoder_ && !data.empty()) { - image_decoder_->DecodeImage( - std::move(data), - base::Bind(&NTPSnippetsService::OnSnippetImageDecoded, - base::Unretained(this), snippet_id, callback)); - return; - } - - // Fetching from the DB failed; start a network fetch. - FetchSnippetImageFromNetwork(snippet_id, callback); -} - -void NTPSnippetsService::OnSnippetImageDecoded( - const std::string& snippet_id, - const ImageFetchedCallback& callback, - const gfx::Image& image) { - if (!image.IsEmpty()) { - callback.Run(snippet_id, image); - return; - } - - // If decoding the image failed, delete the DB entry. - database_->DeleteImage(snippet_id); - - FetchSnippetImageFromNetwork(snippet_id, callback); -} - -void NTPSnippetsService::FetchSnippetImageFromNetwork( - const std::string& snippet_id, - const ImageFetchedCallback& callback) { - auto it = - std::find_if(snippets_.begin(), snippets_.end(), - [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) { - return snippet->id() == snippet_id; - }); - if (it == snippets_.end()) { - callback.Run(snippet_id, gfx::Image()); - return; - } - - const NTPSnippet& snippet = *it->get(); - image_fetcher_->StartOrQueueNetworkRequest( - snippet.id(), snippet.salient_image_url(), callback); -} - -void NTPSnippetsService::EnterStateEnabled(bool fetch_snippets) { - if (fetch_snippets) - FetchSnippets(); - - // If host restrictions are enabled, register for host list updates. - // |suggestions_service_| can be null in tests. - if (snippets_fetcher_->UsesHostRestrictions() && suggestions_service_) { - suggestions_service_subscription_ = - suggestions_service_->AddCallback(base::Bind( - &NTPSnippetsService::OnSuggestionsChanged, base::Unretained(this))); - } - - RescheduleFetching(); -} - -void NTPSnippetsService::EnterStateDisabled() { - ClearSnippets(); - ClearDiscardedSnippets(); - - expiry_timer_.Stop(); - suggestions_service_subscription_.reset(); - RescheduleFetching(); -} - -void NTPSnippetsService::EnterStateShutdown() { - FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_, - NTPSnippetsServiceShutdown()); - - expiry_timer_.Stop(); - suggestions_service_subscription_.reset(); - RescheduleFetching(); - - snippets_status_service_.reset(); -} - -void NTPSnippetsService::FinishInitialization() { - snippets_fetcher_->SetCallback( - base::Bind(&NTPSnippetsService::OnFetchFinished, base::Unretained(this))); - - // |image_fetcher_| can be null in tests. - if (image_fetcher_) - image_fetcher_->SetImageFetcherDelegate(this); - - // Note: Initializing the status service will run the callback right away with - // the current state. - snippets_status_service_->Init(base::Bind( - &NTPSnippetsService::UpdateStateForStatus, base::Unretained(this))); - - FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_, - NTPSnippetsServiceLoaded()); -} - -void NTPSnippetsService::UpdateStateForStatus(DisabledReason disabled_reason) { - FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_, - NTPSnippetsServiceDisabledReasonChanged(disabled_reason)); - - State new_state; - switch (disabled_reason) { - case DisabledReason::NONE: - new_state = State::READY; - break; - - case DisabledReason::HISTORY_SYNC_STATE_UNKNOWN: - // HistorySync is not initialized yet, so we don't know what the actual - // state is and we just return the current one. If things change, - // |OnStateChanged| will call this function again to update the state. - DVLOG(1) << "Sync configuration incomplete, continuing based on the " - "current state."; - new_state = state_; - break; - - case DisabledReason::EXPLICITLY_DISABLED: - case DisabledReason::SIGNED_OUT: - case DisabledReason::SYNC_DISABLED: - case DisabledReason::PASSPHRASE_ENCRYPTION_ENABLED: - case DisabledReason::HISTORY_SYNC_DISABLED: - new_state = State::DISABLED; - break; - - default: - // All cases should be handled by the above switch - NOTREACHED(); - new_state = State::DISABLED; - break; - } - - EnterState(new_state); -} - -void NTPSnippetsService::EnterState(State state) { - if (state == state_) - return; - - switch (state) { - case State::NOT_INITED: - // Initial state, it should not be possible to get back there. - NOTREACHED(); - return; - - case State::READY: { - DCHECK(state_ == State::NOT_INITED || state_ == State::DISABLED); - - bool fetch_snippets = snippets_.empty() || fetch_after_load_; - DVLOG(1) << "Entering state: READY"; - state_ = State::READY; - fetch_after_load_ = false; - EnterStateEnabled(fetch_snippets); - return; - } - - case State::DISABLED: - DCHECK(state_ == State::NOT_INITED || state_ == State::READY); - - DVLOG(1) << "Entering state: DISABLED"; - state_ = State::DISABLED; - EnterStateDisabled(); - return; - - case State::SHUT_DOWN: - DVLOG(1) << "Entering state: SHUT_DOWN"; - state_ = State::SHUT_DOWN; - EnterStateShutdown(); - return; - } -} - -void NTPSnippetsService::ClearDeprecatedPrefs() { - pref_service_->ClearPref(prefs::kDeprecatedSnippets); - pref_service_->ClearPref(prefs::kDeprecatedDiscardedSnippets); -} - -} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_service.h b/chromium/components/ntp_snippets/ntp_snippets_service.h deleted file mode 100644 index faf064a98bb..00000000000 --- a/chromium/components/ntp_snippets/ntp_snippets_service.h +++ /dev/null @@ -1,326 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_SERVICE_H_ -#define COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_SERVICE_H_ - -#include <stddef.h> - -#include <memory> -#include <set> -#include <string> -#include <vector> - -#include "base/gtest_prod_util.h" -#include "base/macros.h" -#include "base/observer_list.h" -#include "base/timer/timer.h" -#include "components/image_fetcher/image_fetcher_delegate.h" -#include "components/keyed_service/core/keyed_service.h" -#include "components/ntp_snippets/ntp_snippet.h" -#include "components/ntp_snippets/ntp_snippets_fetcher.h" -#include "components/ntp_snippets/ntp_snippets_scheduler.h" -#include "components/ntp_snippets/ntp_snippets_status_service.h" -#include "components/suggestions/suggestions_service.h" -#include "components/sync_driver/sync_service_observer.h" - -class PrefRegistrySimple; -class PrefService; -class SigninManagerBase; - -namespace base { -class RefCountedMemory; -class Value; -} - -namespace gfx { -class Image; -} - -namespace image_fetcher { -class ImageDecoder; -class ImageFetcher; -} - -namespace suggestions { -class SuggestionsProfile; -} - -namespace sync_driver { -class SyncService; -} - -namespace ntp_snippets { - -class NTPSnippetsDatabase; -class NTPSnippetsServiceObserver; - -// Stores and vends fresh content data for the NTP. -class NTPSnippetsService : public KeyedService, - public image_fetcher::ImageFetcherDelegate { - public: - using ImageFetchedCallback = - base::Callback<void(const std::string& snippet_id, const gfx::Image&)>; - - // |application_language_code| should be a ISO 639-1 compliant string, e.g. - // 'en' or 'en-US'. Note that this code should only specify the language, not - // the locale, so 'en_US' (English language with US locale) and 'en-GB_US' - // (British English person in the US) are not language codes. - NTPSnippetsService(bool enabled, - PrefService* pref_service, - suggestions::SuggestionsService* suggestions_service, - const std::string& application_language_code, - NTPSnippetsScheduler* scheduler, - std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher, - std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, - std::unique_ptr<image_fetcher::ImageDecoder> image_decoder, - std::unique_ptr<NTPSnippetsDatabase> database, - std::unique_ptr<NTPSnippetsStatusService> status_service); - - ~NTPSnippetsService() override; - - static void RegisterProfilePrefs(PrefRegistrySimple* registry); - - // Inherited from KeyedService. - void Shutdown() override; - - // Returns whether the service is ready. While this is false, the list of - // snippets will be empty, and all modifications to it (fetch, discard, etc) - // will be ignored. - bool ready() const { return state_ == State::READY; } - - // Returns whether the service is initialized. While this is false, some - // calls may trigger DCHECKs. - bool initialized() const { return ready() || state_ == State::DISABLED; } - - // Fetches snippets from the server and adds them to the current ones. - void FetchSnippets(); - // Fetches snippets from the server for specified hosts (overriding - // suggestions from the suggestion service) and adds them to the current ones. - // Only called from chrome://snippets-internals, DO NOT USE otherwise! - // Ignored while |loaded()| is false. - void FetchSnippetsFromHosts(const std::set<std::string>& hosts); - - // Available snippets. - const NTPSnippet::PtrVector& snippets() const { return snippets_; } - - // Returns the list of snippets previously discarded by the user (that are - // not expired yet). - const NTPSnippet::PtrVector& discarded_snippets() const { - return discarded_snippets_; - } - - const NTPSnippetsFetcher* snippets_fetcher() const { - return snippets_fetcher_.get(); - } - - // Returns a reason why the service is disabled, or DisabledReason::NONE - // if it's not. - DisabledReason disabled_reason() const { - return snippets_status_service_->disabled_reason(); - } - - // (Re)schedules the periodic fetching of snippets. This is necessary because - // the schedule depends on the time of day. - void RescheduleFetching(); - - // Fetches the image for the snippet with the given |snippet_id| and runs the - // |callback|. If that snippet doesn't exist or the fetch fails, the callback - // gets a null image. - void FetchSnippetImage(const std::string& snippet_id, - const ImageFetchedCallback& callback); - - // Deletes all currently stored snippets. - void ClearSnippets(); - - // Discards the snippet with the given |snippet_id|, if it exists. Returns - // true iff a snippet was discarded. - bool DiscardSnippet(const std::string& snippet_id); - - // Clears the lists of snippets previously discarded by the user. - void ClearDiscardedSnippets(); - - // Returns the lists of suggestion hosts the snippets are restricted to. - std::set<std::string> GetSuggestionsHosts() const; - - // Observer accessors. - void AddObserver(NTPSnippetsServiceObserver* observer); - void RemoveObserver(NTPSnippetsServiceObserver* observer); - - // Returns the maximum number of snippets that will be shown at once. - static int GetMaxSnippetCountForTesting(); - - private: - FRIEND_TEST_ALL_PREFIXES(NTPSnippetsServiceTest, HistorySyncStateChanges); - - // Possible state transitions: - // +------- NOT_INITED ------+ - // | / \ | - // | READY <--> DISABLED <-+ - // | \ / - // +-----> SHUT_DOWN - enum class State { - // The service has just been created. Can change to states: - // - DISABLED: if the constructor was called with |enabled == false| . In - // that case the service will stay disabled until it is shut - // down. It can also enter this state after the database is - // done loading and GetStateForDependenciesStatus identifies - // the next state to be DISABLED. - // - READY: if GetStateForDependenciesStatus returns it, after the database - // is done loading. - NOT_INITED, - - // The service registered observers, timers, etc. and is ready to answer to - // queries, fetch snippets... Can change to states: - // - DISABLED: when the global Chrome state changes, for example after - // |OnStateChanged| is called and sync is disabled. - // - SHUT_DOWN: when |Shutdown| is called, during the browser shutdown. - READY, - - // The service is disabled and unregistered the related resources. - // Can change to states: - // - READY: when the global Chrome state changes, for example after - // |OnStateChanged| is called and sync is enabled. - // - SHUT_DOWN: when |Shutdown| is called, during the browser shutdown. - DISABLED, - - // The service shutdown and can't be used anymore. This state is checked - // for early exit in callbacks from observers. - SHUT_DOWN - }; - - // image_fetcher::ImageFetcherDelegate implementation. - void OnImageDataFetched(const std::string& snippet_id, - const std::string& image_data) override; - - // Callbacks for the NTPSnippetsDatabase. - void OnDatabaseLoaded(NTPSnippet::PtrVector snippets); - void OnDatabaseError(); - - // Callback for the SuggestionsService. - void OnSuggestionsChanged(const suggestions::SuggestionsProfile& suggestions); - - // Callback for the NTPSnippetsFetcher. - void OnFetchFinished(NTPSnippetsFetcher::OptionalSnippets snippets); - - // Merges newly available snippets with the previously available list. - void MergeSnippets(NTPSnippet::PtrVector new_snippets); - - std::set<std::string> GetSnippetHostsFromPrefs() const; - void StoreSnippetHostsToPrefs(const std::set<std::string>& hosts); - - // Removes the expired snippets (including discarded) from the service and the - // database, and schedules another pass for the next expiration. - void ClearExpiredSnippets(); - - // Completes the initialization phase of the service, registering the last - // observers. This is done after construction, once the database is loaded. - void FinishInitialization(); - - void LoadingSnippetsFinished(); - - void OnSnippetImageFetchedFromDatabase(const std::string& snippet_id, - const ImageFetchedCallback& callback, - std::string data); - - void OnSnippetImageDecoded(const std::string& snippet_id, - const ImageFetchedCallback& callback, - const gfx::Image& image); - - void FetchSnippetImageFromNetwork(const std::string& snippet_id, - const ImageFetchedCallback& callback); - - // Triggers a state transition depending on the provided reason to be - // disabled (or lack thereof). This method is called when a change is detected - // by |snippets_status_service_| - void UpdateStateForStatus(DisabledReason disabled_reason); - - // Verifies state transitions (see |State|'s documentation) and applies them. - // Does nothing if called with the current state. - void EnterState(State state); - - // Enables the service and triggers a fetch if required. Do not call directly, - // use |EnterState| instead. - void EnterStateEnabled(bool fetch_snippets); - - // Disables the service. Do not call directly, use |EnterState| instead. - void EnterStateDisabled(); - - // Applies the effects of the transition to the SHUT_DOWN state. Do not call - // directly, use |EnterState| instead. - void EnterStateShutdown(); - - void ClearDeprecatedPrefs(); - - State state_; - - PrefService* pref_service_; - - suggestions::SuggestionsService* suggestions_service_; - - // All current suggestions (i.e. not discarded ones). - NTPSnippet::PtrVector snippets_; - - // Suggestions that the user discarded. We keep these around until they expire - // so we won't re-add them on the next fetch. - NTPSnippet::PtrVector discarded_snippets_; - - // The ISO 639-1 code of the language used by the application. - const std::string application_language_code_; - - // The observers. - base::ObserverList<NTPSnippetsServiceObserver> observers_; - - // Scheduler for fetching snippets. Not owned. - NTPSnippetsScheduler* scheduler_; - - // The subscription to the SuggestionsService. When the suggestions change, - // SuggestionsService will call |OnSuggestionsChanged|, which triggers an - // update to the set of snippets. - using SuggestionsSubscription = - suggestions::SuggestionsService::ResponseCallbackList::Subscription; - std::unique_ptr<SuggestionsSubscription> suggestions_service_subscription_; - - // The snippets fetcher. - std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher_; - - // Timer that calls us back when the next snippet expires. - base::OneShotTimer expiry_timer_; - - std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher_; - std::unique_ptr<image_fetcher::ImageDecoder> image_decoder_; - - // The database for persisting snippets. - std::unique_ptr<NTPSnippetsDatabase> database_; - - // The service that provides events and data about the signin and sync state. - std::unique_ptr<NTPSnippetsStatusService> snippets_status_service_; - - // Set to true if FetchSnippets is called before the database has been loaded. - // The fetch will be executed after the database load finishes. - bool fetch_after_load_; - - DISALLOW_COPY_AND_ASSIGN(NTPSnippetsService); -}; - -class NTPSnippetsServiceObserver { - public: - // Sent every time the service loads a new set of data. - virtual void NTPSnippetsServiceLoaded() = 0; - - // Sent when the service is shutting down. - virtual void NTPSnippetsServiceShutdown() = 0; - - // Sent when the state of the service is changing. Something changed in its - // dependencies so it's notifying observers about incoming data changes. - // If the service might be enabled, DisabledReason::NONE will be provided. - virtual void NTPSnippetsServiceDisabledReasonChanged(DisabledReason) = 0; - - protected: - virtual ~NTPSnippetsServiceObserver() {} -}; - -} // namespace ntp_snippets - -#endif // COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_SERVICE_H_ diff --git a/chromium/components/ntp_snippets/ntp_snippets_service_unittest.cc b/chromium/components/ntp_snippets/ntp_snippets_service_unittest.cc deleted file mode 100644 index 22678af60f9..00000000000 --- a/chromium/components/ntp_snippets/ntp_snippets_service_unittest.cc +++ /dev/null @@ -1,876 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/ntp_snippets/ntp_snippets_service.h" - -#include <memory> -#include <vector> - -#include "base/command_line.h" -#include "base/files/file_path.h" -#include "base/files/scoped_temp_dir.h" -#include "base/json/json_reader.h" -#include "base/macros.h" -#include "base/memory/ptr_util.h" -#include "base/message_loop/message_loop.h" -#include "base/run_loop.h" -#include "base/strings/string_number_conversions.h" -#include "base/strings/string_util.h" -#include "base/strings/stringprintf.h" -#include "base/test/histogram_tester.h" -#include "base/threading/thread_task_runner_handle.h" -#include "base/time/time.h" -#include "components/image_fetcher/image_decoder.h" -#include "components/image_fetcher/image_fetcher.h" -#include "components/ntp_snippets/ntp_snippet.h" -#include "components/ntp_snippets/ntp_snippets_database.h" -#include "components/ntp_snippets/ntp_snippets_fetcher.h" -#include "components/ntp_snippets/ntp_snippets_scheduler.h" -#include "components/ntp_snippets/ntp_snippets_test_utils.h" -#include "components/ntp_snippets/switches.h" -#include "components/prefs/testing_pref_service.h" -#include "components/signin/core/browser/fake_profile_oauth2_token_service.h" -#include "components/signin/core/browser/fake_signin_manager.h" -#include "google_apis/google_api_keys.h" -#include "net/url_request/test_url_fetcher_factory.h" -#include "net/url_request/url_request_test_util.h" -#include "testing/gmock/include/gmock/gmock.h" -#include "testing/gtest/include/gtest/gtest.h" - -using testing::ElementsAre; -using testing::Eq; -using testing::Return; -using testing::IsEmpty; -using testing::SizeIs; -using testing::StartsWith; -using testing::_; - -namespace ntp_snippets { - -namespace { - -MATCHER_P(IdEq, value, "") { - return arg->id() == value; -} - -const base::Time::Exploded kDefaultCreationTime = {2015, 11, 4, 25, 13, 46, 45}; -const char kTestContentSnippetsServerFormat[] = - "https://chromereader-pa.googleapis.com/v1/fetch?key=%s"; - -const char kSnippetUrl[] = "http://localhost/foobar"; -const char kSnippetTitle[] = "Title"; -const char kSnippetText[] = "Snippet"; -const char kSnippetSalientImage[] = "http://localhost/salient_image"; -const char kSnippetPublisherName[] = "Foo News"; -const char kSnippetAmpUrl[] = "http://localhost/amp"; -const float kSnippetScore = 5.0; - -base::Time GetDefaultCreationTime() { - return base::Time::FromUTCExploded(kDefaultCreationTime); -} - -base::Time GetDefaultExpirationTime() { - return base::Time::Now() + base::TimeDelta::FromHours(1); -} - -std::string GetTestJson(const std::vector<std::string>& snippets) { - return base::StringPrintf("{\"recos\":[%s]}", - base::JoinString(snippets, ", ").c_str()); -} - -std::string GetSnippetWithUrlAndTimesAndSources( - const std::string& url, - const std::string& content_creation_time_str, - const std::string& expiry_time_str, - const std::vector<std::string>& source_urls, - const std::vector<std::string>& publishers, - const std::vector<std::string>& amp_urls) { - char json_str_format[] = - "{ \"contentInfo\": {" - "\"url\" : \"%s\"," - "\"title\" : \"%s\"," - "\"snippet\" : \"%s\"," - "\"thumbnailUrl\" : \"%s\"," - "\"creationTimestampSec\" : \"%s\"," - "\"expiryTimestampSec\" : \"%s\"," - "\"sourceCorpusInfo\" : [%s]" - "}, " - "\"score\" : %f}"; - - char source_corpus_info_format[] = - "{\"corpusId\": \"%s\"," - "\"publisherData\": {" - "\"sourceName\": \"%s\"" - "}," - "\"ampUrl\": \"%s\"}"; - - std::string source_corpus_info_list_str; - for (size_t i = 0; i < source_urls.size(); ++i) { - std::string source_corpus_info_str = - base::StringPrintf(source_corpus_info_format, - source_urls[i].empty() ? "" : source_urls[i].c_str(), - publishers[i].empty() ? "" : publishers[i].c_str(), - amp_urls[i].empty() ? "" : amp_urls[i].c_str()); - source_corpus_info_list_str.append(source_corpus_info_str); - source_corpus_info_list_str.append(","); - } - // Remove the last comma - source_corpus_info_list_str.erase(source_corpus_info_list_str.size()-1, 1); - return base::StringPrintf(json_str_format, url.c_str(), kSnippetTitle, - kSnippetText, kSnippetSalientImage, - content_creation_time_str.c_str(), - expiry_time_str.c_str(), - source_corpus_info_list_str.c_str(), kSnippetScore); -} - -std::string GetSnippetWithSources(const std::vector<std::string>& source_urls, - const std::vector<std::string>& publishers, - const std::vector<std::string>& amp_urls) { - return GetSnippetWithUrlAndTimesAndSources( - kSnippetUrl, NTPSnippet::TimeToJsonString(GetDefaultCreationTime()), - NTPSnippet::TimeToJsonString(GetDefaultExpirationTime()), source_urls, - publishers, amp_urls); -} - -std::string GetSnippetWithUrlAndTimes( - const std::string& url, - const std::string& content_creation_time_str, - const std::string& expiry_time_str) { - return GetSnippetWithUrlAndTimesAndSources( - url, content_creation_time_str, expiry_time_str, - {std::string(url)}, {std::string(kSnippetPublisherName)}, - {std::string(kSnippetAmpUrl)}); -} - -std::string GetSnippetWithTimes(const std::string& content_creation_time_str, - const std::string& expiry_time_str) { - return GetSnippetWithUrlAndTimes(kSnippetUrl, content_creation_time_str, - expiry_time_str); -} - -std::string GetSnippetWithUrl(const std::string& url) { - return GetSnippetWithUrlAndTimes( - url, NTPSnippet::TimeToJsonString(GetDefaultCreationTime()), - NTPSnippet::TimeToJsonString(GetDefaultExpirationTime())); -} - -std::string GetSnippet() { - return GetSnippetWithUrlAndTimes( - kSnippetUrl, NTPSnippet::TimeToJsonString(GetDefaultCreationTime()), - NTPSnippet::TimeToJsonString(GetDefaultExpirationTime())); -} - -std::string GetExpiredSnippet() { - return GetSnippetWithTimes( - NTPSnippet::TimeToJsonString(GetDefaultCreationTime()), - NTPSnippet::TimeToJsonString(base::Time::Now())); -} - -std::string GetInvalidSnippet() { - std::string json_str = GetSnippet(); - // Make the json invalid by removing the final closing brace. - return json_str.substr(0, json_str.size() - 1); -} - -std::string GetIncompleteSnippet() { - std::string json_str = GetSnippet(); - // Rename the "url" entry. The result is syntactically valid json that will - // fail to parse as snippets. - size_t pos = json_str.find("\"url\""); - if (pos == std::string::npos) { - NOTREACHED(); - return std::string(); - } - json_str[pos + 1] = 'x'; - return json_str; -} - -void ParseJson( - const std::string& json, - const ntp_snippets::NTPSnippetsFetcher::SuccessCallback& success_callback, - const ntp_snippets::NTPSnippetsFetcher::ErrorCallback& error_callback) { - base::JSONReader json_reader; - std::unique_ptr<base::Value> value = json_reader.ReadToValue(json); - if (value) { - success_callback.Run(std::move(value)); - } else { - error_callback.Run(json_reader.GetErrorMessage()); - } -} - -// Factory for FakeURLFetcher objects that always generate errors. -class FailingFakeURLFetcherFactory : public net::URLFetcherFactory { - public: - std::unique_ptr<net::URLFetcher> CreateURLFetcher( - int id, const GURL& url, net::URLFetcher::RequestType request_type, - net::URLFetcherDelegate* d) override { - return base::MakeUnique<net::FakeURLFetcher>( - url, d, /*response_data=*/std::string(), net::HTTP_NOT_FOUND, - net::URLRequestStatus::FAILED); - } -}; - -class MockScheduler : public NTPSnippetsScheduler { - public: - MOCK_METHOD4(Schedule, - bool(base::TimeDelta period_wifi_charging, - base::TimeDelta period_wifi, - base::TimeDelta period_fallback, - base::Time reschedule_time)); - MOCK_METHOD0(Unschedule, bool()); -}; - -class MockServiceObserver : public NTPSnippetsServiceObserver { - public: - MOCK_METHOD0(NTPSnippetsServiceLoaded, void()); - MOCK_METHOD0(NTPSnippetsServiceShutdown, void()); - MOCK_METHOD1(NTPSnippetsServiceDisabledReasonChanged, - void(DisabledReason disabled_reason)); -}; - -class WaitForDBLoad : public NTPSnippetsServiceObserver { - public: - WaitForDBLoad(NTPSnippetsService* service) : service_(service) { - service_->AddObserver(this); - if (!service_->ready()) - run_loop_.Run(); - } - - ~WaitForDBLoad() override { - service_->RemoveObserver(this); - } - - private: - void NTPSnippetsServiceLoaded() override { - EXPECT_TRUE(service_->ready()); - run_loop_.Quit(); - } - - void NTPSnippetsServiceShutdown() override {} - void NTPSnippetsServiceDisabledReasonChanged( - DisabledReason disabled_reason) override {} - - NTPSnippetsService* service_; - base::RunLoop run_loop_; - - DISALLOW_COPY_AND_ASSIGN(WaitForDBLoad); -}; - -} // namespace - -class NTPSnippetsServiceTest : public test::NTPSnippetsTestBase { - public: - NTPSnippetsServiceTest() - : fake_url_fetcher_factory_( - /*default_factory=*/&failing_url_fetcher_factory_), - test_url_(base::StringPrintf(kTestContentSnippetsServerFormat, - google_apis::GetAPIKey().c_str())) { - NTPSnippetsService::RegisterProfilePrefs(pref_service()->registry()); - - // Since no SuggestionsService is injected in tests, we need to force the - // service to fetch from all hosts. - base::CommandLine::ForCurrentProcess()->AppendSwitch( - switches::kDontRestrict); - EXPECT_TRUE(database_dir_.CreateUniqueTempDir()); - } - - ~NTPSnippetsServiceTest() override { - if (service_) - service_->Shutdown(); - - // We need to run the message loop after deleting the database, because - // ProtoDatabaseImpl deletes the actual LevelDB asynchronously on the task - // runner. Without this, we'd get reports of memory leaks. - service_.reset(); - base::RunLoop().RunUntilIdle(); - } - - void SetUp() override { - test::NTPSnippetsTestBase::SetUp(); - EXPECT_CALL(mock_scheduler(), Schedule(_, _, _, _)).Times(1); - CreateSnippetsService(/*enabled=*/true); - } - - void CreateSnippetsService(bool enabled) { - if (service_) - service_->Shutdown(); - - scoped_refptr<base::SingleThreadTaskRunner> task_runner( - base::ThreadTaskRunnerHandle::Get()); - scoped_refptr<net::TestURLRequestContextGetter> request_context_getter = - new net::TestURLRequestContextGetter(task_runner.get()); - - // Delete the current service, so that the database is destroyed before we - // create the new one, otherwise opening the new database will fail. - service_.reset(); - - ResetSigninManager(); - std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher = - base::MakeUnique<NTPSnippetsFetcher>( - fake_signin_manager(), fake_token_service_.get(), - std::move(request_context_getter), base::Bind(&ParseJson), - /*is_stable_channel=*/true); - - fake_signin_manager()->SignIn("foo@bar.com"); - snippets_fetcher->SetPersonalizationForTesting( - NTPSnippetsFetcher::Personalization::kNonPersonal); - - service_.reset(new NTPSnippetsService( - enabled, pref_service(), nullptr, "fr", &scheduler_, - std::move(snippets_fetcher), /*image_fetcher=*/nullptr, - /*image_fetcher=*/nullptr, base::MakeUnique<NTPSnippetsDatabase>( - database_dir_.path(), task_runner), - base::MakeUnique<NTPSnippetsStatusService>(fake_signin_manager(), - mock_sync_service()))); - - if (enabled) - WaitForDBLoad(service_.get()); - } - - protected: - const GURL& test_url() { return test_url_; } - NTPSnippetsService* service() { return service_.get(); } - MockScheduler& mock_scheduler() { return scheduler_; } - - // Provide the json to be returned by the fake fetcher. - void SetUpFetchResponse(const std::string& json) { - fake_url_fetcher_factory_.SetFakeResponse(test_url_, json, net::HTTP_OK, - net::URLRequestStatus::SUCCESS); - } - - void LoadFromJSONString(const std::string& json) { - SetUpFetchResponse(json); - service()->FetchSnippets(); - base::RunLoop().RunUntilIdle(); - } - - private: - base::MessageLoop message_loop_; - FailingFakeURLFetcherFactory failing_url_fetcher_factory_; - // Instantiation of factory automatically sets itself as URLFetcher's factory. - net::FakeURLFetcherFactory fake_url_fetcher_factory_; - const GURL test_url_; - std::unique_ptr<OAuth2TokenService> fake_token_service_; - MockScheduler scheduler_; - // Last so that the dependencies are deleted after the service. - std::unique_ptr<NTPSnippetsService> service_; - - base::ScopedTempDir database_dir_; - - DISALLOW_COPY_AND_ASSIGN(NTPSnippetsServiceTest); -}; - -class NTPSnippetsServiceDisabledTest : public NTPSnippetsServiceTest { - public: - void SetUp() override { - test::NTPSnippetsTestBase::SetUp(); - EXPECT_CALL(mock_scheduler(), Unschedule()).Times(1); - CreateSnippetsService(/*enabled=*/false); - } -}; - -TEST_F(NTPSnippetsServiceTest, ScheduleIfEnabled) { - // SetUp() checks that Schedule is called. -} - -TEST_F(NTPSnippetsServiceDisabledTest, Unschedule) { - // SetUp() checks that Unschedule is called. -} - -TEST_F(NTPSnippetsServiceTest, Full) { - std::string json_str(GetTestJson({GetSnippet()})); - - LoadFromJSONString(json_str); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - const NTPSnippet& snippet = *service()->snippets().front(); - - EXPECT_EQ(snippet.id(), kSnippetUrl); - EXPECT_EQ(snippet.title(), kSnippetTitle); - EXPECT_EQ(snippet.snippet(), kSnippetText); - EXPECT_EQ(snippet.salient_image_url(), GURL(kSnippetSalientImage)); - EXPECT_EQ(GetDefaultCreationTime(), snippet.publish_date()); - EXPECT_EQ(snippet.best_source().publisher_name, kSnippetPublisherName); - EXPECT_EQ(snippet.best_source().amp_url, GURL(kSnippetAmpUrl)); -} - -TEST_F(NTPSnippetsServiceTest, Clear) { - std::string json_str(GetTestJson({GetSnippet()})); - - LoadFromJSONString(json_str); - EXPECT_THAT(service()->snippets(), SizeIs(1)); - - service()->ClearSnippets(); - EXPECT_THAT(service()->snippets(), IsEmpty()); -} - -TEST_F(NTPSnippetsServiceTest, InsertAtFront) { - std::string first("http://first"); - LoadFromJSONString(GetTestJson({GetSnippetWithUrl(first)})); - EXPECT_THAT(service()->snippets(), ElementsAre(IdEq(first))); - - std::string second("http://second"); - LoadFromJSONString(GetTestJson({GetSnippetWithUrl(second)})); - // The snippet loaded last should be at the first position in the list now. - EXPECT_THAT(service()->snippets(), ElementsAre(IdEq(second), IdEq(first))); -} - -TEST_F(NTPSnippetsServiceTest, LimitNumSnippets) { - int max_snippet_count = NTPSnippetsService::GetMaxSnippetCountForTesting(); - int snippets_per_load = max_snippet_count / 2 + 1; - char url_format[] = "http://localhost/%i"; - - std::vector<std::string> snippets1; - std::vector<std::string> snippets2; - for (int i = 0; i < snippets_per_load; i++) { - snippets1.push_back(GetSnippetWithUrl(base::StringPrintf(url_format, i))); - snippets2.push_back(GetSnippetWithUrl( - base::StringPrintf(url_format, snippets_per_load + i))); - } - - LoadFromJSONString(GetTestJson(snippets1)); - ASSERT_THAT(service()->snippets(), SizeIs(snippets1.size())); - - LoadFromJSONString(GetTestJson(snippets2)); - EXPECT_THAT(service()->snippets(), SizeIs(max_snippet_count)); -} - -TEST_F(NTPSnippetsServiceTest, LoadInvalidJson) { - LoadFromJSONString(GetTestJson({GetInvalidSnippet()})); - EXPECT_THAT(service()->snippets_fetcher()->last_status(), - StartsWith("Received invalid JSON")); - EXPECT_THAT(service()->snippets(), IsEmpty()); -} - -TEST_F(NTPSnippetsServiceTest, LoadInvalidJsonWithExistingSnippets) { - LoadFromJSONString(GetTestJson({GetSnippet()})); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - ASSERT_EQ("OK", service()->snippets_fetcher()->last_status()); - - LoadFromJSONString(GetTestJson({GetInvalidSnippet()})); - EXPECT_THAT(service()->snippets_fetcher()->last_status(), - StartsWith("Received invalid JSON")); - // This should not have changed the existing snippets. - EXPECT_THAT(service()->snippets(), SizeIs(1)); -} - -TEST_F(NTPSnippetsServiceTest, LoadIncompleteJson) { - LoadFromJSONString(GetTestJson({GetIncompleteSnippet()})); - EXPECT_EQ("Invalid / empty list.", - service()->snippets_fetcher()->last_status()); - EXPECT_THAT(service()->snippets(), IsEmpty()); -} - -TEST_F(NTPSnippetsServiceTest, LoadIncompleteJsonWithExistingSnippets) { - LoadFromJSONString(GetTestJson({GetSnippet()})); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - - LoadFromJSONString(GetTestJson({GetIncompleteSnippet()})); - EXPECT_EQ("Invalid / empty list.", - service()->snippets_fetcher()->last_status()); - // This should not have changed the existing snippets. - EXPECT_THAT(service()->snippets(), SizeIs(1)); -} - -TEST_F(NTPSnippetsServiceTest, Discard) { - std::vector<std::string> source_urls, publishers, amp_urls; - source_urls.push_back(std::string("http://site.com")); - publishers.push_back(std::string("Source 1")); - amp_urls.push_back(std::string()); - std::string json_str( - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)})); - - LoadFromJSONString(json_str); - - ASSERT_THAT(service()->snippets(), SizeIs(1)); - - // Discarding a non-existent snippet shouldn't do anything. - EXPECT_FALSE(service()->DiscardSnippet("http://othersite.com")); - EXPECT_THAT(service()->snippets(), SizeIs(1)); - - // Discard the snippet. - EXPECT_TRUE(service()->DiscardSnippet(kSnippetUrl)); - EXPECT_THAT(service()->snippets(), IsEmpty()); - - // Make sure that fetching the same snippet again does not re-add it. - LoadFromJSONString(json_str); - EXPECT_THAT(service()->snippets(), IsEmpty()); - - // The snippet should stay discarded even after re-creating the service. - EXPECT_CALL(mock_scheduler(), Schedule(_, _, _, _)).Times(1); - CreateSnippetsService(/*enabled=*/true); - LoadFromJSONString(json_str); - EXPECT_THAT(service()->snippets(), IsEmpty()); - - // The snippet can be added again after clearing discarded snippets. - service()->ClearDiscardedSnippets(); - EXPECT_THAT(service()->snippets(), IsEmpty()); - LoadFromJSONString(json_str); - EXPECT_THAT(service()->snippets(), SizeIs(1)); -} - -TEST_F(NTPSnippetsServiceTest, GetDiscarded) { - LoadFromJSONString(GetTestJson({GetSnippet()})); - - // For the test, we need the snippet to get discarded. - ASSERT_TRUE(service()->DiscardSnippet(kSnippetUrl)); - const NTPSnippet::PtrVector& snippets = service()->discarded_snippets(); - EXPECT_EQ(1u, snippets.size()); - for (auto& snippet : snippets) { - EXPECT_EQ(kSnippetUrl, snippet->id()); - } - - // There should be no discarded snippet after clearing the list. - service()->ClearDiscardedSnippets(); - EXPECT_EQ(0u, service()->discarded_snippets().size()); -} - -TEST_F(NTPSnippetsServiceTest, CreationTimestampParseFail) { - std::string json_str(GetTestJson({GetSnippetWithTimes( - "aaa1448459205", - NTPSnippet::TimeToJsonString(GetDefaultExpirationTime()))})); - - LoadFromJSONString(json_str); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - const NTPSnippet& snippet = *service()->snippets().front(); - EXPECT_EQ(snippet.id(), kSnippetUrl); - EXPECT_EQ(snippet.title(), kSnippetTitle); - EXPECT_EQ(snippet.snippet(), kSnippetText); - EXPECT_EQ(base::Time::UnixEpoch(), snippet.publish_date()); -} - -TEST_F(NTPSnippetsServiceTest, RemoveExpiredContent) { - std::string json_str(GetTestJson({GetExpiredSnippet()})); - - LoadFromJSONString(json_str); - EXPECT_THAT(service()->snippets(), IsEmpty()); -} - -TEST_F(NTPSnippetsServiceTest, TestSingleSource) { - std::vector<std::string> source_urls, publishers, amp_urls; - source_urls.push_back(std::string("http://source1.com")); - publishers.push_back(std::string("Source 1")); - amp_urls.push_back(std::string("http://source1.amp.com")); - std::string json_str( - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)})); - - LoadFromJSONString(json_str); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - const NTPSnippet& snippet = *service()->snippets().front(); - EXPECT_EQ(snippet.sources().size(), 1u); - EXPECT_EQ(snippet.id(), kSnippetUrl); - EXPECT_EQ(snippet.best_source().url, GURL("http://source1.com")); - EXPECT_EQ(snippet.best_source().publisher_name, std::string("Source 1")); - EXPECT_EQ(snippet.best_source().amp_url, GURL("http://source1.amp.com")); -} - -TEST_F(NTPSnippetsServiceTest, TestSingleSourceWithMalformedUrl) { - std::vector<std::string> source_urls, publishers, amp_urls; - source_urls.push_back(std::string("aaaa")); - publishers.push_back(std::string("Source 1")); - amp_urls.push_back(std::string("http://source1.amp.com")); - std::string json_str( - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)})); - - LoadFromJSONString(json_str); - EXPECT_THAT(service()->snippets(), IsEmpty()); -} - -TEST_F(NTPSnippetsServiceTest, TestSingleSourceWithMissingData) { - std::vector<std::string> source_urls, publishers, amp_urls; - source_urls.push_back(std::string("http://source1.com")); - publishers.push_back(std::string()); - amp_urls.push_back(std::string()); - std::string json_str( - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)})); - - LoadFromJSONString(json_str); - EXPECT_THAT(service()->snippets(), IsEmpty()); -} - -TEST_F(NTPSnippetsServiceTest, TestMultipleSources) { - std::vector<std::string> source_urls, publishers, amp_urls; - source_urls.push_back(std::string("http://source1.com")); - source_urls.push_back(std::string("http://source2.com")); - publishers.push_back(std::string("Source 1")); - publishers.push_back(std::string("Source 2")); - amp_urls.push_back(std::string("http://source1.amp.com")); - amp_urls.push_back(std::string("http://source2.amp.com")); - std::string json_str( - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)})); - - LoadFromJSONString(json_str); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - const NTPSnippet& snippet = *service()->snippets().front(); - // Expect the first source to be chosen - EXPECT_EQ(snippet.sources().size(), 2u); - EXPECT_EQ(snippet.id(), kSnippetUrl); - EXPECT_EQ(snippet.best_source().url, GURL("http://source1.com")); - EXPECT_EQ(snippet.best_source().publisher_name, std::string("Source 1")); - EXPECT_EQ(snippet.best_source().amp_url, GURL("http://source1.amp.com")); -} - -TEST_F(NTPSnippetsServiceTest, TestMultipleIncompleteSources) { - // Set Source 2 to have no AMP url, and Source 1 to have no publisher name - // Source 2 should win since we favor publisher name over amp url - std::vector<std::string> source_urls, publishers, amp_urls; - source_urls.push_back(std::string("http://source1.com")); - source_urls.push_back(std::string("http://source2.com")); - publishers.push_back(std::string()); - publishers.push_back(std::string("Source 2")); - amp_urls.push_back(std::string("http://source1.amp.com")); - amp_urls.push_back(std::string()); - std::string json_str( - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)})); - - LoadFromJSONString(json_str); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - { - const NTPSnippet& snippet = *service()->snippets().front(); - EXPECT_EQ(snippet.sources().size(), 2u); - EXPECT_EQ(snippet.id(), kSnippetUrl); - EXPECT_EQ(snippet.best_source().url, GURL("http://source2.com")); - EXPECT_EQ(snippet.best_source().publisher_name, std::string("Source 2")); - EXPECT_EQ(snippet.best_source().amp_url, GURL()); - } - - service()->ClearSnippets(); - // Set Source 1 to have no AMP url, and Source 2 to have no publisher name - // Source 1 should win in this case since we prefer publisher name to AMP url - source_urls.clear(); - source_urls.push_back(std::string("http://source1.com")); - source_urls.push_back(std::string("http://source2.com")); - publishers.clear(); - publishers.push_back(std::string("Source 1")); - publishers.push_back(std::string()); - amp_urls.clear(); - amp_urls.push_back(std::string()); - amp_urls.push_back(std::string("http://source2.amp.com")); - json_str = - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)}); - - LoadFromJSONString(json_str); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - { - const NTPSnippet& snippet = *service()->snippets().front(); - EXPECT_EQ(snippet.sources().size(), 2u); - EXPECT_EQ(snippet.id(), kSnippetUrl); - EXPECT_EQ(snippet.best_source().url, GURL("http://source1.com")); - EXPECT_EQ(snippet.best_source().publisher_name, std::string("Source 1")); - EXPECT_EQ(snippet.best_source().amp_url, GURL()); - } - - service()->ClearSnippets(); - // Set source 1 to have no AMP url and no source, and source 2 to only have - // amp url. There should be no snippets since we only add sources we consider - // complete - source_urls.clear(); - source_urls.push_back(std::string("http://source1.com")); - source_urls.push_back(std::string("http://source2.com")); - publishers.clear(); - publishers.push_back(std::string()); - publishers.push_back(std::string()); - amp_urls.clear(); - amp_urls.push_back(std::string()); - amp_urls.push_back(std::string("http://source2.amp.com")); - json_str = - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)}); - - LoadFromJSONString(json_str); - EXPECT_THAT(service()->snippets(), IsEmpty()); -} - -TEST_F(NTPSnippetsServiceTest, TestMultipleCompleteSources) { - // Test 2 complete sources, we should choose the first complete source - std::vector<std::string> source_urls, publishers, amp_urls; - source_urls.push_back(std::string("http://source1.com")); - source_urls.push_back(std::string("http://source2.com")); - source_urls.push_back(std::string("http://source3.com")); - publishers.push_back(std::string("Source 1")); - publishers.push_back(std::string()); - publishers.push_back(std::string("Source 3")); - amp_urls.push_back(std::string("http://source1.amp.com")); - amp_urls.push_back(std::string("http://source2.amp.com")); - amp_urls.push_back(std::string("http://source3.amp.com")); - std::string json_str( - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)})); - - LoadFromJSONString(json_str); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - { - const NTPSnippet& snippet = *service()->snippets().front(); - EXPECT_EQ(snippet.sources().size(), 3u); - EXPECT_EQ(snippet.id(), kSnippetUrl); - EXPECT_EQ(snippet.best_source().url, GURL("http://source1.com")); - EXPECT_EQ(snippet.best_source().publisher_name, std::string("Source 1")); - EXPECT_EQ(snippet.best_source().amp_url, GURL("http://source1.amp.com")); - } - - // Test 2 complete sources, we should choose the first complete source - service()->ClearSnippets(); - source_urls.clear(); - source_urls.push_back(std::string("http://source1.com")); - source_urls.push_back(std::string("http://source2.com")); - source_urls.push_back(std::string("http://source3.com")); - publishers.clear(); - publishers.push_back(std::string()); - publishers.push_back(std::string("Source 2")); - publishers.push_back(std::string("Source 3")); - amp_urls.clear(); - amp_urls.push_back(std::string("http://source1.amp.com")); - amp_urls.push_back(std::string("http://source2.amp.com")); - amp_urls.push_back(std::string("http://source3.amp.com")); - json_str = - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)}); - - LoadFromJSONString(json_str); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - { - const NTPSnippet& snippet = *service()->snippets().front(); - EXPECT_EQ(snippet.sources().size(), 3u); - EXPECT_EQ(snippet.id(), kSnippetUrl); - EXPECT_EQ(snippet.best_source().url, GURL("http://source2.com")); - EXPECT_EQ(snippet.best_source().publisher_name, std::string("Source 2")); - EXPECT_EQ(snippet.best_source().amp_url, GURL("http://source2.amp.com")); - } - - // Test 3 complete sources, we should choose the first complete source - service()->ClearSnippets(); - source_urls.clear(); - source_urls.push_back(std::string("http://source1.com")); - source_urls.push_back(std::string("http://source2.com")); - source_urls.push_back(std::string("http://source3.com")); - publishers.clear(); - publishers.push_back(std::string("Source 1")); - publishers.push_back(std::string("Source 2")); - publishers.push_back(std::string("Source 3")); - amp_urls.clear(); - amp_urls.push_back(std::string()); - amp_urls.push_back(std::string("http://source2.amp.com")); - amp_urls.push_back(std::string("http://source3.amp.com")); - json_str = - GetTestJson({GetSnippetWithSources(source_urls, publishers, amp_urls)}); - - LoadFromJSONString(json_str); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - { - const NTPSnippet& snippet = *service()->snippets().front(); - EXPECT_EQ(snippet.sources().size(), 3u); - EXPECT_EQ(snippet.id(), kSnippetUrl); - EXPECT_EQ(snippet.best_source().url, GURL("http://source2.com")); - EXPECT_EQ(snippet.best_source().publisher_name, std::string("Source 2")); - EXPECT_EQ(snippet.best_source().amp_url, GURL("http://source2.amp.com")); - } -} - -TEST_F(NTPSnippetsServiceTest, LogNumArticlesHistogram) { - base::HistogramTester tester; - LoadFromJSONString(GetTestJson({GetInvalidSnippet()})); - - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); - // Invalid JSON shouldn't contribute to NumArticlesFetched. - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), - IsEmpty()); - // Valid JSON with empty list. - LoadFromJSONString(GetTestJson(std::vector<std::string>())); - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/2))); - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); - // Snippet list should be populated with size 1. - LoadFromJSONString(GetTestJson({GetSnippet()})); - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/2), - base::Bucket(/*min=*/1, /*count=*/1))); - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/1), - base::Bucket(/*min=*/1, /*count=*/1))); - // Duplicate snippet shouldn't increase the list size. - LoadFromJSONString(GetTestJson({GetSnippet()})); - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/2), - base::Bucket(/*min=*/1, /*count=*/2))); - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/1), - base::Bucket(/*min=*/1, /*count=*/2))); - EXPECT_THAT( - tester.GetAllSamples("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded"), - IsEmpty()); - // Discarding a snippet should decrease the list size. This will only be - // logged after the next fetch. - EXPECT_TRUE(service()->DiscardSnippet(kSnippetUrl)); - LoadFromJSONString(GetTestJson({GetSnippet()})); - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/3), - base::Bucket(/*min=*/1, /*count=*/2))); - // Discarded snippets shouldn't influence NumArticlesFetched. - EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), - ElementsAre(base::Bucket(/*min=*/0, /*count=*/1), - base::Bucket(/*min=*/1, /*count=*/3))); - EXPECT_THAT( - tester.GetAllSamples("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded"), - ElementsAre(base::Bucket(/*min=*/1, /*count=*/1))); - // Recreating the service and loading from prefs shouldn't count as fetched - // articles. - EXPECT_CALL(mock_scheduler(), Schedule(_, _, _, _)).Times(1); - CreateSnippetsService(/*enabled=*/true); - tester.ExpectTotalCount("NewTabPage.Snippets.NumArticlesFetched", 4); -} - -TEST_F(NTPSnippetsServiceTest, DiscardShouldRespectAllKnownUrls) { - const std::string creation = - NTPSnippet::TimeToJsonString(GetDefaultCreationTime()); - const std::string expiry = - NTPSnippet::TimeToJsonString(GetDefaultExpirationTime()); - const std::vector<std::string> source_urls = { - "http://mashable.com/2016/05/11/stolen", - "http://www.aol.com/article/2016/05/stolen-doggie", - "http://mashable.com/2016/05/11/stolen?utm_cid=1"}; - const std::vector<std::string> publishers = {"Mashable", "AOL", "Mashable"}; - const std::vector<std::string> amp_urls = { - "http://mashable-amphtml.googleusercontent.com/1", - "http://t2.gstatic.com/images?q=tbn:3", - "http://t2.gstatic.com/images?q=tbn:3"}; - - // Add the snippet from the mashable domain. - LoadFromJSONString(GetTestJson({GetSnippetWithUrlAndTimesAndSources( - source_urls[0], creation, expiry, source_urls, publishers, amp_urls)})); - ASSERT_THAT(service()->snippets(), SizeIs(1)); - // Discard the snippet via the mashable source corpus ID. - EXPECT_TRUE(service()->DiscardSnippet(source_urls[0])); - EXPECT_THAT(service()->snippets(), IsEmpty()); - - // The same article from the AOL domain should now be detected as discarded. - LoadFromJSONString(GetTestJson({GetSnippetWithUrlAndTimesAndSources( - source_urls[1], creation, expiry, source_urls, publishers, amp_urls)})); - ASSERT_THAT(service()->snippets(), IsEmpty()); -} - -TEST_F(NTPSnippetsServiceTest, HistorySyncStateChanges) { - MockServiceObserver mock_observer; - service()->AddObserver(&mock_observer); - - // Simulate user signed out - SetUpFetchResponse(GetTestJson({GetSnippet()})); - EXPECT_CALL(mock_observer, NTPSnippetsServiceDisabledReasonChanged( - DisabledReason::SIGNED_OUT)); - service()->UpdateStateForStatus(DisabledReason::SIGNED_OUT); - base::RunLoop().RunUntilIdle(); - EXPECT_EQ(NTPSnippetsService::State::DISABLED, service()->state_); - EXPECT_THAT(service()->snippets(), IsEmpty()); // No fetch should be made. - - // Simulate user sign in. The service should be ready again and load snippets. - SetUpFetchResponse(GetTestJson({GetSnippet()})); - EXPECT_CALL(mock_observer, - NTPSnippetsServiceDisabledReasonChanged(DisabledReason::NONE)); - EXPECT_CALL(mock_scheduler(), Schedule(_, _, _, _)).Times(1); - service()->UpdateStateForStatus(DisabledReason::NONE); - base::RunLoop().RunUntilIdle(); - EXPECT_EQ(NTPSnippetsService::State::READY, service()->state_); - EXPECT_FALSE(service()->snippets().empty()); - - service()->RemoveObserver(&mock_observer); -} - -} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_status_service.cc b/chromium/components/ntp_snippets/ntp_snippets_status_service.cc deleted file mode 100644 index 5c279991b1e..00000000000 --- a/chromium/components/ntp_snippets/ntp_snippets_status_service.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/ntp_snippets/ntp_snippets_status_service.h" -#include "components/signin/core/browser/signin_manager.h" -#include "components/sync_driver/sync_service.h" - -namespace ntp_snippets { - -NTPSnippetsStatusService::NTPSnippetsStatusService( - SigninManagerBase* signin_manager, - sync_driver::SyncService* sync_service) - : disabled_reason_(DisabledReason::EXPLICITLY_DISABLED), - signin_manager_(signin_manager), - sync_service_(sync_service), - sync_service_observer_(this) {} - -NTPSnippetsStatusService::~NTPSnippetsStatusService() {} - -void NTPSnippetsStatusService::Init( - const DisabledReasonChangeCallback& callback) { - DCHECK(disabled_reason_change_callback_.is_null()); - - disabled_reason_change_callback_ = callback; - - // Notify about the current state before registering the observer, to make - // sure we don't get a double notification due to an undefined start state. - disabled_reason_ = GetDisabledReasonFromDeps(); - disabled_reason_change_callback_.Run(disabled_reason_); - - sync_service_observer_.Add(sync_service_); -} - -void NTPSnippetsStatusService::OnStateChanged() { - DisabledReason new_disabled_reason = GetDisabledReasonFromDeps(); - - if (new_disabled_reason == disabled_reason_) - return; - - disabled_reason_ = new_disabled_reason; - disabled_reason_change_callback_.Run(disabled_reason_); -} - -DisabledReason NTPSnippetsStatusService::GetDisabledReasonFromDeps() const { - if (!signin_manager_ || !signin_manager_->IsAuthenticated()) { - DVLOG(1) << "[GetNewDisabledReason] Signed out"; - return DisabledReason::SIGNED_OUT; - } - - if (!sync_service_ || !sync_service_->CanSyncStart()) { - DVLOG(1) << "[GetNewDisabledReason] Sync disabled"; - return DisabledReason::SYNC_DISABLED; - } - - // !IsSyncActive in cases where CanSyncStart is true hints at the backend not - // being initialized. - // ConfigurationDone() verifies that the sync service has properly loaded its - // configuration and is aware of the different data types to sync. - if (!sync_service_->IsSyncActive() || !sync_service_->ConfigurationDone()) { - DVLOG(1) << "[GetNewDisabledReason] Sync initialization is not complete."; - return DisabledReason::HISTORY_SYNC_STATE_UNKNOWN; - } - - if (sync_service_->IsEncryptEverythingEnabled()) { - DVLOG(1) << "[GetNewDisabledReason] Encryption is enabled"; - return DisabledReason::PASSPHRASE_ENCRYPTION_ENABLED; - } - - if (!sync_service_->GetActiveDataTypes().Has( - syncer::HISTORY_DELETE_DIRECTIVES)) { - DVLOG(1) << "[GetNewDisabledReason] History sync disabled"; - return DisabledReason::HISTORY_SYNC_DISABLED; - } - - DVLOG(1) << "[GetNewDisabledReason] Enabled"; - return DisabledReason::NONE; -} - -} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_status_service.h b/chromium/components/ntp_snippets/ntp_snippets_status_service.h deleted file mode 100644 index b5a73eea1c1..00000000000 --- a/chromium/components/ntp_snippets/ntp_snippets_status_service.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_STATUS_SERVICE_H_ -#define COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_STATUS_SERVICE_H_ - -#include "base/callback.h" -#include "base/gtest_prod_util.h" -#include "base/scoped_observer.h" -#include "components/sync_driver/sync_service_observer.h" - -class SigninManagerBase; - -namespace sync_driver { -class SyncService; -} - -namespace ntp_snippets { - -// On Android builds, a Java counterpart will be generated for this enum. -// GENERATED_JAVA_ENUM_PACKAGE: org.chromium.chrome.browser.ntp.snippets -enum class DisabledReason : int { - // Snippets are enabled - NONE, - // Snippets have been disabled as part of the service configuration. - EXPLICITLY_DISABLED, - // The user is not signed in, and the service requires it to be enabled. - SIGNED_OUT, - // Sync is not enabled, and the service requires it to be enabled. - SYNC_DISABLED, - // The service requires passphrase encryption to be disabled. - PASSPHRASE_ENCRYPTION_ENABLED, - // History sync is not enabled, and the service requires it to be enabled. - HISTORY_SYNC_DISABLED, - // The sync service is not completely initialized, and the status is unknown. - HISTORY_SYNC_STATE_UNKNOWN -}; - -// Aggregates data from sync and signin to notify the snippet service of -// relevant changes in their states. -class NTPSnippetsStatusService : public sync_driver::SyncServiceObserver { - public: - typedef base::Callback<void(DisabledReason)> DisabledReasonChangeCallback; - - NTPSnippetsStatusService(SigninManagerBase* signin_manager, - sync_driver::SyncService* sync_service); - - ~NTPSnippetsStatusService() override; - - // Starts listening for changes from the dependencies. |callback| will be - // called when a significant change in state is detected. - void Init(const DisabledReasonChangeCallback& callback); - - DisabledReason disabled_reason() const { return disabled_reason_; } - - private: - FRIEND_TEST_ALL_PREFIXES(NTPSnippetsStatusServiceTest, - SyncStateCompatibility); - - // sync_driver::SyncServiceObserver implementation - void OnStateChanged() override; - - DisabledReason GetDisabledReasonFromDeps() const; - - DisabledReason disabled_reason_; - DisabledReasonChangeCallback disabled_reason_change_callback_; - - SigninManagerBase* signin_manager_; - sync_driver::SyncService* sync_service_; - - // The observer for the SyncService. When the sync state changes, - // SyncService will call |OnStateChanged|. - ScopedObserver<sync_driver::SyncService, sync_driver::SyncServiceObserver> - sync_service_observer_; - - DISALLOW_COPY_AND_ASSIGN(NTPSnippetsStatusService); -}; - -} // namespace ntp_snippets - -#endif // COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_STATUS_SERVICE_H_ diff --git a/chromium/components/ntp_snippets/ntp_snippets_status_service_unittest.cc b/chromium/components/ntp_snippets/ntp_snippets_status_service_unittest.cc deleted file mode 100644 index 5568d805349..00000000000 --- a/chromium/components/ntp_snippets/ntp_snippets_status_service_unittest.cc +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/ntp_snippets/ntp_snippets_status_service.h" - -#include <memory> - -#include "components/ntp_snippets/ntp_snippets_test_utils.h" -#include "components/prefs/pref_registry_simple.h" -#include "components/prefs/testing_pref_service.h" -#include "components/signin/core/browser/account_tracker_service.h" -#include "components/signin/core/browser/fake_signin_manager.h" -#include "components/signin/core/browser/test_signin_client.h" -#include "components/signin/core/common/signin_pref_names.h" -#include "components/sync_driver/fake_sync_service.h" -#include "testing/gmock/include/gmock/gmock.h" -#include "testing/gtest/include/gtest/gtest.h" - -using testing::Return; - -namespace ntp_snippets { - -class NTPSnippetsStatusServiceTest : public test::NTPSnippetsTestBase { - public: - void SetUp() override { - test::NTPSnippetsTestBase::SetUp(); - - service_.reset(new NTPSnippetsStatusService(fake_signin_manager(), - mock_sync_service())); - } - - protected: - NTPSnippetsStatusService* service() { return service_.get(); } - - private: - std::unique_ptr<NTPSnippetsStatusService> service_; -}; - -TEST_F(NTPSnippetsStatusServiceTest, SyncStateCompatibility) { - // The default test setup is signed out. - EXPECT_EQ(DisabledReason::SIGNED_OUT, service()->GetDisabledReasonFromDeps()); - - // Once signed in, we should be in a compatible sync state. - fake_signin_manager()->SignIn("foo@bar.com"); - EXPECT_EQ(DisabledReason::NONE, service()->GetDisabledReasonFromDeps()); - - // History sync disabled. - mock_sync_service()->active_data_types_ = syncer::ModelTypeSet(); - EXPECT_EQ(DisabledReason::HISTORY_SYNC_DISABLED, - service()->GetDisabledReasonFromDeps()); - - // Encryption enabled. - mock_sync_service()->is_encrypt_everything_enabled_ = true; - EXPECT_EQ(DisabledReason::PASSPHRASE_ENCRYPTION_ENABLED, - service()->GetDisabledReasonFromDeps()); - - // Not done loading. - mock_sync_service()->configuration_done_ = false; - mock_sync_service()->active_data_types_ = syncer::ModelTypeSet(); - EXPECT_EQ(DisabledReason::HISTORY_SYNC_STATE_UNKNOWN, - service()->GetDisabledReasonFromDeps()); - - // Sync disabled. - mock_sync_service()->can_sync_start_ = false; - EXPECT_EQ(DisabledReason::SYNC_DISABLED, - service()->GetDisabledReasonFromDeps()); -} - -} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/offline_pages/DEPS b/chromium/components/ntp_snippets/offline_pages/DEPS new file mode 100644 index 00000000000..984d8cbcbb5 --- /dev/null +++ b/chromium/components/ntp_snippets/offline_pages/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+components/offline_pages" +] diff --git a/chromium/components/ntp_snippets/offline_pages/offline_page_proxy.cc b/chromium/components/ntp_snippets/offline_pages/offline_page_proxy.cc new file mode 100644 index 00000000000..4e3f17212ce --- /dev/null +++ b/chromium/components/ntp_snippets/offline_pages/offline_page_proxy.cc @@ -0,0 +1,66 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/offline_pages/offline_page_proxy.h" + +#include "base/bind.h" + +using offline_pages::MultipleOfflinePageItemResult; +using offline_pages::MultipleOfflinePageItemCallback; +using offline_pages::OfflinePageModel; + +namespace ntp_snippets { + +OfflinePageProxy::OfflinePageProxy(OfflinePageModel* offline_page_model) + : offline_page_model_(offline_page_model), weak_ptr_factory_(this) { + offline_page_model_->AddObserver(this); +} + +void OfflinePageProxy::GetAllPages( + const MultipleOfflinePageItemCallback& callback) { + offline_page_model_->GetAllPages(callback); +} + +void OfflinePageProxy::AddObserver(Observer* observer) { + observers_.AddObserver(observer); +} + +void OfflinePageProxy::RemoveObserver(Observer* observer) { + observers_.RemoveObserver(observer); +} + +//////////////////////////////////////////////////////////////////////////////// +// Private methods + +OfflinePageProxy::~OfflinePageProxy() { + offline_page_model_->RemoveObserver(this); +} + +void OfflinePageProxy::OfflinePageModelLoaded(OfflinePageModel* model) { + DCHECK_EQ(offline_page_model_, model); +} + +void OfflinePageProxy::OfflinePageModelChanged(OfflinePageModel* model) { + DCHECK_EQ(offline_page_model_, model); + FetchOfflinePagesAndNotify(); +} + +void OfflinePageProxy::OfflinePageDeleted( + int64_t offline_id, + const offline_pages::ClientId& client_id) { + FOR_EACH_OBSERVER(Observer, observers_, + OfflinePageDeleted(offline_id, client_id)); +} + +void OfflinePageProxy::FetchOfflinePagesAndNotify() { + offline_page_model_->GetAllPages(base::Bind( + &OfflinePageProxy::OnOfflinePagesLoaded, weak_ptr_factory_.GetWeakPtr())); +} + +void OfflinePageProxy::OnOfflinePagesLoaded( + const MultipleOfflinePageItemResult& result) { + FOR_EACH_OBSERVER(Observer, observers_, OfflinePageModelChanged(result)); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/offline_pages/offline_page_proxy.h b/chromium/components/ntp_snippets/offline_pages/offline_page_proxy.h new file mode 100644 index 00000000000..a3fa62ea0ca --- /dev/null +++ b/chromium/components/ntp_snippets/offline_pages/offline_page_proxy.h @@ -0,0 +1,84 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_OFFLINE_PAGES_PROXY_OFFLINE_PAGE_PROXY_H_ +#define COMPONENTS_NTP_SNIPPETS_OFFLINE_PAGES_PROXY_OFFLINE_PAGE_PROXY_H_ + +#include <string> +#include <vector> + +#include "base/callback_forward.h" +#include "base/memory/ref_counted.h" +#include "base/memory/weak_ptr.h" +#include "base/observer_list.h" +#include "components/offline_pages/offline_page_model.h" +#include "components/offline_pages/offline_page_types.h" + +namespace ntp_snippets { + +// Observes offline pages model and propagates updates to its observers. +class OfflinePageProxy : public offline_pages::OfflinePageModel::Observer, + public base::RefCounted<OfflinePageProxy> { + public: + class Observer { + public: + // Corresponds to OfflinePageModel::Observer::OfflinePageModelChanged. + // Invoked when the model is being updated, due to adding, removing or + // updating an offline page. |offline_pages| contains all offline pages + // after the update. + virtual void OfflinePageModelChanged( + const std::vector<offline_pages::OfflinePageItem>& offline_pages) = 0; + + // Corresponds to OfflinePageModel::Observer::OfflinePageDeleted. + // Invoked when an offline copy related to |offline_id| was deleted. + virtual void OfflinePageDeleted( + int64_t offline_id, + const offline_pages::ClientId& client_id) = 0; + + protected: + virtual ~Observer() = default; + }; + + explicit OfflinePageProxy( + offline_pages::OfflinePageModel* offline_page_model); + + // TODO(vitaliii): Remove this function and provide a better way for providers + // to get data at the start up, while querying OfflinePagesModel only once. + // Queries OfflinePageModel for all pages and returns them through |callback|. + void GetAllPages( + const offline_pages::MultipleOfflinePageItemCallback& callback); + + // Observer accessors. + void AddObserver(Observer* observer); + void RemoveObserver(Observer* observer); + + private: + friend class base::RefCounted<OfflinePageProxy>; + + ~OfflinePageProxy() override; + + // OfflinePageModel::Observer implementation. + void OfflinePageModelLoaded(offline_pages::OfflinePageModel* model) override; + void OfflinePageModelChanged(offline_pages::OfflinePageModel* model) override; + void OfflinePageDeleted(int64_t offline_id, + const offline_pages::ClientId& client_id) override; + + // Queries the OfflinePageModel for offline pages and notifies observers + // through |OfflinePageModelChanged|. + void FetchOfflinePagesAndNotify(); + + // Callback from the |OfflinePageModel::GetAllPages|. + void OnOfflinePagesLoaded( + const offline_pages::MultipleOfflinePageItemResult& result); + + offline_pages::OfflinePageModel* offline_page_model_; + base::ObserverList<Observer> observers_; + base::WeakPtrFactory<OfflinePageProxy> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(OfflinePageProxy); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_OFFLINE_PAGES_PROXY_OFFLINE_PAGE_PROXY_H_ diff --git a/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider.cc b/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider.cc new file mode 100644 index 00000000000..777e2a32254 --- /dev/null +++ b/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider.cc @@ -0,0 +1,274 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/offline_pages/recent_tab_suggestions_provider.h" + +#include <algorithm> +#include <utility> + +#include "base/bind.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/utf_string_conversions.h" +#include "base/threading/thread_task_runner_handle.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/ntp_snippets/pref_util.h" +#include "components/offline_pages/client_namespace_constants.h" +#include "components/offline_pages/offline_page_item.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" +#include "grit/components_strings.h" +#include "ui/base/l10n/l10n_util.h" +#include "ui/gfx/image/image.h" + +using offline_pages::ClientId; +using offline_pages::OfflinePageItem; + +namespace ntp_snippets { + +namespace { + +const int kMaxSuggestionsCount = 5; + +struct OrderOfflinePagesByMostRecentlyVisitedFirst { + bool operator()(const OfflinePageItem* left, + const OfflinePageItem* right) const { + return left->last_access_time > right->last_access_time; + } +}; + +bool IsRecentTab(const ClientId& client_id) { + return client_id.name_space == offline_pages::kLastNNamespace; +} + +} // namespace + +RecentTabSuggestionsProvider::RecentTabSuggestionsProvider( + ContentSuggestionsProvider::Observer* observer, + CategoryFactory* category_factory, + scoped_refptr<OfflinePageProxy> offline_page_proxy, + PrefService* pref_service) + : ContentSuggestionsProvider(observer, category_factory), + category_status_(CategoryStatus::AVAILABLE_LOADING), + provided_category_( + category_factory->FromKnownCategory(KnownCategories::RECENT_TABS)), + offline_page_proxy_(offline_page_proxy), + pref_service_(pref_service), + weak_ptr_factory_(this) { + observer->OnCategoryStatusChanged(this, provided_category_, category_status_); + offline_page_proxy_->AddObserver(this); + FetchRecentTabs(); +} + +RecentTabSuggestionsProvider::~RecentTabSuggestionsProvider() { + offline_page_proxy_->RemoveObserver(this); +} + +CategoryStatus RecentTabSuggestionsProvider::GetCategoryStatus( + Category category) { + if (category == provided_category_) + return category_status_; + NOTREACHED() << "Unknown category " << category.id(); + return CategoryStatus::NOT_PROVIDED; +} + +CategoryInfo RecentTabSuggestionsProvider::GetCategoryInfo(Category category) { + if (category == provided_category_) { + return CategoryInfo(l10n_util::GetStringUTF16( + IDS_NTP_RECENT_TAB_SUGGESTIONS_SECTION_HEADER), + ContentSuggestionsCardLayout::MINIMAL_CARD, + /*has_more_button=*/false, + /*show_if_empty=*/false); + } + NOTREACHED() << "Unknown category " << category.id(); + return CategoryInfo(base::string16(), + ContentSuggestionsCardLayout::MINIMAL_CARD, + /*has_more_button=*/false, + /*show_if_empty=*/false); +} + +void RecentTabSuggestionsProvider::DismissSuggestion( + const ContentSuggestion::ID& suggestion_id) { + DCHECK_EQ(provided_category_, suggestion_id.category()); + std::set<std::string> dismissed_ids = ReadDismissedIDsFromPrefs(); + dismissed_ids.insert(suggestion_id.id_within_category()); + StoreDismissedIDsToPrefs(dismissed_ids); +} + +void RecentTabSuggestionsProvider::FetchSuggestionImage( + const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) { + // TODO(vitaliii): Fetch proper thumbnail from OfflinePageModel once it's + // available there. + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::Bind(callback, gfx::Image())); +} + +void RecentTabSuggestionsProvider::ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) { + ClearDismissedSuggestionsForDebugging(provided_category_); + FetchRecentTabs(); +} + +void RecentTabSuggestionsProvider::ClearCachedSuggestions(Category category) { + // Ignored. +} + +void RecentTabSuggestionsProvider::GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) { + DCHECK_EQ(provided_category_, category); + offline_page_proxy_->GetAllPages( + base::Bind(&RecentTabSuggestionsProvider:: + GetAllPagesCallbackForGetDismissedSuggestions, + weak_ptr_factory_.GetWeakPtr(), callback)); +} + +void RecentTabSuggestionsProvider::ClearDismissedSuggestionsForDebugging( + Category category) { + DCHECK_EQ(provided_category_, category); + StoreDismissedIDsToPrefs(std::set<std::string>()); + FetchRecentTabs(); +} + +// static +void RecentTabSuggestionsProvider::RegisterProfilePrefs( + PrefRegistrySimple* registry) { + registry->RegisterListPref(prefs::kDismissedRecentOfflineTabSuggestions); +} + +//////////////////////////////////////////////////////////////////////////////// +// Private methods + +void RecentTabSuggestionsProvider:: + GetAllPagesCallbackForGetDismissedSuggestions( + const DismissedSuggestionsCallback& callback, + const std::vector<OfflinePageItem>& offline_pages) const { + std::set<std::string> dismissed_ids = ReadDismissedIDsFromPrefs(); + std::vector<ContentSuggestion> suggestions; + for (const OfflinePageItem& item : offline_pages) { + if (!IsRecentTab(item.client_id) || + !dismissed_ids.count(base::IntToString(item.offline_id))) + continue; + suggestions.push_back(ConvertOfflinePage(item)); + } + callback.Run(std::move(suggestions)); +} + +void RecentTabSuggestionsProvider::OfflinePageModelChanged( + const std::vector<OfflinePageItem>& offline_pages) { + NotifyStatusChanged(CategoryStatus::AVAILABLE); + std::set<std::string> old_dismissed_ids = ReadDismissedIDsFromPrefs(); + std::set<std::string> new_dismissed_ids; + std::vector<const OfflinePageItem*> recent_tab_items; + for (const OfflinePageItem& item : offline_pages) { + std::string offline_page_id = base::IntToString(item.offline_id); + if (!IsRecentTab(item.client_id)) { + continue; + } + + if (old_dismissed_ids.count(offline_page_id)) + new_dismissed_ids.insert(offline_page_id); + else + recent_tab_items.push_back(&item); + } + + observer()->OnNewSuggestions( + this, provided_category_, + GetMostRecentlyVisited(std::move(recent_tab_items))); + if (new_dismissed_ids.size() != old_dismissed_ids.size()) + StoreDismissedIDsToPrefs(new_dismissed_ids); +} + +void RecentTabSuggestionsProvider::OfflinePageDeleted( + int64_t offline_id, + const ClientId& client_id) { + // Because we never switch to NOT_PROVIDED dynamically, there can be no open + // UI containing an invalidated suggestion unless the status is something + // other than NOT_PROVIDED, so only notify invalidation in that case. + if (category_status_ != CategoryStatus::NOT_PROVIDED && + IsRecentTab(client_id)) { + InvalidateSuggestion(offline_id); + } +} + +void RecentTabSuggestionsProvider::FetchRecentTabs() { + // TODO(vitaliii): When something other than GetAllPages is used here, the + // dismissed IDs cleanup in OfflinePageModelChanged needs to be changed to + // avoid accidentally undismissing suggestions. + offline_page_proxy_->GetAllPages( + base::Bind(&RecentTabSuggestionsProvider::OfflinePageModelChanged, + weak_ptr_factory_.GetWeakPtr())); +} + +void RecentTabSuggestionsProvider::NotifyStatusChanged( + CategoryStatus new_status) { + DCHECK_NE(CategoryStatus::NOT_PROVIDED, category_status_); + if (category_status_ == new_status) + return; + category_status_ = new_status; + observer()->OnCategoryStatusChanged(this, provided_category_, new_status); +} + +ContentSuggestion RecentTabSuggestionsProvider::ConvertOfflinePage( + const OfflinePageItem& offline_page) const { + // TODO(vitaliii): Make sure the URL is opened in the existing tab. + ContentSuggestion suggestion(provided_category_, + base::IntToString(offline_page.offline_id), + offline_page.url); + + if (offline_page.title.empty()) { + // TODO(vitaliii): Remove this fallback once the OfflinePageModel provides + // titles for all (relevant) OfflinePageItems. + suggestion.set_title(base::UTF8ToUTF16(offline_page.url.spec())); + } else { + suggestion.set_title(offline_page.title); + } + suggestion.set_publish_date(offline_page.creation_time); + suggestion.set_publisher_name(base::UTF8ToUTF16(offline_page.url.host())); + return suggestion; +} + +std::vector<ContentSuggestion> +RecentTabSuggestionsProvider::GetMostRecentlyVisited( + std::vector<const OfflinePageItem*> offline_page_items) const { + std::sort(offline_page_items.begin(), offline_page_items.end(), + OrderOfflinePagesByMostRecentlyVisitedFirst()); + std::vector<ContentSuggestion> suggestions; + for (const OfflinePageItem* offline_page_item : offline_page_items) { + suggestions.push_back(ConvertOfflinePage(*offline_page_item)); + if (suggestions.size() == kMaxSuggestionsCount) + break; + } + return suggestions; +} + +void RecentTabSuggestionsProvider::InvalidateSuggestion(int64_t offline_id) { + std::string offline_page_id = base::IntToString(offline_id); + observer()->OnSuggestionInvalidated( + this, ContentSuggestion::ID(provided_category_, offline_page_id)); + + std::set<std::string> dismissed_ids = ReadDismissedIDsFromPrefs(); + auto it = dismissed_ids.find(offline_page_id); + if (it != dismissed_ids.end()) { + dismissed_ids.erase(it); + StoreDismissedIDsToPrefs(dismissed_ids); + } +} + +std::set<std::string> RecentTabSuggestionsProvider::ReadDismissedIDsFromPrefs() + const { + return prefs::ReadDismissedIDsFromPrefs( + *pref_service_, prefs::kDismissedRecentOfflineTabSuggestions); +} + +void RecentTabSuggestionsProvider::StoreDismissedIDsToPrefs( + const std::set<std::string>& dismissed_ids) { + prefs::StoreDismissedIDsToPrefs(pref_service_, + prefs::kDismissedRecentOfflineTabSuggestions, + dismissed_ids); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider.h b/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider.h new file mode 100644 index 00000000000..033a6533d5f --- /dev/null +++ b/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider.h @@ -0,0 +1,119 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_OFFLINE_PAGES_RECENT_TAB_SUGGESTIONS_PROVIDER_H_ +#define COMPONENTS_NTP_SNIPPETS_OFFLINE_PAGES_RECENT_TAB_SUGGESTIONS_PROVIDER_H_ + +#include <set> +#include <string> +#include <vector> + +#include "base/callback_forward.h" +#include "base/memory/ref_counted.h" +#include "base/memory/weak_ptr.h" +#include "base/time/time.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/category_status.h" +#include "components/ntp_snippets/content_suggestion.h" +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "components/ntp_snippets/offline_pages/offline_page_proxy.h" + +class PrefRegistrySimple; +class PrefService; + +namespace gfx { +class Image; +} // namespace gfx + +namespace ntp_snippets { + +// Provides recent tabs content suggestions from the offline pages model +// obtaining the data through OfflinePageProxy. +class RecentTabSuggestionsProvider : public ContentSuggestionsProvider, + public OfflinePageProxy::Observer { + public: + RecentTabSuggestionsProvider( + ContentSuggestionsProvider::Observer* observer, + CategoryFactory* category_factory, + scoped_refptr<OfflinePageProxy> offline_page_proxy, + PrefService* pref_service); + ~RecentTabSuggestionsProvider() override; + + // ContentSuggestionsProvider implementation. + CategoryStatus GetCategoryStatus(Category category) override; + CategoryInfo GetCategoryInfo(Category category) override; + void DismissSuggestion(const ContentSuggestion::ID& suggestion_id) override; + void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) override; + void ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) override; + void ClearCachedSuggestions(Category category) override; + void GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) override; + void ClearDismissedSuggestionsForDebugging(Category category) override; + + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + private: + friend class RecentTabSuggestionsProviderTest; + + void GetAllPagesCallbackForGetDismissedSuggestions( + const DismissedSuggestionsCallback& callback, + const std::vector<offline_pages::OfflinePageItem>& offline_pages) const; + + // OfflinePageProxy::Observer implementation. + void OfflinePageModelChanged( + const std::vector<offline_pages::OfflinePageItem>& offline_pages) + override; + void OfflinePageDeleted(int64_t offline_id, + const offline_pages::ClientId& client_id) override; + + // Updates the |category_status_| of the |provided_category_| and notifies the + // |observer_|, if necessary. + void NotifyStatusChanged(CategoryStatus new_status); + + // Manually requests all offline pages and updates the suggestions. + void FetchRecentTabs(); + + // Converts an OfflinePageItem to a ContentSuggestion for the + // |provided_category_|. + ContentSuggestion ConvertOfflinePage( + const offline_pages::OfflinePageItem& offline_page) const; + + // Gets the |kMaxSuggestionsCount| most recently visited OfflinePageItems from + // the list, orders them by last visit date and converts them to + // ContentSuggestions for the |provided_category_|. + std::vector<ContentSuggestion> GetMostRecentlyVisited( + std::vector<const offline_pages::OfflinePageItem*> offline_page_items) + const; + + // Fires the |OnSuggestionInvalidated| event for the suggestion corresponding + // to the given |offline_id| and clears it from the dismissed IDs list, if + // necessary. + void InvalidateSuggestion(int64_t offline_id); + + // Reads dismissed IDs from Prefs. + std::set<std::string> ReadDismissedIDsFromPrefs() const; + + // Writes |dismissed_ids| into Prefs. + void StoreDismissedIDsToPrefs(const std::set<std::string>& dismissed_ids); + + CategoryStatus category_status_; + const Category provided_category_; + scoped_refptr<OfflinePageProxy> offline_page_proxy_; + + PrefService* pref_service_; + + base::WeakPtrFactory<RecentTabSuggestionsProvider> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(RecentTabSuggestionsProvider); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_OFFLINE_PAGES_RECENT_TAB_SUGGESTIONS_PROVIDER_H_ diff --git a/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider_unittest.cc b/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider_unittest.cc new file mode 100644 index 00000000000..2688b518a11 --- /dev/null +++ b/chromium/components/ntp_snippets/offline_pages/recent_tab_suggestions_provider_unittest.cc @@ -0,0 +1,272 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/offline_pages/recent_tab_suggestions_provider.h" + +#include <string> +#include <vector> + +#include "base/bind.h" +#include "base/files/file_path.h" +#include "base/guid.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/utf_string_conversions.h" +#include "base/time/time.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "components/ntp_snippets/mock_content_suggestions_provider_observer.h" +#include "components/offline_pages/client_namespace_constants.h" +#include "components/offline_pages/offline_page_item.h" +#include "components/offline_pages/stub_offline_page_model.h" +#include "components/prefs/testing_pref_service.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using offline_pages::ClientId; +using offline_pages::MultipleOfflinePageItemCallback; +using offline_pages::OfflinePageItem; +using offline_pages::StubOfflinePageModel; +using testing::_; +using testing::IsEmpty; +using testing::Mock; +using testing::Property; +using testing::SizeIs; + +namespace ntp_snippets { + +namespace { + +OfflinePageItem CreateDummyRecentTab(int id) { + std::string strid = base::IntToString(id); + return OfflinePageItem( + GURL("http://dummy.com/" + strid), id, + ClientId(offline_pages::kLastNNamespace, base::GenerateGUID()), + base::FilePath::FromUTF8Unsafe("some/folder/test" + strid + ".mhtml"), 0, + base::Time::Now()); +} + +std::vector<OfflinePageItem> CreateDummyRecentTabs( + const std::vector<int>& ids) { + std::vector<OfflinePageItem> result; + for (int id : ids) { + result.push_back(CreateDummyRecentTab(id)); + } + return result; +} + +OfflinePageItem CreateDummyRecentTab(int id, base::Time time) { + OfflinePageItem item = CreateDummyRecentTab(id); + item.last_access_time = time; + return item; +} + +void CaptureDismissedSuggestions( + std::vector<ContentSuggestion>* captured_suggestions, + std::vector<ContentSuggestion> dismissed_suggestions) { + std::move(dismissed_suggestions.begin(), dismissed_suggestions.end(), + std::back_inserter(*captured_suggestions)); +} + +} // namespace + +// This model is needed only when a provider is expected to call |GetAllPages|. +// In other cases, keeping this model empty ensures that provider listens to +// proxy notifications without calling |GetAllPages|. +class FakeOfflinePageModel : public StubOfflinePageModel { + public: + FakeOfflinePageModel() {} + + void GetAllPages(const MultipleOfflinePageItemCallback& callback) override { + callback.Run(items_); + } + + const std::vector<OfflinePageItem>& items() { return items_; } + std::vector<OfflinePageItem>* mutable_items() { return &items_; } + + private: + std::vector<OfflinePageItem> items_; +}; + +class RecentTabSuggestionsProviderTest : public testing::Test { + public: + RecentTabSuggestionsProviderTest() + : pref_service_(new TestingPrefServiceSimple()) { + RecentTabSuggestionsProvider::RegisterProfilePrefs( + pref_service()->registry()); + + scoped_refptr<OfflinePageProxy> proxy(new OfflinePageProxy(&model_)); + provider_.reset(new RecentTabSuggestionsProvider( + &observer_, &category_factory_, proxy, pref_service())); + } + + Category recent_tabs_category() { + return category_factory_.FromKnownCategory(KnownCategories::RECENT_TABS); + } + + ContentSuggestion::ID GetDummySuggestionId(int id) { + return ContentSuggestion::ID(recent_tabs_category(), base::IntToString(id)); + } + + void FireOfflinePageModelChanged(const std::vector<OfflinePageItem>& items) { + provider_->OfflinePageModelChanged(items); + } + + void FireOfflinePageDeleted(const OfflinePageItem& item) { + provider_->OfflinePageDeleted(item.offline_id, item.client_id); + } + + std::set<std::string> ReadDismissedIDsFromPrefs() { + return provider_->ReadDismissedIDsFromPrefs(); + } + + ContentSuggestionsProvider* provider() { return provider_.get(); } + FakeOfflinePageModel* model() { return &model_; } + MockContentSuggestionsProviderObserver* observer() { return &observer_; } + TestingPrefServiceSimple* pref_service() { return pref_service_.get(); } + + private: + FakeOfflinePageModel model_; + MockContentSuggestionsProviderObserver observer_; + CategoryFactory category_factory_; + std::unique_ptr<TestingPrefServiceSimple> pref_service_; + // Last so that the dependencies are deleted after the provider. + std::unique_ptr<RecentTabSuggestionsProvider> provider_; + + DISALLOW_COPY_AND_ASSIGN(RecentTabSuggestionsProviderTest); +}; + +TEST_F(RecentTabSuggestionsProviderTest, ShouldConvertToSuggestions) { + std::vector<OfflinePageItem> offline_pages = CreateDummyRecentTabs({1, 2, 3}); + + EXPECT_CALL( + *observer(), + OnNewSuggestions(_, recent_tabs_category(), + UnorderedElementsAre( + Property(&ContentSuggestion::url, + GURL("http://dummy.com/1")), + Property(&ContentSuggestion::url, + GURL("http://dummy.com/2")), + Property(&ContentSuggestion::url, + GURL("http://dummy.com/3"))))); + FireOfflinePageModelChanged(offline_pages); +} + +TEST_F(RecentTabSuggestionsProviderTest, ShouldSortByMostRecentlyVisited) { + base::Time now = base::Time::Now(); + base::Time yesterday = now - base::TimeDelta::FromDays(1); + base::Time tomorrow = now + base::TimeDelta::FromDays(1); + std::vector<OfflinePageItem> offline_pages = { + CreateDummyRecentTab(1, now), CreateDummyRecentTab(2, yesterday), + CreateDummyRecentTab(3, tomorrow)}; + + EXPECT_CALL( + *observer(), + OnNewSuggestions( + _, recent_tabs_category(), + ElementsAre(Property(&ContentSuggestion::url, + GURL("http://dummy.com/3")), + Property(&ContentSuggestion::url, + GURL("http://dummy.com/1")), + Property(&ContentSuggestion::url, + GURL("http://dummy.com/2"))))); + FireOfflinePageModelChanged(offline_pages); +} + +TEST_F(RecentTabSuggestionsProviderTest, ShouldDeliverCorrectCategoryInfo) { + EXPECT_FALSE( + provider()->GetCategoryInfo(recent_tabs_category()).has_more_button()); +} + +TEST_F(RecentTabSuggestionsProviderTest, ShouldDismiss) { + // OfflinePageModel is initialised here because + // |GetDismissedSuggestionsForDebugging| may need to call |GetAllPages| + *(model()->mutable_items()) = CreateDummyRecentTabs({1, 2, 3, 4}); + FireOfflinePageModelChanged(model()->items()); + + // Dismiss 2 and 3. + EXPECT_CALL(*observer(), OnNewSuggestions(_, _, _)).Times(0); + provider()->DismissSuggestion(GetDummySuggestionId(2)); + provider()->DismissSuggestion(GetDummySuggestionId(3)); + Mock::VerifyAndClearExpectations(observer()); + + // They should disappear from the reported suggestions. + EXPECT_CALL( + *observer(), + OnNewSuggestions(_, recent_tabs_category(), + UnorderedElementsAre( + Property(&ContentSuggestion::url, + GURL("http://dummy.com/1")), + Property(&ContentSuggestion::url, + GURL("http://dummy.com/4"))))); + + FireOfflinePageModelChanged(model()->items()); + Mock::VerifyAndClearExpectations(observer()); + + // And appear in the dismissed suggestions. + std::vector<ContentSuggestion> dismissed_suggestions; + provider()->GetDismissedSuggestionsForDebugging( + recent_tabs_category(), + base::Bind(&CaptureDismissedSuggestions, &dismissed_suggestions)); + EXPECT_THAT( + dismissed_suggestions, + UnorderedElementsAre(Property(&ContentSuggestion::url, + GURL("http://dummy.com/2")), + Property(&ContentSuggestion::url, + GURL("http://dummy.com/3")))); + + // Clear dismissed suggestions. + provider()->ClearDismissedSuggestionsForDebugging(recent_tabs_category()); + + // They should be gone from the dismissed suggestions. + dismissed_suggestions.clear(); + provider()->GetDismissedSuggestionsForDebugging( + recent_tabs_category(), + base::Bind(&CaptureDismissedSuggestions, &dismissed_suggestions)); + EXPECT_THAT(dismissed_suggestions, IsEmpty()); + + // And appear in the reported suggestions for the category again. + EXPECT_CALL(*observer(), + OnNewSuggestions(_, recent_tabs_category(), SizeIs(4))); + FireOfflinePageModelChanged(model()->items()); + Mock::VerifyAndClearExpectations(observer()); +} + +TEST_F(RecentTabSuggestionsProviderTest, + ShouldInvalidateWhenOfflinePageDeleted) { + std::vector<OfflinePageItem> offline_pages = CreateDummyRecentTabs({1, 2, 3}); + FireOfflinePageModelChanged(offline_pages); + + // Invalidation of suggestion 2 should be forwarded. + EXPECT_CALL(*observer(), OnSuggestionInvalidated(_, GetDummySuggestionId(2))); + FireOfflinePageDeleted(offline_pages[1]); +} + +TEST_F(RecentTabSuggestionsProviderTest, ShouldClearDismissedOnInvalidate) { + std::vector<OfflinePageItem> offline_pages = CreateDummyRecentTabs({1, 2, 3}); + FireOfflinePageModelChanged(offline_pages); + EXPECT_THAT(ReadDismissedIDsFromPrefs(), IsEmpty()); + + provider()->DismissSuggestion(GetDummySuggestionId(2)); + EXPECT_THAT(ReadDismissedIDsFromPrefs(), SizeIs(1)); + + FireOfflinePageDeleted(offline_pages[1]); + EXPECT_THAT(ReadDismissedIDsFromPrefs(), IsEmpty()); +} + +TEST_F(RecentTabSuggestionsProviderTest, ShouldClearDismissedOnFetch) { + FireOfflinePageModelChanged(CreateDummyRecentTabs({1, 2, 3})); + + provider()->DismissSuggestion(GetDummySuggestionId(2)); + provider()->DismissSuggestion(GetDummySuggestionId(3)); + EXPECT_THAT(ReadDismissedIDsFromPrefs(), SizeIs(2)); + + FireOfflinePageModelChanged(CreateDummyRecentTabs({2})); + EXPECT_THAT(ReadDismissedIDsFromPrefs(), SizeIs(1)); + + FireOfflinePageModelChanged(std::vector<OfflinePageItem>()); + EXPECT_THAT(ReadDismissedIDsFromPrefs(), IsEmpty()); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/physical_web_pages/DEPS b/chromium/components/ntp_snippets/physical_web_pages/DEPS new file mode 100644 index 00000000000..179f98f1617 --- /dev/null +++ b/chromium/components/ntp_snippets/physical_web_pages/DEPS @@ -0,0 +1,2 @@ +include_rules = [ +]
\ No newline at end of file diff --git a/chromium/components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider.cc b/chromium/components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider.cc new file mode 100644 index 00000000000..b87f20ed039 --- /dev/null +++ b/chromium/components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider.cc @@ -0,0 +1,125 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider.h" + +#include "base/bind.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_util.h" +#include "base/strings/utf_string_conversions.h" +#include "base/threading/thread_task_runner_handle.h" +#include "ui/gfx/image/image.h" + +namespace ntp_snippets { + +namespace { + +const size_t kMaxSuggestionsCount = 10; + +} // namespace + +// TODO(vitaliii): remove when Physical Web C++ interface is provided. +UrlInfo::UrlInfo() = default; +UrlInfo::~UrlInfo() = default; +UrlInfo::UrlInfo(const UrlInfo& other) = default; + +PhysicalWebPageSuggestionsProvider::PhysicalWebPageSuggestionsProvider( + ContentSuggestionsProvider::Observer* observer, + CategoryFactory* category_factory) + : ContentSuggestionsProvider(observer, category_factory), + category_status_(CategoryStatus::AVAILABLE_LOADING), + provided_category_(category_factory->FromKnownCategory( + KnownCategories::PHYSICAL_WEB_PAGES)) { + observer->OnCategoryStatusChanged(this, provided_category_, category_status_); +} + +PhysicalWebPageSuggestionsProvider::~PhysicalWebPageSuggestionsProvider() = + default; + +void PhysicalWebPageSuggestionsProvider::OnDisplayableUrlsChanged( + const std::vector<UrlInfo>& urls) { + NotifyStatusChanged(CategoryStatus::AVAILABLE); + std::vector<ContentSuggestion> suggestions; + + for (const UrlInfo& url_info : urls) { + if (suggestions.size() >= kMaxSuggestionsCount) break; + + ContentSuggestion suggestion(provided_category_, url_info.site_url.spec(), + url_info.site_url); + + suggestion.set_title(base::UTF8ToUTF16(url_info.title)); + suggestion.set_snippet_text(base::UTF8ToUTF16(url_info.description)); + suggestion.set_publish_date(url_info.scan_time); + suggestion.set_publisher_name(base::UTF8ToUTF16(url_info.site_url.host())); + suggestions.push_back(std::move(suggestion)); + } + + observer()->OnNewSuggestions(this, provided_category_, + std::move(suggestions)); +} + +CategoryStatus PhysicalWebPageSuggestionsProvider::GetCategoryStatus( + Category category) { + return category_status_; +} + +CategoryInfo PhysicalWebPageSuggestionsProvider::GetCategoryInfo( + Category category) { + // TODO(vitaliii): Use a proper string once it's been agreed on. + return CategoryInfo(base::ASCIIToUTF16("Physical web pages"), + ContentSuggestionsCardLayout::MINIMAL_CARD, + /* has_more_button */ true, + /* show_if_empty */ false); +} + +void PhysicalWebPageSuggestionsProvider::DismissSuggestion( + const ContentSuggestion::ID& suggestion_id) { + // TODO(vitaliii): Implement this and then + // ClearDismissedSuggestionsForDebugging. +} + +void PhysicalWebPageSuggestionsProvider::FetchSuggestionImage( + const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) { + // TODO(vitaliii): Implement. + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::Bind(callback, gfx::Image())); +} + +void PhysicalWebPageSuggestionsProvider::ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) { + ClearDismissedSuggestionsForDebugging(provided_category_); +} + +void PhysicalWebPageSuggestionsProvider::ClearCachedSuggestions( + Category category) { + // Ignored +} + +void PhysicalWebPageSuggestionsProvider::GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) { + // Not implemented. + callback.Run(std::vector<ContentSuggestion>()); +} + +void PhysicalWebPageSuggestionsProvider::ClearDismissedSuggestionsForDebugging( + Category category) { + // TODO(vitaliii): Implement when dismissed suggestions are supported. +} + +//////////////////////////////////////////////////////////////////////////////// +// Private methods + +// Updates the |category_status_| and notifies the |observer_|, if necessary. +void PhysicalWebPageSuggestionsProvider::NotifyStatusChanged( + CategoryStatus new_status) { + if (category_status_ == new_status) return; + category_status_ = new_status; + observer()->OnCategoryStatusChanged(this, provided_category_, new_status); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider.h b/chromium/components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider.h new file mode 100644 index 00000000000..934dd74038e --- /dev/null +++ b/chromium/components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider.h @@ -0,0 +1,72 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_PHYSICAL_WEB_PAGES_PHYSICAL_WEB_PAGE_SUGGESTIONS_PROVIDER_H_ +#define COMPONENTS_NTP_SNIPPETS_PHYSICAL_WEB_PAGES_PHYSICAL_WEB_PAGE_SUGGESTIONS_PROVIDER_H_ + +#include <string> +#include <vector> + +#include "base/callback_forward.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/category_status.h" +#include "components/ntp_snippets/content_suggestion.h" +#include "components/ntp_snippets/content_suggestions_provider.h" + +namespace gfx { +class Image; +} // namespace gfx + +namespace ntp_snippets { + +// TODO(vitaliii): remove when Physical Web C++ interface is provided. +struct UrlInfo { + UrlInfo(); + UrlInfo(const UrlInfo& other); + ~UrlInfo(); + base::Time scan_time; + GURL site_url; + std::string title; + std::string description; +}; + +// Provides content suggestions from the Physical Web Service. +class PhysicalWebPageSuggestionsProvider : public ContentSuggestionsProvider { + public: + PhysicalWebPageSuggestionsProvider( + ContentSuggestionsProvider::Observer* observer, + CategoryFactory* category_factory); + ~PhysicalWebPageSuggestionsProvider() override; + + void OnDisplayableUrlsChanged(const std::vector<UrlInfo>& urls); + + // ContentSuggestionsProvider implementation. + CategoryStatus GetCategoryStatus(Category category) override; + CategoryInfo GetCategoryInfo(Category category) override; + void DismissSuggestion(const ContentSuggestion::ID& suggestion_id) override; + void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) override; + void ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) override; + void ClearCachedSuggestions(Category category) override; + void GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) override; + void ClearDismissedSuggestionsForDebugging(Category category) override; + + private: + void NotifyStatusChanged(CategoryStatus new_status); + + CategoryStatus category_status_; + const Category provided_category_; + + DISALLOW_COPY_AND_ASSIGN(PhysicalWebPageSuggestionsProvider); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_PHYSICAL_WEB_PAGES_PHYSICAL_WEB_PAGE_SUGGESTIONS_PROVIDER_H_ diff --git a/chromium/components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider_unittest.cc b/chromium/components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider_unittest.cc new file mode 100644 index 00000000000..b70d2c3d9cb --- /dev/null +++ b/chromium/components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider_unittest.cc @@ -0,0 +1,63 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/physical_web_pages/physical_web_page_suggestions_provider.h" + +#include <string> +#include <vector> + +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "components/ntp_snippets/mock_content_suggestions_provider_observer.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using testing::UnorderedElementsAre; + +namespace ntp_snippets { + +namespace { + +UrlInfo CreateUrlInfo(const std::string& site_url) { + UrlInfo url_info; + url_info.site_url = GURL(site_url); + return url_info; +} + +std::vector<UrlInfo> CreateUrlInfos(const std::vector<std::string>& site_urls) { + std::vector<UrlInfo> url_infos; + for (const std::string& site_url : site_urls) { + url_infos.emplace_back(CreateUrlInfo(site_url)); + } + return url_infos; +} + +MATCHER_P(HasUrl, url, "") { + *result_listener << "expected URL: " << url + << "has URL: " << arg.url().spec(); + return arg.url().spec() == url; +} + +} // namespace + +TEST(PhysicalWebPageSuggestionsProviderTest, ShouldCreateSuggestions) { + MockContentSuggestionsProviderObserver observer; + CategoryFactory category_factory; + Category category = + category_factory.FromKnownCategory(KnownCategories::PHYSICAL_WEB_PAGES); + PhysicalWebPageSuggestionsProvider provider(&observer, &category_factory); + const std::string first_url = "http://test1.com/"; + const std::string second_url = "http://test2.com/"; + const std::vector<UrlInfo> url_infos = + CreateUrlInfos({first_url, second_url}); + + EXPECT_CALL(observer, + OnNewSuggestions( + &provider, category, + UnorderedElementsAre(HasUrl(first_url), HasUrl(second_url)))); + provider.OnDisplayableUrlsChanged(url_infos); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/pref_names.cc b/chromium/components/ntp_snippets/pref_names.cc index 57822e3f047..7b0c9bdcb3d 100644 --- a/chromium/components/ntp_snippets/pref_names.cc +++ b/chromium/components/ntp_snippets/pref_names.cc @@ -7,10 +7,55 @@ namespace ntp_snippets { namespace prefs { -const char kDeprecatedSnippets[] = "ntp_snippets.snippets"; -const char kDeprecatedDiscardedSnippets[] = "ntp_snippets.discarded_snippets"; +const char kEnableSnippets[] = "ntp_snippets.enable"; const char kSnippetHosts[] = "ntp_snippets.hosts"; +const char kRemoteSuggestionCategories[] = "ntp_snippets.remote_categories"; + +const char kSnippetBackgroundFetchingIntervalWifi[] = + "ntp_snippets.fetching_interval_wifi"; + +const char kSnippetBackgroundFetchingIntervalFallback[] = + "ntp_snippets.fetching_interval_fallback"; + +const char kSnippetFetcherRequestCount[] = + "ntp.request_throttler.suggestion_fetcher.count"; +const char kSnippetFetcherInteractiveRequestCount[] = + "ntp.request_throttler.suggestion_fetcher.interactive_count"; +const char kSnippetFetcherRequestsDay[] = + "ntp.request_throttler.suggestion_fetcher.day"; + +const char kSnippetThumbnailsRequestCount[] = + "ntp.request_throttler.suggestion_thumbnails.count"; +const char kSnippetThumbnailsInteractiveRequestCount[] = + "ntp.request_throttler.suggestion_thumbnails.interactive_count"; +const char kSnippetThumbnailsRequestsDay[] = + "ntp.request_throttler.suggestion_thumbnails.day"; + +const char kDismissedRecentOfflineTabSuggestions[] = + "ntp_suggestions.offline_pages.recent_tabs.dismissed_ids"; +const char kDismissedDownloadSuggestions[] = + "ntp_suggestions.offline_pages.downloads.dismissed_ids"; +const char kDismissedForeignSessionsSuggestions[] = + "ntp_suggestions.foreign_sessions.dismissed_ids"; + +const char kBookmarksFirstM54Start[] = + "ntp_suggestions.bookmarks.first_M54_start"; + +const char kUserClassifierAverageNTPOpenedPerHour[] = + "ntp_suggestions.user_classifier.average_ntp_opened_per_hour"; +const char kUserClassifierAverageSuggestionsShownPerHour[] = + "ntp_suggestions.user_classifier.average_suggestions_shown_per_hour"; +const char kUserClassifierAverageSuggestionsUsedPerHour[] = + "ntp_suggestions.user_classifier.average_suggestions_used_per_hour"; + +const char kUserClassifierLastTimeToOpenNTP[] = + "ntp_suggestions.user_classifier.last_time_to_open_ntp"; +const char kUserClassifierLastTimeToShowSuggestions[] = + "ntp_suggestions.user_classifier.last_time_to_show_suggestions"; +const char kUserClassifierLastTimeToUseSuggestions[] = + "ntp_suggestions.user_classifier.last_time_to_use_suggestions"; + } // namespace prefs } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/pref_names.h b/chromium/components/ntp_snippets/pref_names.h index 14f53cee403..2f9c3778abd 100644 --- a/chromium/components/ntp_snippets/pref_names.h +++ b/chromium/components/ntp_snippets/pref_names.h @@ -8,13 +8,67 @@ namespace ntp_snippets { namespace prefs { -// TODO(treib): Completely remove these after M53. -extern const char kDeprecatedSnippets[]; -extern const char kDeprecatedDiscardedSnippets[]; +// If set to false, remote suggestions are completely disabled. This is set by +// an enterprise policy. +extern const char kEnableSnippets[]; +// TODO(treib): Remove this after M56. extern const char kSnippetHosts[]; +// The pref name under which remote suggestion categories (including their ID +// and title) are stored. +extern const char kRemoteSuggestionCategories[]; + +// The pref name for the currently-scheduled background fetching interval when +// there is WiFi connectivity. +extern const char kSnippetBackgroundFetchingIntervalWifi[]; +// The pref name for the currently-scheduled background fetching interval when +// there is no WiFi connectivity. +extern const char kSnippetBackgroundFetchingIntervalFallback[]; + +// The pref name for today's count of NTPSnippetsFetcher requests, so far. +extern const char kSnippetFetcherRequestCount[]; +// The pref name for today's count of NTPSnippetsFetcher interactive requests. +extern const char kSnippetFetcherInteractiveRequestCount[]; +// The pref name for the current day for the counter of NTPSnippetsFetcher +// requests. +extern const char kSnippetFetcherRequestsDay[]; + +// The pref name for today's count of requests for article thumbnails, so far. +extern const char kSnippetThumbnailsRequestCount[]; +// The pref name for today's count of interactive requests for article +// thumbnails, so far. +extern const char kSnippetThumbnailsInteractiveRequestCount[]; +// The pref name for the current day for the counter of requests for article +// thumbnails. +extern const char kSnippetThumbnailsRequestsDay[]; + +extern const char kDismissedRecentOfflineTabSuggestions[]; +extern const char kDismissedDownloadSuggestions[]; +extern const char kDismissedForeignSessionsSuggestions[]; + +// The pref name for the time when M54 was first started on the device. +extern const char kBookmarksFirstM54Start[]; + +// The pref name for the discounted average number of browsing sessions per hour +// that involve opening a new NTP. +extern const char kUserClassifierAverageNTPOpenedPerHour[]; +// The pref name for the discounted average number of browsing sessions per hour +// that involve opening showing the content suggestions. +extern const char kUserClassifierAverageSuggestionsShownPerHour[]; +// The pref name for the discounted average number of browsing sessions per hour +// that involve using content suggestions (i.e. opening one or clicking on the +// "More" button). +extern const char kUserClassifierAverageSuggestionsUsedPerHour[]; + +// The pref name for the last time a new NTP was opened. +extern const char kUserClassifierLastTimeToOpenNTP[]; +// The pref name for the last time content suggestions were shown to the user. +extern const char kUserClassifierLastTimeToShowSuggestions[]; +// The pref name for the last time content suggestions were used by the user. +extern const char kUserClassifierLastTimeToUseSuggestions[]; + } // namespace prefs } // namespace ntp_snippets -#endif +#endif // COMPONENTS_NTP_SNIPPETS_PREF_NAMES_H_ diff --git a/chromium/components/ntp_snippets/pref_util.cc b/chromium/components/ntp_snippets/pref_util.cc new file mode 100644 index 00000000000..a6288f92535 --- /dev/null +++ b/chromium/components/ntp_snippets/pref_util.cc @@ -0,0 +1,40 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/pref_util.h" + +#include <memory> + +#include "base/values.h" +#include "components/prefs/pref_service.h" + +namespace ntp_snippets { +namespace prefs { + +std::set<std::string> ReadDismissedIDsFromPrefs(const PrefService& pref_service, + const std::string& pref_name) { + std::set<std::string> dismissed_ids; + const base::ListValue* list = pref_service.GetList(pref_name); + for (const std::unique_ptr<base::Value>& value : *list) { + std::string dismissed_id; + bool success = value->GetAsString(&dismissed_id); + DCHECK(success) << "Failed to parse dismissed id from prefs param " + << pref_name << " into string."; + dismissed_ids.insert(dismissed_id); + } + return dismissed_ids; +} + +void StoreDismissedIDsToPrefs(PrefService* pref_service, + const std::string& pref_name, + const std::set<std::string>& dismissed_ids) { + base::ListValue list; + for (const std::string& dismissed_id : dismissed_ids) { + list.AppendString(dismissed_id); + } + pref_service->Set(pref_name, list); +} + +} // namespace prefs +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/pref_util.h b/chromium/components/ntp_snippets/pref_util.h new file mode 100644 index 00000000000..06ea838bbc5 --- /dev/null +++ b/chromium/components/ntp_snippets/pref_util.h @@ -0,0 +1,28 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_PREF_UTIL_H_ +#define COMPONENTS_NTP_SNIPPETS_PREF_UTIL_H_ + +#include <set> +#include <string> + +class PrefService; + +namespace ntp_snippets { +namespace prefs { + +// Reads a given preference and then deserializes it into a set of strings. +std::set<std::string> ReadDismissedIDsFromPrefs(const PrefService& pref_service, + const std::string& pref_name); + +// Serializes a set of strings into a given preference. +void StoreDismissedIDsToPrefs(PrefService* pref_service, + const std::string& pref_name, + const std::set<std::string>& dismissed_ids); + +} // namespace prefs +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_PREF_UTIL_H_ diff --git a/chromium/components/ntp_snippets/ntp_snippet.cc b/chromium/components/ntp_snippets/remote/ntp_snippet.cc index f7870f919d6..767d7e616da 100644 --- a/chromium/components/ntp_snippets/ntp_snippet.cc +++ b/chromium/components/ntp_snippets/remote/ntp_snippet.cc @@ -2,16 +2,14 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/ntp_snippets/ntp_snippet.h" +#include "components/ntp_snippets/remote/ntp_snippet.h" #include "base/memory/ptr_util.h" #include "base/strings/string_number_conversions.h" #include "base/strings/stringprintf.h" #include "base/values.h" -#include "components/ntp_snippets/content_suggestion.h" -#include "components/ntp_snippets/content_suggestion_category.h" -#include "components/ntp_snippets/content_suggestions_provider_type.h" -#include "components/ntp_snippets/proto/ntp_snippets.pb.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/remote/proto/ntp_snippets.pb.h" namespace { @@ -40,10 +38,21 @@ bool GetURLValue(const base::DictionaryValue& dict, namespace ntp_snippets { -NTPSnippet::NTPSnippet(const std::string& id) - : id_(id), score_(0), is_discarded_(false), best_source_index_(0) {} +const int kArticlesRemoteId = 1; +static_assert( + static_cast<int>(KnownCategories::ARTICLES) - + static_cast<int>(KnownCategories::REMOTE_CATEGORIES_OFFSET) == + kArticlesRemoteId, + "kArticlesRemoteId has a wrong value?!"); -NTPSnippet::~NTPSnippet() {} +NTPSnippet::NTPSnippet(const std::string& id, int remote_category_id) + : id_(id), + score_(0), + is_dismissed_(false), + remote_category_id_(remote_category_id), + best_source_index_(0) {} + +NTPSnippet::~NTPSnippet() = default; // static std::unique_ptr<NTPSnippet> NTPSnippet::CreateFromChromeReaderDictionary( @@ -57,7 +66,7 @@ std::unique_ptr<NTPSnippet> NTPSnippet::CreateFromChromeReaderDictionary( if (!content->GetString("url", &id) || id.empty()) return nullptr; - std::unique_ptr<NTPSnippet> snippet(new NTPSnippet(id)); + std::unique_ptr<NTPSnippet> snippet(new NTPSnippet(id, kArticlesRemoteId)); std::string title; if (content->GetString("title", &title)) @@ -140,25 +149,26 @@ std::unique_ptr<NTPSnippet> NTPSnippet::CreateFromChromeReaderDictionary( // static std::unique_ptr<NTPSnippet> NTPSnippet::CreateFromContentSuggestionsDictionary( - const base::DictionaryValue& dict) { - const base::ListValue* id_list; + const base::DictionaryValue& dict, + int remote_category_id) { + const base::ListValue* ids; std::string id; - if (!(dict.GetList("id", &id_list) && - id_list->GetString(0, &id))) { // TODO(sfiera): multiple IDs + if (!(dict.GetList("ids", &ids) && + ids->GetString(0, &id))) { // TODO(sfiera): multiple IDs return nullptr; } - auto snippet = base::MakeUnique<NTPSnippet>(id); + auto snippet = base::MakeUnique<NTPSnippet>(id, remote_category_id); snippet->sources_.emplace_back(GURL(), std::string(), GURL()); - auto source = &snippet->sources_.back(); + auto* source = &snippet->sources_.back(); snippet->best_source_index_ = 0; if (!(dict.GetString("title", &snippet->title_) && - dict.GetString("summaryText", &snippet->snippet_) && - GetTimeValue(dict, "publishTime", &snippet->publish_date_) && + dict.GetString("snippet", &snippet->snippet_) && + GetTimeValue(dict, "creationTime", &snippet->publish_date_) && GetTimeValue(dict, "expirationTime", &snippet->expiry_date_) && GetURLValue(dict, "imageUrl", &snippet->salient_image_url_) && - dict.GetString("publisherName", &source->publisher_name) && + dict.GetString("attribution", &source->publisher_name) && GetURLValue(dict, "fullPageUrl", &source->url))) { return nullptr; } @@ -177,7 +187,12 @@ std::unique_ptr<NTPSnippet> NTPSnippet::CreateFromProto( if (!proto.has_id() || proto.id().empty()) return nullptr; - std::unique_ptr<NTPSnippet> snippet(new NTPSnippet(proto.id())); + int remote_category_id = proto.has_remote_category_id() + ? proto.remote_category_id() + : kArticlesRemoteId; + + std::unique_ptr<NTPSnippet> snippet( + new NTPSnippet(proto.id(), remote_category_id)); snippet->set_title(proto.title()); snippet->set_snippet(proto.snippet()); @@ -186,7 +201,7 @@ std::unique_ptr<NTPSnippet> NTPSnippet::CreateFromProto( base::Time::FromInternalValue(proto.publish_date())); snippet->set_expiry_date(base::Time::FromInternalValue(proto.expiry_date())); snippet->set_score(proto.score()); - snippet->set_discarded(proto.discarded()); + snippet->set_dismissed(proto.dismissed()); for (int i = 0; i < proto.sources_size(); ++i) { const SnippetSourceProto& source_proto = proto.sources(i); @@ -196,7 +211,6 @@ std::unique_ptr<NTPSnippet> NTPSnippet::CreateFromProto( DLOG(WARNING) << "Invalid article url " << source_proto.url(); continue; } - std::string publisher_name = source_proto.publisher_name(); GURL amp_url; if (source_proto.has_amp_url()) { amp_url = GURL(source_proto.amp_url()); @@ -204,7 +218,8 @@ std::unique_ptr<NTPSnippet> NTPSnippet::CreateFromProto( << source_proto.amp_url(); } - snippet->add_source(SnippetSource(url, publisher_name, amp_url)); + snippet->add_source( + SnippetSource(url, source_proto.publisher_name(), amp_url)); } if (snippet->sources_.empty()) { @@ -232,7 +247,8 @@ SnippetProto NTPSnippet::ToProto() const { if (!expiry_date_.is_null()) result.set_expiry_date(expiry_date_.ToInternalValue()); result.set_score(score_); - result.set_discarded(is_discarded_); + result.set_dismissed(is_dismissed_); + result.set_remote_category_id(remote_category_id_); for (const SnippetSource& source : sources_) { SnippetSourceProto* source_proto = result.add_sources(); @@ -246,19 +262,6 @@ SnippetProto NTPSnippet::ToProto() const { return result; } -std::unique_ptr<ContentSuggestion> NTPSnippet::ToContentSuggestion() const { - std::unique_ptr<ContentSuggestion> result(new ContentSuggestion( - id_, ContentSuggestionsProviderType::ARTICLES, - ContentSuggestionCategory::ARTICLE, best_source().url)); - result->set_amp_url(best_source().amp_url); - result->set_title(title_); - result->set_snippet_text(snippet_); - result->set_publish_date(publish_date_); - result->set_publisher_name(best_source().publisher_name); - result->set_score(score_); - return result; -} - // static base::Time NTPSnippet::TimeFromJsonString(const std::string& timestamp_str) { int64_t timestamp; diff --git a/chromium/components/ntp_snippets/ntp_snippet.h b/chromium/components/ntp_snippets/remote/ntp_snippet.h index 416cffca7db..d169eb8b387 100644 --- a/chromium/components/ntp_snippets/ntp_snippet.h +++ b/chromium/components/ntp_snippets/remote/ntp_snippet.h @@ -2,24 +2,26 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_NTP_SNIPPETS_NTP_SNIPPET_H_ -#define COMPONENTS_NTP_SNIPPETS_NTP_SNIPPET_H_ +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPET_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPET_H_ +#include <map> #include <memory> #include <string> #include <vector> #include "base/macros.h" #include "base/time/time.h" -#include "components/ntp_snippets/content_suggestion.h" #include "url/gurl.h" namespace base { class DictionaryValue; -} +} // namespace base namespace ntp_snippets { +extern const int kArticlesRemoteId; + class SnippetProto; struct SnippetSource { @@ -39,7 +41,7 @@ class NTPSnippet { // Creates a new snippet with the given |id|. // Public for testing only - create snippets using the Create* methods below. // TODO(treib): Make this private and add a CreateSnippetForTest? - NTPSnippet(const std::string& id); + NTPSnippet(const std::string& id, int remote_category_id); ~NTPSnippet(); @@ -54,7 +56,8 @@ class NTPSnippet { // Suggestions. Returns a null pointer if the dictionary doesn't correspond to // a valid snippet. Maps field names to Chrome Reader field names. static std::unique_ptr<NTPSnippet> CreateFromContentSuggestionsDictionary( - const base::DictionaryValue& dict); + const base::DictionaryValue& dict, + int remote_category_id); // Creates an NTPSnippet from a protocol buffer. Returns a null pointer if the // protocol buffer doesn't correspond to a valid snippet. @@ -65,8 +68,6 @@ class NTPSnippet { // A unique ID for identifying the snippet. If initialized by // CreateFromChromeReaderDictionary() the relevant key is 'url'. - // TODO(treib): For now, the ID has to be a valid URL spec, otherwise - // fetching the salient image will fail. See TODO in ntp_snippets_service.cc. const std::string& id() const { return id_; } // Title of the snippet. @@ -123,10 +124,12 @@ class NTPSnippet { float score() const { return score_; } void set_score(float score) { score_ = score; } - bool is_discarded() const { return is_discarded_; } - void set_discarded(bool discarded) { is_discarded_ = discarded; } + bool is_dismissed() const { return is_dismissed_; } + void set_dismissed(bool dismissed) { is_dismissed_ = dismissed; } - std::unique_ptr<ContentSuggestion> ToContentSuggestion() const; + // The ID of the remote category this snippet belongs to, for use with + // CategoryFactory::FromRemoteCategory. + int remote_category_id() const { return remote_category_id_; } // Public for testing. static base::Time TimeFromJsonString(const std::string& timestamp_str); @@ -142,7 +145,8 @@ class NTPSnippet { base::Time publish_date_; base::Time expiry_date_; float score_; - bool is_discarded_; + bool is_dismissed_; + int remote_category_id_; size_t best_source_index_; @@ -153,4 +157,4 @@ class NTPSnippet { } // namespace ntp_snippets -#endif // COMPONENTS_NTP_SNIPPETS_NTP_SNIPPET_H_ +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPET_H_ diff --git a/chromium/components/ntp_snippets/remote/ntp_snippet_unittest.cc b/chromium/components/ntp_snippets/remote/ntp_snippet_unittest.cc new file mode 100644 index 00000000000..44498837e42 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/ntp_snippet_unittest.cc @@ -0,0 +1,268 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/ntp_snippet.h" + +#include "base/json/json_reader.h" +#include "base/values.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace ntp_snippets { +namespace { + +std::unique_ptr<NTPSnippet> SnippetFromContentSuggestionJSON( + const std::string& json) { + auto json_value = base::JSONReader::Read(json); + base::DictionaryValue* json_dict; + if (!json_value->GetAsDictionary(&json_dict)) { + return nullptr; + } + return NTPSnippet::CreateFromContentSuggestionsDictionary(*json_dict, + kArticlesRemoteId); +} + +TEST(NTPSnippetTest, FromChromeContentSuggestionsDictionary) { + const std::string kJsonStr = + "{" + " \"ids\" : [\"http://localhost/foobar\"]," + " \"title\" : \"Foo Barred from Baz\"," + " \"snippet\" : \"...\"," + " \"fullPageUrl\" : \"http://localhost/foobar\"," + " \"creationTime\" : \"2016-06-30T11:01:37.000Z\"," + " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," + " \"attribution\" : \"Foo News\"," + " \"imageUrl\" : \"http://localhost/foobar.jpg\"," + " \"ampUrl\" : \"http://localhost/amp\"," + " \"faviconUrl\" : \"http://localhost/favicon.ico\" " + "}"; + auto snippet = SnippetFromContentSuggestionJSON(kJsonStr); + ASSERT_THAT(snippet, testing::NotNull()); + + EXPECT_EQ(snippet->id(), "http://localhost/foobar"); + EXPECT_EQ(snippet->title(), "Foo Barred from Baz"); + EXPECT_EQ(snippet->snippet(), "..."); + EXPECT_EQ(snippet->salient_image_url(), GURL("http://localhost/foobar.jpg")); + auto unix_publish_date = snippet->publish_date() - base::Time::UnixEpoch(); + auto expiry_duration = snippet->expiry_date() - snippet->publish_date(); + EXPECT_FLOAT_EQ(unix_publish_date.InSecondsF(), 1467284497.000000f); + EXPECT_FLOAT_EQ(expiry_duration.InSecondsF(), 86400.000000f); + + EXPECT_EQ(snippet->best_source().publisher_name, "Foo News"); + EXPECT_EQ(snippet->best_source().url, GURL("http://localhost/foobar")); + EXPECT_EQ(snippet->best_source().amp_url, GURL("http://localhost/amp")); +} + +std::unique_ptr<NTPSnippet> SnippetFromChromeReaderDict( + std::unique_ptr<base::DictionaryValue> dict) { + if (!dict) { + return nullptr; + } + return NTPSnippet::CreateFromChromeReaderDictionary(*dict); +} + +std::unique_ptr<base::DictionaryValue> SnippetWithTwoSources() { + const std::string kJsonStr = + "{\n" + " \"contentInfo\": {\n" + " \"url\": \"http://url.com\",\n" + " \"title\": \"Source 1 Title\",\n" + " \"snippet\": \"Source 1 Snippet\",\n" + " \"thumbnailUrl\": \"http://url.com/thumbnail\",\n" + " \"creationTimestampSec\": 1234567890,\n" + " \"expiryTimestampSec\": 2345678901,\n" + " \"sourceCorpusInfo\": [{\n" + " \"corpusId\": \"http://source1.com\",\n" + " \"publisherData\": {\n" + " \"sourceName\": \"Source 1\"\n" + " },\n" + " \"ampUrl\": \"http://source1.amp.com\"\n" + " }, {\n" + " \"corpusId\": \"http://source2.com\",\n" + " \"publisherData\": {\n" + " \"sourceName\": \"Source 2\"\n" + " },\n" + " \"ampUrl\": \"http://source2.amp.com\"\n" + " }]\n" + " },\n" + " \"score\": 5.0\n" + "}\n"; + + auto json_value = base::JSONReader::Read(kJsonStr); + base::DictionaryValue* json_dict; + if (!json_value->GetAsDictionary(&json_dict)) { + return nullptr; + } + return json_dict->CreateDeepCopy(); +} + +TEST(NTPSnippetTest, TestMultipleSources) { + auto snippet = SnippetFromChromeReaderDict(SnippetWithTwoSources()); + ASSERT_THAT(snippet, testing::NotNull()); + + // Expect the first source to be chosen. + EXPECT_EQ(snippet->sources().size(), 2u); + EXPECT_EQ(snippet->id(), "http://url.com"); + EXPECT_EQ(snippet->best_source().url, GURL("http://source1.com")); + EXPECT_EQ(snippet->best_source().publisher_name, std::string("Source 1")); + EXPECT_EQ(snippet->best_source().amp_url, GURL("http://source1.amp.com")); +} + +TEST(NTPSnippetTest, TestMultipleIncompleteSources1) { + // Set Source 2 to have no AMP url, and Source 1 to have no publisher name + // Source 2 should win since we favor publisher name over amp url + auto dict = SnippetWithTwoSources(); + base::ListValue* sources; + ASSERT_TRUE(dict->GetList("contentInfo.sourceCorpusInfo", &sources)); + base::DictionaryValue* source; + ASSERT_TRUE(sources->GetDictionary(0, &source)); + source->Remove("publisherData.sourceName", nullptr); + ASSERT_TRUE(sources->GetDictionary(1, &source)); + source->Remove("ampUrl", nullptr); + + auto snippet = SnippetFromChromeReaderDict(std::move(dict)); + ASSERT_THAT(snippet, testing::NotNull()); + + EXPECT_EQ(snippet->sources().size(), 2u); + EXPECT_EQ(snippet->id(), "http://url.com"); + EXPECT_EQ(snippet->best_source().url, GURL("http://source2.com")); + EXPECT_EQ(snippet->best_source().publisher_name, std::string("Source 2")); + EXPECT_EQ(snippet->best_source().amp_url, GURL()); +} + +TEST(NTPSnippetTest, TestMultipleIncompleteSources2) { + // Set Source 1 to have no AMP url, and Source 2 to have no publisher name + // Source 1 should win in this case since we prefer publisher name to AMP url + auto dict = SnippetWithTwoSources(); + base::ListValue* sources; + ASSERT_TRUE(dict->GetList("contentInfo.sourceCorpusInfo", &sources)); + base::DictionaryValue* source; + ASSERT_TRUE(sources->GetDictionary(0, &source)); + source->Remove("ampUrl", nullptr); + ASSERT_TRUE(sources->GetDictionary(1, &source)); + source->Remove("publisherData.sourceName", nullptr); + + auto snippet = SnippetFromChromeReaderDict(std::move(dict)); + ASSERT_THAT(snippet, testing::NotNull()); + + EXPECT_EQ(snippet->sources().size(), 2u); + EXPECT_EQ(snippet->id(), "http://url.com"); + EXPECT_EQ(snippet->best_source().url, GURL("http://source1.com")); + EXPECT_EQ(snippet->best_source().publisher_name, std::string("Source 1")); + EXPECT_EQ(snippet->best_source().amp_url, GURL()); +} + +TEST(NTPSnippetTest, TestMultipleIncompleteSources3) { + // Set source 1 to have no AMP url and no source, and source 2 to only have + // amp url. There should be no snippets since we only add sources we consider + // complete + auto dict = SnippetWithTwoSources(); + base::ListValue* sources; + ASSERT_TRUE(dict->GetList("contentInfo.sourceCorpusInfo", &sources)); + base::DictionaryValue* source; + ASSERT_TRUE(sources->GetDictionary(0, &source)); + source->Remove("publisherData.sourceName", nullptr); + source->Remove("ampUrl", nullptr); + ASSERT_TRUE(sources->GetDictionary(1, &source)); + source->Remove("publisherData.sourceName", nullptr); + + auto snippet = SnippetFromChromeReaderDict(std::move(dict)); + ASSERT_THAT(snippet, testing::NotNull()); + ASSERT_FALSE(snippet->is_complete()); +} + +std::unique_ptr<base::DictionaryValue> SnippetWithThreeSources() { + const std::string kJsonStr = + "{\n" + " \"contentInfo\": {\n" + " \"url\": \"http://url.com\",\n" + " \"title\": \"Source 1 Title\",\n" + " \"snippet\": \"Source 1 Snippet\",\n" + " \"thumbnailUrl\": \"http://url.com/thumbnail\",\n" + " \"creationTimestampSec\": 1234567890,\n" + " \"expiryTimestampSec\": 2345678901,\n" + " \"sourceCorpusInfo\": [{\n" + " \"corpusId\": \"http://source1.com\",\n" + " \"publisherData\": {\n" + " \"sourceName\": \"Source 1\"\n" + " },\n" + " \"ampUrl\": \"http://source1.amp.com\"\n" + " }, {\n" + " \"corpusId\": \"http://source2.com\",\n" + " \"publisherData\": {\n" + " \"sourceName\": \"Source 2\"\n" + " },\n" + " \"ampUrl\": \"http://source2.amp.com\"\n" + " }, {\n" + " \"corpusId\": \"http://source3.com\",\n" + " \"publisherData\": {\n" + " \"sourceName\": \"Source 3\"\n" + " },\n" + " \"ampUrl\": \"http://source3.amp.com\"\n" + " }]\n" + " },\n" + " \"score\": 5.0\n" + "}\n"; + + auto json_value = base::JSONReader::Read(kJsonStr); + base::DictionaryValue* json_dict; + if (!json_value->GetAsDictionary(&json_dict)) { + return nullptr; + } + return json_dict->CreateDeepCopy(); +} + +TEST(NTPSnippetTest, TestMultipleCompleteSources1) { + // Test 2 complete sources, we should choose the first complete source + auto dict = SnippetWithThreeSources(); + base::ListValue* sources; + ASSERT_TRUE(dict->GetList("contentInfo.sourceCorpusInfo", &sources)); + base::DictionaryValue* source; + ASSERT_TRUE(sources->GetDictionary(1, &source)); + source->Remove("publisherData.sourceName", nullptr); + + auto snippet = SnippetFromChromeReaderDict(std::move(dict)); + ASSERT_THAT(snippet, testing::NotNull()); + + EXPECT_EQ(snippet->sources().size(), 3u); + EXPECT_EQ(snippet->id(), "http://url.com"); + EXPECT_EQ(snippet->best_source().url, GURL("http://source1.com")); + EXPECT_EQ(snippet->best_source().publisher_name, std::string("Source 1")); + EXPECT_EQ(snippet->best_source().amp_url, GURL("http://source1.amp.com")); +} + +TEST(NTPSnippetTest, TestMultipleCompleteSources2) { + // Test 2 complete sources, we should choose the first complete source + auto dict = SnippetWithThreeSources(); + base::ListValue* sources; + ASSERT_TRUE(dict->GetList("contentInfo.sourceCorpusInfo", &sources)); + base::DictionaryValue* source; + ASSERT_TRUE(sources->GetDictionary(0, &source)); + source->Remove("publisherData.sourceName", nullptr); + + auto snippet = SnippetFromChromeReaderDict(std::move(dict)); + ASSERT_THAT(snippet, testing::NotNull()); + + EXPECT_EQ(snippet->sources().size(), 3u); + EXPECT_EQ(snippet->id(), "http://url.com"); + EXPECT_EQ(snippet->best_source().url, GURL("http://source2.com")); + EXPECT_EQ(snippet->best_source().publisher_name, std::string("Source 2")); + EXPECT_EQ(snippet->best_source().amp_url, GURL("http://source2.amp.com")); +} + +TEST(NTPSnippetTest, TestMultipleCompleteSources3) { + // Test 3 complete sources, we should choose the first complete source + auto dict = SnippetWithThreeSources(); + auto snippet = SnippetFromChromeReaderDict(std::move(dict)); + ASSERT_THAT(snippet, testing::NotNull()); + + EXPECT_EQ(snippet->sources().size(), 3u); + EXPECT_EQ(snippet->id(), "http://url.com"); + EXPECT_EQ(snippet->best_source().url, GURL("http://source1.com")); + EXPECT_EQ(snippet->best_source().publisher_name, std::string("Source 1")); + EXPECT_EQ(snippet->best_source().amp_url, GURL("http://source1.amp.com")); +} + +} // namespace +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_database.cc b/chromium/components/ntp_snippets/remote/ntp_snippets_database.cc index 4b142796158..c11143a172d 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_database.cc +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_database.cc @@ -2,13 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/ntp_snippets/ntp_snippets_database.h" +#include "components/ntp_snippets/remote/ntp_snippets_database.h" #include <utility> #include "base/files/file_path.h" #include "components/leveldb_proto/proto_database_impl.h" -#include "components/ntp_snippets/proto/ntp_snippets.pb.h" +#include "components/ntp_snippets/remote/proto/ntp_snippets.pb.h" using leveldb_proto::ProtoDatabaseImpl; @@ -22,7 +22,7 @@ const char kImageDatabaseUMAClientName[] = "NTPSnippetImages"; const char kSnippetDatabaseFolder[] = "snippets"; const char kImageDatabaseFolder[] = "images"; -} +} // namespace namespace ntp_snippets { @@ -47,7 +47,7 @@ NTPSnippetsDatabase::NTPSnippetsDatabase( weak_ptr_factory_.GetWeakPtr())); } -NTPSnippetsDatabase::~NTPSnippetsDatabase() {} +NTPSnippetsDatabase::~NTPSnippetsDatabase() = default; bool NTPSnippetsDatabase::IsInitialized() const { return !IsErrorState() && database_initialized_ && @@ -84,17 +84,18 @@ void NTPSnippetsDatabase::SaveSnippets(const NTPSnippet::PtrVector& snippets) { } void NTPSnippetsDatabase::DeleteSnippet(const std::string& snippet_id) { - DeleteSnippetsImpl( - base::WrapUnique(new std::vector<std::string>(1, snippet_id))); + DeleteSnippets(base::MakeUnique<std::vector<std::string>>(1, snippet_id)); } void NTPSnippetsDatabase::DeleteSnippets( - const NTPSnippet::PtrVector& snippets) { - std::unique_ptr<std::vector<std::string>> keys_to_remove( - new std::vector<std::string>()); - for (const std::unique_ptr<NTPSnippet>& snippet : snippets) - keys_to_remove->emplace_back(snippet->id()); - DeleteSnippetsImpl(std::move(keys_to_remove)); + std::unique_ptr<std::vector<std::string>> keys_to_remove) { + DCHECK(IsInitialized()); + + std::unique_ptr<KeyEntryVector> entries_to_save(new KeyEntryVector()); + database_->UpdateEntries(std::move(entries_to_save), + std::move(keys_to_remove), + base::Bind(&NTPSnippetsDatabase::OnDatabaseSaved, + weak_ptr_factory_.GetWeakPtr())); } void NTPSnippetsDatabase::LoadImage(const std::string& snippet_id, @@ -117,15 +118,31 @@ void NTPSnippetsDatabase::SaveImage(const std::string& snippet_id, entries_to_save->emplace_back(snippet_id, std::move(image_proto)); image_database_->UpdateEntries( - std::move(entries_to_save), - base::WrapUnique(new std::vector<std::string>()), + std::move(entries_to_save), base::MakeUnique<std::vector<std::string>>(), base::Bind(&NTPSnippetsDatabase::OnImageDatabaseSaved, weak_ptr_factory_.GetWeakPtr())); } void NTPSnippetsDatabase::DeleteImage(const std::string& snippet_id) { - DeleteImagesImpl( - base::WrapUnique(new std::vector<std::string>(1, snippet_id))); + DeleteImages(base::MakeUnique<std::vector<std::string>>(1, snippet_id)); +} + +void NTPSnippetsDatabase::DeleteImages( + std::unique_ptr<std::vector<std::string>> keys_to_remove) { + DCHECK(IsInitialized()); + image_database_->UpdateEntries( + base::MakeUnique<ImageKeyEntryVector>(), std::move(keys_to_remove), + base::Bind(&NTPSnippetsDatabase::OnImageDatabaseSaved, + weak_ptr_factory_.GetWeakPtr())); +} + +void NTPSnippetsDatabase::GarbageCollectImages( + std::unique_ptr<std::set<std::string>> alive_snippet_ids) { + DCHECK(image_database_initialized_); + image_database_->LoadKeys( + base::Bind(&NTPSnippetsDatabase::DeleteUnreferencedImages, + weak_ptr_factory_.GetWeakPtr(), + base::Passed(std::move(alive_snippet_ids)))); } void NTPSnippetsDatabase::OnDatabaseInited(bool success) { @@ -169,7 +186,7 @@ void NTPSnippetsDatabase::OnDatabaseLoaded( // If any of the snippet protos couldn't be converted to actual snippets, // clean them up now. if (!keys_to_remove->empty()) - DeleteSnippetsImpl(std::move(keys_to_remove)); + DeleteSnippets(std::move(keys_to_remove)); } void NTPSnippetsDatabase::OnDatabaseSaved(bool success) { @@ -255,20 +272,6 @@ void NTPSnippetsDatabase::SaveSnippetsImpl( weak_ptr_factory_.GetWeakPtr())); } -void NTPSnippetsDatabase::DeleteSnippetsImpl( - std::unique_ptr<std::vector<std::string>> keys_to_remove) { - DCHECK(IsInitialized()); - - DeleteImagesImpl( - base::WrapUnique(new std::vector<std::string>(*keys_to_remove))); - - std::unique_ptr<KeyEntryVector> entries_to_save(new KeyEntryVector()); - database_->UpdateEntries(std::move(entries_to_save), - std::move(keys_to_remove), - base::Bind(&NTPSnippetsDatabase::OnDatabaseSaved, - weak_ptr_factory_.GetWeakPtr())); -} - void NTPSnippetsDatabase::LoadImageImpl(const std::string& snippet_id, const SnippetImageCallback& callback) { DCHECK(IsInitialized()); @@ -278,15 +281,23 @@ void NTPSnippetsDatabase::LoadImageImpl(const std::string& snippet_id, weak_ptr_factory_.GetWeakPtr(), callback)); } -void NTPSnippetsDatabase::DeleteImagesImpl( - std::unique_ptr<std::vector<std::string>> keys_to_remove) { - DCHECK(IsInitialized()); - - image_database_->UpdateEntries( - base::WrapUnique(new ImageKeyEntryVector()), - std::move(keys_to_remove), - base::Bind(&NTPSnippetsDatabase::OnImageDatabaseSaved, - weak_ptr_factory_.GetWeakPtr())); +void NTPSnippetsDatabase::DeleteUnreferencedImages( + std::unique_ptr<std::set<std::string>> references, + bool load_keys_success, + std::unique_ptr<std::vector<std::string>> image_keys) { + if (!load_keys_success) { + DVLOG(1) << "NTPSnippetsDatabase garbage collection failed."; + OnDatabaseError(); + return; + } + auto keys_to_remove = base::MakeUnique<std::vector<std::string>>(); + for (const std::string& key : *image_keys) { + if (references->count(key) == 0) { + keys_to_remove->emplace_back(key); + } + } + DeleteImages(std::move(keys_to_remove)); } + } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_database.h b/chromium/components/ntp_snippets/remote/ntp_snippets_database.h index 983166235f0..f711edbab44 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_database.h +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_database.h @@ -2,11 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_DATABASE_H_ -#define COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_DATABASE_H_ +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_DATABASE_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_DATABASE_H_ #include <memory> +#include <set> #include <string> +#include <utility> #include <vector> #include "base/callback.h" @@ -15,11 +17,11 @@ #include "base/memory/weak_ptr.h" #include "base/sequenced_task_runner.h" #include "components/leveldb_proto/proto_database.h" -#include "components/ntp_snippets/ntp_snippet.h" +#include "components/ntp_snippets/remote/ntp_snippet.h" namespace base { class FilePath; -} +} // namespace base namespace ntp_snippets { @@ -56,10 +58,10 @@ class NTPSnippetsDatabase { // Adds or updates all the given snippets. void SaveSnippets(const NTPSnippet::PtrVector& snippets); - // Deletes the snippet with the given ID, and its image. + // Deletes the snippet with the given ID. void DeleteSnippet(const std::string& snippet_id); - // Deletes all the given snippets (identified by their IDs) and their images. - void DeleteSnippets(const NTPSnippet::PtrVector& snippets); + // Deletes all the given snippets (identified by their IDs). + void DeleteSnippets(std::unique_ptr<std::vector<std::string>> keys_to_remove); // Loads the image data for the snippet with the given ID and passes it to // |callback|. Passes an empty string if not found. @@ -71,6 +73,12 @@ class NTPSnippetsDatabase { // Deletes the image data for the given snippet ID. void DeleteImage(const std::string& snippet_id); + // Deletes the image data for the given snippets (identified by their IDs). + void DeleteImages(std::unique_ptr<std::vector<std::string>> snippet_ids); + // Deletes all images which are not associated with any of the provided + // snippets. + void GarbageCollectImages( + std::unique_ptr<std::set<std::string>> alive_snippet_ids); private: friend class NTPSnippetsDatabaseTest; @@ -101,13 +109,13 @@ class NTPSnippetsDatabase { void LoadSnippetsImpl(const SnippetsCallback& callback); void SaveSnippetsImpl(std::unique_ptr<KeyEntryVector> entries_to_save); - void DeleteSnippetsImpl( - std::unique_ptr<std::vector<std::string>> keys_to_remove); void LoadImageImpl(const std::string& snippet_id, const SnippetImageCallback& callback); - void DeleteImagesImpl( - std::unique_ptr<std::vector<std::string>> keys_to_remove); + void DeleteUnreferencedImages( + std::unique_ptr<std::set<std::string>> references, + bool load_keys_success, + std::unique_ptr<std::vector<std::string>> image_keys); std::unique_ptr<leveldb_proto::ProtoDatabase<SnippetProto>> database_; bool database_initialized_; @@ -128,4 +136,4 @@ class NTPSnippetsDatabase { } // namespace ntp_snippets -#endif // COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_DATABASE_H_ +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_DATABASE_H_ diff --git a/chromium/components/ntp_snippets/ntp_snippets_database_unittest.cc b/chromium/components/ntp_snippets/remote/ntp_snippets_database_unittest.cc index 1640d43e313..3a8fc5c300f 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_database_unittest.cc +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_database_unittest.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/ntp_snippets/ntp_snippets_database.h" +#include "components/ntp_snippets/remote/ntp_snippets_database.h" #include <memory> @@ -11,6 +11,7 @@ #include "base/files/file_path.h" #include "base/files/scoped_temp_dir.h" #include "base/macros.h" +#include "base/memory/ptr_util.h" #include "base/message_loop/message_loop.h" #include "base/run_loop.h" #include "base/threading/thread_task_runner_handle.h" @@ -18,6 +19,7 @@ #include "testing/gtest/include/gtest/gtest.h" using testing::ElementsAre; +using testing::Eq; using testing::IsEmpty; using testing::Mock; using testing::_; @@ -37,13 +39,14 @@ bool operator==(const NTPSnippet& lhs, const NTPSnippet& rhs) { lhs.expiry_date() == rhs.expiry_date() && lhs.source_index() == rhs.source_index() && lhs.sources() == rhs.sources() && lhs.score() == rhs.score() && - lhs.is_discarded() == rhs.is_discarded(); + lhs.is_dismissed() == rhs.is_dismissed(); } namespace { std::unique_ptr<NTPSnippet> CreateTestSnippet() { - std::unique_ptr<NTPSnippet> snippet(new NTPSnippet("http://localhost")); + std::unique_ptr<NTPSnippet> snippet(new NTPSnippet("http://localhost", + kArticlesRemoteId)); snippet->add_source( SnippetSource(GURL("http://localhost"), "Publisher", GURL("http://amp"))); return snippet; @@ -74,12 +77,14 @@ class NTPSnippetsDatabaseTest : public testing::Test { // on the file. db_.reset(); - db_.reset(new NTPSnippetsDatabase(database_dir_.path(), + db_.reset(new NTPSnippetsDatabase(database_dir_.GetPath(), base::ThreadTaskRunnerHandle::Get())); } NTPSnippetsDatabase* db() { return db_.get(); } + // TODO(tschumann): MOCK_METHODS on non mock objects are an anti-pattern. + // Clean up. void OnSnippetsLoaded(NTPSnippet::PtrVector snippets) { OnSnippetsLoadedImpl(snippets); } @@ -249,7 +254,7 @@ TEST_F(NTPSnippetsDatabaseTest, Delete) { base::RunLoop().RunUntilIdle(); } -TEST_F(NTPSnippetsDatabaseTest, DeleteSnippetAlsoDeletesImage) { +TEST_F(NTPSnippetsDatabaseTest, DeleteSnippetDoesNotDeleteImage) { CreateDatabase(); base::RunLoop().RunUntilIdle(); ASSERT_TRUE(db()->IsInitialized()); @@ -278,6 +283,38 @@ TEST_F(NTPSnippetsDatabaseTest, DeleteSnippetAlsoDeletesImage) { // Delete the snippet. db()->DeleteSnippet(snippet->id()); + // Make sure the image is still there. + EXPECT_CALL(*this, OnImageLoaded(image_data)); + db()->LoadImage(snippet->id(), + base::Bind(&NTPSnippetsDatabaseTest::OnImageLoaded, + base::Unretained(this))); + base::RunLoop().RunUntilIdle(); +} + +TEST_F(NTPSnippetsDatabaseTest, DeleteImage) { + CreateDatabase(); + base::RunLoop().RunUntilIdle(); + ASSERT_TRUE(db()->IsInitialized()); + + std::unique_ptr<NTPSnippet> snippet = CreateTestSnippet(); + std::string image_data("pretty image"); + + // Store the image. + db()->SaveImage(snippet->id(), image_data); + base::RunLoop().RunUntilIdle(); + + // Make sure the image is there. + EXPECT_CALL(*this, OnImageLoaded(image_data)); + db()->LoadImage(snippet->id(), + base::Bind(&NTPSnippetsDatabaseTest::OnImageLoaded, + base::Unretained(this))); + base::RunLoop().RunUntilIdle(); + + Mock::VerifyAndClearExpectations(this); + + // Delete the snippet. + db()->DeleteImage(snippet->id()); + // Make sure the image is gone. EXPECT_CALL(*this, OnImageLoaded(std::string())); db()->LoadImage(snippet->id(), @@ -286,4 +323,49 @@ TEST_F(NTPSnippetsDatabaseTest, DeleteSnippetAlsoDeletesImage) { base::RunLoop().RunUntilIdle(); } +namespace { + +void LoadExpectedImage(NTPSnippetsDatabase* db, + const std::string& id, + const std::string& expected_data) { + base::RunLoop run_loop; + db->LoadImage(id, base::Bind( + [](base::Closure signal, std::string expected_data, + std::string actual_data) { + EXPECT_THAT(actual_data, Eq(expected_data)); + signal.Run(); + }, + run_loop.QuitClosure(), expected_data)); + run_loop.Run(); +} + +} // namespace + +TEST_F(NTPSnippetsDatabaseTest, ShouldGarbageCollectImages) { + CreateDatabase(); + base::RunLoop().RunUntilIdle(); + ASSERT_TRUE(db()->IsInitialized()); + + // Store images. + db()->SaveImage("snippet-id-1", "pretty-image-1"); + db()->SaveImage("snippet-id-2", "pretty-image-2"); + db()->SaveImage("snippet-id-3", "pretty-image-3"); + base::RunLoop().RunUntilIdle(); + + // Make sure the to-be-garbage collected images are there. + LoadExpectedImage(db(), "snippet-id-1", "pretty-image-1"); + LoadExpectedImage(db(), "snippet-id-3", "pretty-image-3"); + + // Garbage collect all except the second. + db()->GarbageCollectImages(base::MakeUnique<std::set<std::string>>( + std::set<std::string>({"snippet-id-2"}))); + base::RunLoop().RunUntilIdle(); + + // Make sure the images are gone. + LoadExpectedImage(db(), "snippet-id-1", ""); + LoadExpectedImage(db(), "snippet-id-3", ""); + // Make sure the second still exists. + LoadExpectedImage(db(), "snippet-id-2", "pretty-image-2"); +} + } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_fetcher.cc b/chromium/components/ntp_snippets/remote/ntp_snippets_fetcher.cc index 4490ef24c3c..59a534460d5 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_fetcher.cc +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_fetcher.cc @@ -2,9 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/ntp_snippets/ntp_snippets_fetcher.h" +#include "components/ntp_snippets/remote/ntp_snippets_fetcher.h" -#include <stdlib.h> +#include <cstdlib> #include "base/command_line.h" #include "base/files/file_path.h" @@ -17,17 +17,17 @@ #include "base/strings/string_number_conversions.h" #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" +#include "base/strings/utf_string_conversions.h" #include "base/time/default_tick_clock.h" #include "base/values.h" #include "components/data_use_measurement/core/data_use_user_data.h" +#include "components/ntp_snippets/category_factory.h" #include "components/ntp_snippets/ntp_snippets_constants.h" -#include "components/ntp_snippets/switches.h" #include "components/signin/core/browser/profile_oauth2_token_service.h" #include "components/signin/core/browser/signin_manager.h" #include "components/signin/core/browser/signin_manager_base.h" #include "components/variations/net/variations_http_headers.h" #include "components/variations/variations_associated_data.h" -#include "google_apis/google_api_keys.h" #include "net/base/load_flags.h" #include "net/http/http_request_headers.h" #include "net/http/http_response_headers.h" @@ -49,20 +49,11 @@ const char kChromeReaderApiScope[] = "https://www.googleapis.com/auth/webhistory"; const char kContentSuggestionsApiScope[] = "https://www.googleapis.com/auth/chrome-content-suggestions"; -const char kChromeReaderServer[] = - "https://chromereader-pa.googleapis.com/v1/fetch"; -const char kContentSuggestionsServer[] = - "https://chromecontentsuggestions-pa.googleapis.com/v1/snippets/list"; -const char kContentSuggestionsSandboxServer[] = - "https://chromecontentsuggestions-pa.sandbox.googleapis.com/v1/snippets/" - "list"; const char kSnippetsServerNonAuthorizedFormat[] = "%s?key=%s"; const char kAuthorizationRequestHeaderFormat[] = "Bearer %s"; // Variation parameter for personalizing fetching of snippets. const char kPersonalizationName[] = "fetching_personalization"; -// Variation parameter for setting whether to restrict to a passed set of hosts. -const char kHostRestrictionName[] = "fetching_host_restrict"; // Variation parameter for chrome-content-suggestions backend. const char kContentSuggestionsBackend[] = "content_suggestions_backend"; @@ -72,15 +63,13 @@ const char kPersonalizationPersonalString[] = "personal"; const char kPersonalizationNonPersonalString[] = "non_personal"; const char kPersonalizationBothString[] = "both"; // the default value -// Constants for possible values of the "fetching_host_restrict" parameter. -const char kHostRestrictionOnString[] = "on"; // the default value -const char kHostRestrictionOffString[] = "off"; +const int kMaxExcludedIds = 100; std::string FetchResultToString(NTPSnippetsFetcher::FetchResult result) { switch (result) { case NTPSnippetsFetcher::FetchResult::SUCCESS: return "OK"; - case NTPSnippetsFetcher::FetchResult::EMPTY_HOSTS: + case NTPSnippetsFetcher::FetchResult::DEPRECATED_EMPTY_HOSTS: return "Cannot fetch for empty hosts list."; case NTPSnippetsFetcher::FetchResult::URL_REQUEST_STATUS_ERROR: return "URLRequestStatus error"; @@ -92,6 +81,10 @@ std::string FetchResultToString(NTPSnippetsFetcher::FetchResult result) { return "Invalid / empty list."; case NTPSnippetsFetcher::FetchResult::OAUTH_TOKEN_ERROR: return "Error in obtaining an OAuth2 access token."; + case NTPSnippetsFetcher::FetchResult::INTERACTIVE_QUOTA_ERROR: + return "Out of interactive quota."; + case NTPSnippetsFetcher::FetchResult::NON_INTERACTIVE_QUOTA_ERROR: + return "Out of non-interactive quota."; case NTPSnippetsFetcher::FetchResult::RESULT_MAX: break; } @@ -99,6 +92,25 @@ std::string FetchResultToString(NTPSnippetsFetcher::FetchResult result) { return "Unknown error"; } +bool IsFetchPreconditionFailed(NTPSnippetsFetcher::FetchResult result) { + switch (result) { + case NTPSnippetsFetcher::FetchResult::DEPRECATED_EMPTY_HOSTS: + case NTPSnippetsFetcher::FetchResult::OAUTH_TOKEN_ERROR: + case NTPSnippetsFetcher::FetchResult::INTERACTIVE_QUOTA_ERROR: + case NTPSnippetsFetcher::FetchResult::NON_INTERACTIVE_QUOTA_ERROR: + return true; + case NTPSnippetsFetcher::FetchResult::SUCCESS: + case NTPSnippetsFetcher::FetchResult::URL_REQUEST_STATUS_ERROR: + case NTPSnippetsFetcher::FetchResult::HTTP_ERROR: + case NTPSnippetsFetcher::FetchResult::JSON_PARSE_ERROR: + case NTPSnippetsFetcher::FetchResult::INVALID_SNIPPET_CONTENT_ERROR: + case NTPSnippetsFetcher::FetchResult::RESULT_MAX: + return false; + } + NOTREACHED(); + return true; +} + std::string GetFetchEndpoint() { std::string endpoint = variations::GetVariationParamValue( ntp_snippets::kStudyName, kContentSuggestionsBackend); @@ -106,34 +118,41 @@ std::string GetFetchEndpoint() { } bool UsesChromeContentSuggestionsAPI(const GURL& endpoint) { - if (endpoint == GURL(kChromeReaderServer)) { + if (endpoint == GURL(kChromeReaderServer)) return false; - } else if (endpoint != GURL(kContentSuggestionsServer) && - endpoint != GURL(kContentSuggestionsSandboxServer)) { + + if (endpoint != GURL(kContentSuggestionsServer) && + endpoint != GURL(kContentSuggestionsDevServer) && + endpoint != GURL(kContentSuggestionsAlphaServer)) { LOG(WARNING) << "Unknown value for " << kContentSuggestionsBackend << ": " - << "assuming chromecontentsuggestions-style API"; + << "assuming chromecontentsuggestions-style API"; } return true; } // Creates snippets from dictionary values in |list| and adds them to // |snippets|. Returns true on success, false if anything went wrong. +// |remote_category_id| is only used if |content_suggestions_api| is true. bool AddSnippetsFromListValue(bool content_suggestions_api, + int remote_category_id, const base::ListValue& list, NTPSnippet::PtrVector* snippets) { for (const auto& value : list) { const base::DictionaryValue* dict = nullptr; - if (!value->GetAsDictionary(&dict)) + if (!value->GetAsDictionary(&dict)) { return false; + } std::unique_ptr<NTPSnippet> snippet; if (content_suggestions_api) { - snippet = NTPSnippet::CreateFromContentSuggestionsDictionary(*dict); + snippet = NTPSnippet::CreateFromContentSuggestionsDictionary( + *dict, remote_category_id); } else { snippet = NTPSnippet::CreateFromChromeReaderDictionary(*dict); } - if (!snippet) + if (!snippet) { return false; + } snippets->push_back(std::move(snippet)); } @@ -154,24 +173,42 @@ std::string PosixLocaleFromBCP47Language(const std::string& language_code) { } // namespace +NTPSnippetsFetcher::FetchedCategory::FetchedCategory(Category c) + : category(c) {} + +NTPSnippetsFetcher::FetchedCategory::FetchedCategory(FetchedCategory&&) = + default; +NTPSnippetsFetcher::FetchedCategory::~FetchedCategory() = default; +NTPSnippetsFetcher::FetchedCategory& NTPSnippetsFetcher::FetchedCategory:: +operator=(FetchedCategory&&) = default; + NTPSnippetsFetcher::NTPSnippetsFetcher( SigninManagerBase* signin_manager, OAuth2TokenService* token_service, scoped_refptr<URLRequestContextGetter> url_request_context_getter, + PrefService* pref_service, + CategoryFactory* category_factory, const ParseJSONCallback& parse_json_callback, - bool is_stable_channel) + const std::string& api_key) : OAuth2TokenService::Consumer("ntp_snippets"), signin_manager_(signin_manager), token_service_(token_service), waiting_for_refresh_token_(false), - url_request_context_getter_(url_request_context_getter), + url_request_context_getter_(std::move(url_request_context_getter)), + category_factory_(category_factory), parse_json_callback_(parse_json_callback), + count_to_fetch_(0), fetch_url_(GetFetchEndpoint()), fetch_api_(UsesChromeContentSuggestionsAPI(fetch_url_) ? CHROME_CONTENT_SUGGESTIONS_API : CHROME_READER_API), - is_stable_channel_(is_stable_channel), + api_key_(api_key), + interactive_request_(false), tick_clock_(new base::DefaultTickClock()), + request_throttler_( + pref_service, + RequestThrottler::RequestType::CONTENT_SUGGESTION_FETCHER), + oauth_token_retried_(false), weak_ptr_factory_(this) { // Parse the variation parameters and set the defaults if missing. std::string personalization = variations::GetVariationParamValue( @@ -187,18 +224,6 @@ NTPSnippetsFetcher::NTPSnippetsFetcher( << "Unknown value for " << kPersonalizationName << ": " << personalization; } - - std::string host_restriction = variations::GetVariationParamValue( - ntp_snippets::kStudyName, kHostRestrictionName); - if (host_restriction == kHostRestrictionOnString) { - use_host_restriction_ = true; - } else { - use_host_restriction_ = false; - LOG_IF(WARNING, !host_restriction.empty() && - host_restriction != kHostRestrictionOffString) - << "Unknown value for " << kHostRestrictionName << ": " - << host_restriction; - } } NTPSnippetsFetcher::~NTPSnippetsFetcher() { @@ -214,23 +239,31 @@ void NTPSnippetsFetcher::SetCallback( void NTPSnippetsFetcher::FetchSnippetsFromHosts( const std::set<std::string>& hosts, const std::string& language_code, - int count) { - hosts_ = hosts; - fetch_start_time_ = tick_clock_->NowTicks(); - - if (UsesHostRestrictions() && hosts_.empty()) { - FetchFinished(OptionalSnippets(), FetchResult::EMPTY_HOSTS, + const std::set<std::string>& excluded_ids, + int count, + bool interactive_request) { + if (!request_throttler_.DemandQuotaForRequest(interactive_request)) { + FetchFinished(OptionalFetchedCategories(), + interactive_request + ? FetchResult::INTERACTIVE_QUOTA_ERROR + : FetchResult::NON_INTERACTIVE_QUOTA_ERROR, /*extra_message=*/std::string()); return; } + hosts_ = hosts; + fetch_start_time_ = tick_clock_->NowTicks(); + excluded_ids_ = excluded_ids; + locale_ = PosixLocaleFromBCP47Language(language_code); count_to_fetch_ = count; bool use_authentication = UsesAuthentication(); + interactive_request_ = interactive_request; if (use_authentication && signin_manager_->IsAuthenticated()) { // Signed-in: get OAuth token --> fetch snippets. + oauth_token_retried_ = false; StartTokenRequest(); } else if (use_authentication && signin_manager_->AuthInProgress()) { // Currently signing in: wait for auth to finish (the refresh token) --> @@ -252,68 +285,87 @@ NTPSnippetsFetcher::RequestParams::RequestParams() only_return_personalized_results(), user_locale(), host_restricts(), - count_to_fetch() {} + count_to_fetch(), + interactive_request() {} NTPSnippetsFetcher::RequestParams::~RequestParams() = default; std::string NTPSnippetsFetcher::RequestParams::BuildRequest() { auto request = base::MakeUnique<base::DictionaryValue>(); - if (fetch_api == CHROME_READER_API) { - auto content_params = base::MakeUnique<base::DictionaryValue>(); - content_params->SetBoolean("only_return_personalized_results", - only_return_personalized_results); - - auto content_restricts = base::MakeUnique<base::ListValue>(); - for (const auto& metadata : {"TITLE", "SNIPPET", "THUMBNAIL"}) { - auto entry = base::MakeUnique<base::DictionaryValue>(); - entry->SetString("type", "METADATA"); - entry->SetString("value", metadata); - content_restricts->Append(std::move(entry)); - } - - auto content_selectors = base::MakeUnique<base::ListValue>(); - for (const auto& host : host_restricts) { - auto entry = base::MakeUnique<base::DictionaryValue>(); - entry->SetString("type", "HOST_RESTRICT"); - entry->SetString("value", host); - content_selectors->Append(std::move(entry)); + switch (fetch_api) { + case CHROME_READER_API: { + auto content_params = base::MakeUnique<base::DictionaryValue>(); + content_params->SetBoolean("only_return_personalized_results", + only_return_personalized_results); + + auto content_restricts = base::MakeUnique<base::ListValue>(); + for (const auto* metadata : {"TITLE", "SNIPPET", "THUMBNAIL"}) { + auto entry = base::MakeUnique<base::DictionaryValue>(); + entry->SetString("type", "METADATA"); + entry->SetString("value", metadata); + content_restricts->Append(std::move(entry)); + } + + auto content_selectors = base::MakeUnique<base::ListValue>(); + for (const auto& host : host_restricts) { + auto entry = base::MakeUnique<base::DictionaryValue>(); + entry->SetString("type", "HOST_RESTRICT"); + entry->SetString("value", host); + content_selectors->Append(std::move(entry)); + } + + auto local_scoring_params = base::MakeUnique<base::DictionaryValue>(); + local_scoring_params->Set("content_params", std::move(content_params)); + local_scoring_params->Set("content_restricts", + std::move(content_restricts)); + local_scoring_params->Set("content_selectors", + std::move(content_selectors)); + + auto global_scoring_params = base::MakeUnique<base::DictionaryValue>(); + global_scoring_params->SetInteger("num_to_return", count_to_fetch); + global_scoring_params->SetInteger("sort_type", 1); + + auto advanced = base::MakeUnique<base::DictionaryValue>(); + advanced->Set("local_scoring_params", std::move(local_scoring_params)); + advanced->Set("global_scoring_params", std::move(global_scoring_params)); + + request->SetString("response_detail_level", "STANDARD"); + request->Set("advanced_options", std::move(advanced)); + if (!obfuscated_gaia_id.empty()) { + request->SetString("obfuscated_gaia_id", obfuscated_gaia_id); + } + if (!user_locale.empty()) { + request->SetString("user_locale", user_locale); + } + break; } - auto local_scoring_params = base::MakeUnique<base::DictionaryValue>(); - local_scoring_params->Set("content_params", std::move(content_params)); - local_scoring_params->Set("content_restricts", - std::move(content_restricts)); - local_scoring_params->Set("content_selectors", - std::move(content_selectors)); - - auto global_scoring_params = base::MakeUnique<base::DictionaryValue>(); - global_scoring_params->SetInteger("num_to_return", count_to_fetch); - global_scoring_params->SetInteger("sort_type", 1); - - auto advanced = base::MakeUnique<base::DictionaryValue>(); - advanced->Set("local_scoring_params", std::move(local_scoring_params)); - advanced->Set("global_scoring_params", std::move(global_scoring_params)); - - request->SetString("response_detail_level", "STANDARD"); - request->Set("advanced_options", std::move(advanced)); - if (!obfuscated_gaia_id.empty()) { - request->SetString("obfuscated_gaia_id", obfuscated_gaia_id); - } - if (!user_locale.empty()) { - request->SetString("user_locale", user_locale); - } - } else { - if (!user_locale.empty()) { - request->SetString("uiLanguage", user_locale); - } - auto regular_hosts = base::MakeUnique<base::ListValue>(); - for (const auto& host : host_restricts) { - regular_hosts->AppendString(host); + case CHROME_CONTENT_SUGGESTIONS_API: { + if (!user_locale.empty()) { + request->SetString("uiLanguage", user_locale); + } + + auto regular_hosts = base::MakeUnique<base::ListValue>(); + for (const auto& host : host_restricts) { + regular_hosts->AppendString(host); + } + request->Set("regularlyVisitedHostNames", std::move(regular_hosts)); + request->SetString("priority", interactive_request + ? "USER_ACTION" + : "BACKGROUND_PREFETCH"); + + auto excluded = base::MakeUnique<base::ListValue>(); + for (const auto& id : excluded_ids) { + excluded->AppendString(id); + if (excluded->GetSize() >= kMaxExcludedIds) + break; + } + request->Set("excludedSuggestionIds", std::move(excluded)); + + // TODO(sfiera): support authentication and personalization + // TODO(sfiera): support count_to_fetch + break; } - request->Set("regularlyVisitedHostName", std::move(regular_hosts)); - - // TODO(sfiera): support authentication and personalization - // TODO(sfiera): support count_to_fetch } std::string request_json; @@ -348,7 +400,8 @@ void NTPSnippetsFetcher::FetchSnippetsImpl(const GURL& url, url_fetcher_->SetUploadData("application/json", request); // Log the request for debugging network issues. VLOG(1) << "Sending a NTP snippets request to " << url << ":" << std::endl - << headers.ToString() << std::endl << request; + << headers.ToString() << std::endl + << request; // Fetchers are sometimes cancelled because a network change was detected. url_fetcher_->SetAutomaticallyRetryOnNetworkChanges(3); // Try to make fetching the files bit more robust even with poor connection. @@ -356,12 +409,6 @@ void NTPSnippetsFetcher::FetchSnippetsImpl(const GURL& url, url_fetcher_->Start(); } -bool NTPSnippetsFetcher::UsesHostRestrictions() const { - return use_host_restriction_ && - !base::CommandLine::ForCurrentProcess()->HasSwitch( - switches::kDontRestrict); -} - bool NTPSnippetsFetcher::UsesAuthentication() const { return (personalization_ == Personalization::kPersonal || personalization_ == Personalization::kBoth); @@ -369,17 +416,16 @@ bool NTPSnippetsFetcher::UsesAuthentication() const { void NTPSnippetsFetcher::FetchSnippetsNonAuthenticated() { // When not providing OAuth token, we need to pass the Google API key. - const std::string& key = is_stable_channel_ - ? google_apis::GetAPIKey() - : google_apis::GetNonStableAPIKey(); GURL url(base::StringPrintf(kSnippetsServerNonAuthorizedFormat, - fetch_url_.spec().c_str(), key.c_str())); + fetch_url_.spec().c_str(), api_key_.c_str())); RequestParams params; params.fetch_api = fetch_api_; - params.host_restricts = - UsesHostRestrictions() ? hosts_ : std::set<std::string>(); + params.host_restricts = hosts_; + params.excluded_ids = excluded_ids_; params.count_to_fetch = count_to_fetch_; + params.interactive_request = interactive_request_; + params.user_locale = locale_; FetchSnippetsImpl(url, std::string(), params.BuildRequest()); } @@ -392,9 +438,11 @@ void NTPSnippetsFetcher::FetchSnippetsAuthenticated( params.only_return_personalized_results = personalization_ == Personalization::kPersonal; params.user_locale = locale_; - params.host_restricts = - UsesHostRestrictions() ? hosts_ : std::set<std::string>(); + params.host_restricts = hosts_; + params.excluded_ids = excluded_ids_; params.count_to_fetch = count_to_fetch_; + params.interactive_request = interactive_request_; + // TODO(jkrcal, treib): Add unit-tests for authenticated fetches. FetchSnippetsImpl(fetch_url_, base::StringPrintf(kAuthorizationRequestHeaderFormat, oauth_access_token.c_str()), @@ -429,10 +477,19 @@ void NTPSnippetsFetcher::OnGetTokenFailure( const OAuth2TokenService::Request* request, const GoogleServiceAuthError& error) { oauth_request_.reset(); - DLOG(ERROR) << "Unable to get token: " << error.ToString() - << " - fetching the snippets without authentication."; + + if (!oauth_token_retried_ && + error.state() == GoogleServiceAuthError::State::REQUEST_CANCELED) { + // The request (especially on startup) can get reset by loading the refresh + // token - do it one more time. + oauth_token_retried_ = true; + StartTokenRequest(); + return; + } + + DLOG(ERROR) << "Unable to get token: " << error.ToString(); FetchFinished( - OptionalSnippets(), FetchResult::OAUTH_TOKEN_ERROR, + OptionalFetchedCategories(), FetchResult::OAUTH_TOKEN_ERROR, /*extra_message=*/base::StringPrintf(" (%s)", error.ToString().c_str())); } @@ -446,6 +503,7 @@ void NTPSnippetsFetcher::OnRefreshTokenAvailable( token_service_->RemoveObserver(this); waiting_for_refresh_token_ = false; + oauth_token_retried_ = false; StartTokenRequest(); } @@ -461,7 +519,8 @@ void NTPSnippetsFetcher::OnURLFetchComplete(const URLFetcher* source) { status.is_success() ? source->GetResponseCode() : status.error()); if (!status.is_success()) { - FetchFinished(OptionalSnippets(), FetchResult::URL_REQUEST_STATUS_ERROR, + FetchFinished(OptionalFetchedCategories(), + FetchResult::URL_REQUEST_STATUS_ERROR, /*extra_message=*/base::StringPrintf(" %d", status.error())); } else if (source->GetResponseCode() != net::HTTP_OK) { // TODO(jkrcal): https://crbug.com/609084 @@ -470,64 +529,122 @@ void NTPSnippetsFetcher::OnURLFetchComplete(const URLFetcher* source) { // fetch a new auth token). We should extract that into a common class // instead of adding it to every single class that uses auth tokens. FetchFinished( - OptionalSnippets(), FetchResult::HTTP_ERROR, + OptionalFetchedCategories(), FetchResult::HTTP_ERROR, /*extra_message=*/base::StringPrintf(" %d", source->GetResponseCode())); } else { - bool stores_result_to_string = source->GetResponseAsString( - &last_fetch_json_); + bool stores_result_to_string = + source->GetResponseAsString(&last_fetch_json_); DCHECK(stores_result_to_string); - parse_json_callback_.Run( - last_fetch_json_, - base::Bind(&NTPSnippetsFetcher::OnJsonParsed, - weak_ptr_factory_.GetWeakPtr()), - base::Bind(&NTPSnippetsFetcher::OnJsonError, - weak_ptr_factory_.GetWeakPtr())); + parse_json_callback_.Run(last_fetch_json_, + base::Bind(&NTPSnippetsFetcher::OnJsonParsed, + weak_ptr_factory_.GetWeakPtr()), + base::Bind(&NTPSnippetsFetcher::OnJsonError, + weak_ptr_factory_.GetWeakPtr())); } } -void NTPSnippetsFetcher::OnJsonParsed(std::unique_ptr<base::Value> parsed) { +bool NTPSnippetsFetcher::JsonToSnippets(const base::Value& parsed, + FetchedCategoriesVector* categories) { const base::DictionaryValue* top_dict = nullptr; - const base::ListValue* list = nullptr; - NTPSnippet::PtrVector snippets; - const std::string list_key = - fetch_api_ == CHROME_CONTENT_SUGGESTIONS_API ? "snippet" : "recos"; - if (!parsed->GetAsDictionary(&top_dict) || - !top_dict->GetList(list_key, &list) || - !AddSnippetsFromListValue(fetch_api_ == CHROME_CONTENT_SUGGESTIONS_API, - *list, &snippets)) { - LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_; - FetchFinished(OptionalSnippets(), - FetchResult::INVALID_SNIPPET_CONTENT_ERROR, + if (!parsed.GetAsDictionary(&top_dict)) { + return false; + } + + switch (fetch_api_) { + case CHROME_READER_API: { + const int kUnusedRemoteCategoryId = -1; + categories->push_back(FetchedCategory( + category_factory_->FromKnownCategory(KnownCategories::ARTICLES))); + const base::ListValue* recos = nullptr; + return top_dict->GetList("recos", &recos) && + AddSnippetsFromListValue(/*content_suggestions_api=*/false, + kUnusedRemoteCategoryId, + *recos, &categories->back().snippets); + } + + case CHROME_CONTENT_SUGGESTIONS_API: { + const base::ListValue* categories_value = nullptr; + if (!top_dict->GetList("categories", &categories_value)) { + return false; + } + + for (const auto& v : *categories_value) { + std::string utf8_title; + int remote_category_id = -1; + const base::DictionaryValue* category_value = nullptr; + if (!(v->GetAsDictionary(&category_value) && + category_value->GetString("localizedTitle", &utf8_title) && + category_value->GetInteger("id", &remote_category_id) && + (remote_category_id > 0))) { + return false; + } + + categories->push_back(FetchedCategory( + category_factory_->FromRemoteCategory(remote_category_id))); + categories->back().localized_title = base::UTF8ToUTF16(utf8_title); + + const base::ListValue* suggestions = nullptr; + if (!category_value->GetList("suggestions", &suggestions)) { + // Absence of a list of suggestions is treated as an empty list, which + // is permissible. + continue; + } + if (!AddSnippetsFromListValue( + /*content_suggestions_api=*/true, remote_category_id, + *suggestions, &categories->back().snippets)) { + return false; + } + } + return true; + } + } + NOTREACHED(); + return false; +} + +void NTPSnippetsFetcher::OnJsonParsed(std::unique_ptr<base::Value> parsed) { + FetchedCategoriesVector categories; + if (JsonToSnippets(*parsed, &categories)) { + FetchFinished(OptionalFetchedCategories(std::move(categories)), + FetchResult::SUCCESS, /*extra_message=*/std::string()); } else { - FetchFinished(OptionalSnippets(std::move(snippets)), FetchResult::SUCCESS, + LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_; + FetchFinished(OptionalFetchedCategories(), + FetchResult::INVALID_SNIPPET_CONTENT_ERROR, /*extra_message=*/std::string()); } } void NTPSnippetsFetcher::OnJsonError(const std::string& error) { - LOG(WARNING) << "Received invalid JSON (" << error << "): " - << last_fetch_json_; + LOG(WARNING) << "Received invalid JSON (" << error + << "): " << last_fetch_json_; FetchFinished( - OptionalSnippets(), FetchResult::JSON_PARSE_ERROR, + OptionalFetchedCategories(), FetchResult::JSON_PARSE_ERROR, /*extra_message=*/base::StringPrintf(" (error %s)", error.c_str())); } -void NTPSnippetsFetcher::FetchFinished(OptionalSnippets snippets, - FetchResult result, - const std::string& extra_message) { - DCHECK(result == FetchResult::SUCCESS || !snippets); +void NTPSnippetsFetcher::FetchFinished( + OptionalFetchedCategories fetched_categories, + FetchResult result, + const std::string& extra_message) { + DCHECK(result == FetchResult::SUCCESS || !fetched_categories); last_status_ = FetchResultToString(result) + extra_message; - UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", - tick_clock_->NowTicks() - fetch_start_time_); + // Don't record FetchTimes if the result indicates that a precondition + // failed and we never actually sent a network request + if (!IsFetchPreconditionFailed(result)) { + UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", + tick_clock_->NowTicks() - fetch_start_time_); + } UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult", static_cast<int>(result), static_cast<int>(FetchResult::RESULT_MAX)); + DVLOG(1) << "Fetch finished: " << last_status_; if (!snippets_available_callback_.is_null()) - snippets_available_callback_.Run(std::move(snippets)); + snippets_available_callback_.Run(std::move(fetched_categories)); } } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_fetcher.h b/chromium/components/ntp_snippets/remote/ntp_snippets_fetcher.h index 18cb4eca445..a72e644ff27 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_fetcher.h +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_fetcher.h @@ -2,12 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_FETCHER_H_ -#define COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_FETCHER_H_ +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_FETCHER_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_FETCHER_H_ #include <memory> #include <set> #include <string> +#include <utility> #include <vector> #include "base/callback.h" @@ -15,21 +16,20 @@ #include "base/optional.h" #include "base/time/tick_clock.h" #include "base/time/time.h" -#include "components/ntp_snippets/ntp_snippet.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/remote/ntp_snippet.h" +#include "components/ntp_snippets/remote/request_throttler.h" #include "google_apis/gaia/oauth2_token_service.h" #include "net/url_request/url_fetcher_delegate.h" #include "net/url_request/url_request_context_getter.h" +class PrefService; class SigninManagerBase; namespace base { class Value; } // namespace base -namespace net { -class HttpRequestHeaders; -} // namespace net - namespace ntp_snippets { // Fetches snippet data for the NTP from the server. @@ -44,24 +44,38 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, using ParseJSONCallback = base::Callback< void(const std::string&, const SuccessCallback&, const ErrorCallback&)>; - using OptionalSnippets = base::Optional<NTPSnippet::PtrVector>; + struct FetchedCategory { + Category category; + base::string16 localized_title; // Ignored for non-server categories. + NTPSnippet::PtrVector snippets; + + explicit FetchedCategory(Category c); + FetchedCategory(FetchedCategory&&); // = default, in .cc + ~FetchedCategory(); // = default, in .cc + FetchedCategory& operator=(FetchedCategory&&); // = default, in .cc + }; + using FetchedCategoriesVector = std::vector<FetchedCategory>; + using OptionalFetchedCategories = base::Optional<FetchedCategoriesVector>; + // |snippets| contains parsed snippets if a fetch succeeded. If problems // occur, |snippets| contains no value (no actual vector in base::Optional). // Error details can be retrieved using last_status(). using SnippetsAvailableCallback = - base::Callback<void(OptionalSnippets snippets)>; + base::Callback<void(OptionalFetchedCategories fetched_categories)>; // Enumeration listing all possible outcomes for fetch attempts. Used for UMA // histograms, so do not change existing values. Insert new values at the end, // and update the histogram definition. enum class FetchResult { SUCCESS, - EMPTY_HOSTS, + DEPRECATED_EMPTY_HOSTS, URL_REQUEST_STATUS_ERROR, HTTP_ERROR, JSON_PARSE_ERROR, INVALID_SNIPPET_CONTENT_ERROR, OAUTH_TOKEN_ERROR, + INTERACTIVE_QUOTA_ERROR, + NON_INTERACTIVE_QUOTA_ERROR, RESULT_MAX }; @@ -74,10 +88,12 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, NTPSnippetsFetcher( SigninManagerBase* signin_manager, - OAuth2TokenService* oauth2_token_service, + OAuth2TokenService* token_service, scoped_refptr<net::URLRequestContextGetter> url_request_context_getter, + PrefService* pref_service, + CategoryFactory* category_factory, const ParseJSONCallback& parse_json_callback, - bool is_stable_channel); + const std::string& api_key); ~NTPSnippetsFetcher() override; // Set a callback that is called when a new set of snippets are downloaded, @@ -85,14 +101,23 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, void SetCallback(const SnippetsAvailableCallback& callback); // Fetches snippets from the server. |hosts| restricts the results to a set of - // hosts, e.g. "www.google.com". An empty host set produces an error. + // hosts, e.g. "www.google.com". If |hosts| is empty, no host restrictions are + // applied. + // + // |excluded_ids| will be reported to the server; the server should not return + // suggestions with those IDs. // // If an ongoing fetch exists, it will be cancelled and a new one started, // without triggering an additional callback (i.e. not noticeable by // subscriber of SetCallback()). + // + // Fetches snippets only if the daily quota not exceeded, unless + // |interactive_request| is set to true (use only for user-initiated fetches). void FetchSnippetsFromHosts(const std::set<std::string>& hosts, const std::string& language_code, - int count); + const std::set<std::string>& excluded_ids, + int count, + bool interactive_request); // Debug string representing the status/result of the last fetch attempt. const std::string& last_status() const { return last_status_; } @@ -105,8 +130,9 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, // Returns the personalization setting of the fetcher. Personalization personalization() const { return personalization_; } - // Does the fetcher use host restrictions? - bool UsesHostRestrictions() const; + // Returns the URL endpoint used by the fetcher. + GURL fetch_url() const { return fetch_url_; } + // Does the fetcher use authentication to get personalized results? bool UsesAuthentication() const; @@ -122,6 +148,7 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, private: FRIEND_TEST_ALL_PREFIXES(NTPSnippetsFetcherTest, BuildRequestAuthenticated); FRIEND_TEST_ALL_PREFIXES(NTPSnippetsFetcherTest, BuildRequestUnauthenticated); + FRIEND_TEST_ALL_PREFIXES(NTPSnippetsFetcherTest, BuildRequestExcludedIds); enum FetchAPI { CHROME_READER_API, @@ -134,7 +161,9 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, bool only_return_personalized_results; std::string user_locale; std::set<std::string> host_restricts; + std::set<std::string> excluded_ids; int count_to_fetch; + bool interactive_request; RequestParams(); ~RequestParams(); @@ -163,9 +192,11 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, // URLFetcherDelegate implementation. void OnURLFetchComplete(const net::URLFetcher* source) override; + bool JsonToSnippets(const base::Value& parsed, + FetchedCategoriesVector* categories); void OnJsonParsed(std::unique_ptr<base::Value> parsed); void OnJsonError(const std::string& error); - void FetchFinished(OptionalSnippets snippets, + void FetchFinished(OptionalFetchedCategories fetched_categories, FetchResult result, const std::string& extra_message); @@ -178,6 +209,7 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, // Holds the URL request context. scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_; + CategoryFactory* const category_factory_; const ParseJSONCallback parse_json_callback_; base::TimeTicks fetch_start_time_; std::string last_status_; @@ -186,6 +218,9 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, // Hosts to restrict the snippets to. std::set<std::string> hosts_; + // Snippets to exclude from the results. + std::set<std::string> excluded_ids_; + // Count of snippets to fetch. int count_to_fetch_; @@ -203,21 +238,28 @@ class NTPSnippetsFetcher : public OAuth2TokenService::Consumer, // The callback to notify when new snippets get fetched. SnippetsAvailableCallback snippets_available_callback_; - // Flag for picking the right (stable/non-stable) API key for Chrome Reader. - bool is_stable_channel_; + // API key to use for non-authenticated requests. + const std::string api_key_; // The variant of the fetching to use, loaded from variation parameters. Personalization personalization_; - // Should we apply host restriction? It is loaded from variation parameters. - bool use_host_restriction_; + + // Is the request user initiated? + bool interactive_request_; // Allow for an injectable tick clock for testing. std::unique_ptr<base::TickClock> tick_clock_; + // Request throttler for limiting requests. + RequestThrottler request_throttler_; + + // When a token request gets canceled, we want to retry once. + bool oauth_token_retried_; + base::WeakPtrFactory<NTPSnippetsFetcher> weak_ptr_factory_; DISALLOW_COPY_AND_ASSIGN(NTPSnippetsFetcher); }; } // namespace ntp_snippets -#endif // COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_FETCHER_H_ +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_FETCHER_H_ diff --git a/chromium/components/ntp_snippets/ntp_snippets_fetcher_unittest.cc b/chromium/components/ntp_snippets/remote/ntp_snippets_fetcher_unittest.cc index a0f7893ac6e..daf478dc81f 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_fetcher_unittest.cc +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_fetcher_unittest.cc @@ -2,7 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/ntp_snippets/ntp_snippets_fetcher.h" +#include "components/ntp_snippets/remote/ntp_snippets_fetcher.h" + +#include <map> +#include <utility> #include "base/json/json_reader.h" #include "base/memory/ptr_util.h" @@ -12,15 +15,16 @@ #include "base/threading/thread_task_runner_handle.h" #include "base/time/time.h" #include "base/values.h" -#include "components/ntp_snippets/ntp_snippet.h" +#include "components/ntp_snippets/category_factory.h" #include "components/ntp_snippets/ntp_snippets_constants.h" +#include "components/ntp_snippets/remote/ntp_snippet.h" +#include "components/prefs/testing_pref_service.h" #include "components/signin/core/browser/account_tracker_service.h" #include "components/signin/core/browser/fake_profile_oauth2_token_service.h" #include "components/signin/core/browser/fake_signin_manager.h" #include "components/signin/core/browser/test_signin_client.h" #include "components/variations/entropy_provider.h" #include "components/variations/variations_associated_data.h" -#include "google_apis/google_api_keys.h" #include "net/url_request/test_url_fetcher_factory.h" #include "net/url_request/url_request_test_util.h" #include "testing/gmock/include/gmock/gmock.h" @@ -30,35 +34,46 @@ namespace ntp_snippets { namespace { +using testing::_; using testing::ElementsAre; using testing::Eq; using testing::IsEmpty; -using testing::IsNull; using testing::Not; using testing::NotNull; using testing::PrintToString; -using testing::SizeIs; using testing::StartsWith; +using testing::WithArg; -const char kTestChromeReaderUrlFormat[] = - "https://chromereader-pa.googleapis.com/v1/fetch?key=%s"; -const char kTestChromeContentSuggestionsUrlFormat[] = - "https://chromecontentsuggestions-pa.googleapis.com/v1/snippets/" - "list?key=%s"; -const char kContentSuggestionsBackend[] = - "https://chromecontentsuggestions-pa.googleapis.com/v1/snippets/list"; +const char kAPIKey[] = "fakeAPIkey"; +const char kTestChromeReaderUrl[] = + "https://chromereader-pa.googleapis.com/v1/fetch?key=fakeAPIkey"; +const char kTestChromeContentSuggestionsUrl[] = + "https://chromecontentsuggestions-pa.googleapis.com/v1/suggestions/" + "fetch?key=fakeAPIkey"; // Artificial time delay for JSON parsing. const int64_t kTestJsonParsingLatencyMs = 20; +ACTION_P(MovePointeeTo, ptr) { + *ptr = std::move(*arg0); +} + MATCHER(HasValue, "") { - return static_cast<bool>(arg); + return static_cast<bool>(*arg); +} + +MATCHER(IsEmptyArticleList, "is an empty list of articles") { + NTPSnippetsFetcher::OptionalFetchedCategories& fetched_categories = *arg; + return fetched_categories && fetched_categories->size() == 1 && + fetched_categories->begin()->snippets.empty(); } -MATCHER_P(PointeeSizeIs, - size, - std::string("contains a value with size ") + PrintToString(size)) { - return arg && static_cast<int>(arg->size()) == size; +MATCHER_P(IsSingleArticle, url, "is a list with the single article %(url)s") { + NTPSnippetsFetcher::OptionalFetchedCategories& fetched_categories = *arg; + return fetched_categories && fetched_categories->size() == 1 && + fetched_categories->begin()->snippets.size() == 1 && + fetched_categories->begin()->snippets[0]->best_source().url.spec() == + url; } MATCHER_P(EqualsJSON, json, "equals JSON") { @@ -83,11 +98,14 @@ MATCHER_P(EqualsJSON, json, "equals JSON") { class MockSnippetsAvailableCallback { public: // Workaround for gMock's lack of support for movable arguments. - void WrappedRun(NTPSnippetsFetcher::OptionalSnippets snippets) { - Run(snippets); + void WrappedRun( + NTPSnippetsFetcher::OptionalFetchedCategories fetched_categories) { + Run(&fetched_categories); } - MOCK_METHOD1(Run, void(const NTPSnippetsFetcher::OptionalSnippets& snippets)); + MOCK_METHOD1( + Run, + void(NTPSnippetsFetcher::OptionalFetchedCategories* fetched_categories)); }; // Factory for FakeURLFetcher objects that always generate errors. @@ -96,9 +114,9 @@ class FailingFakeURLFetcherFactory : public net::URLFetcherFactory { std::unique_ptr<net::URLFetcher> CreateURLFetcher( int id, const GURL& url, net::URLFetcher::RequestType request_type, net::URLFetcherDelegate* d) override { - return base::WrapUnique(new net::FakeURLFetcher( + return base::MakeUnique<net::FakeURLFetcher>( url, d, /*response_data=*/std::string(), net::HTTP_NOT_FOUND, - net::URLRequestStatus::FAILED)); + net::URLRequestStatus::FAILED); } }; @@ -123,34 +141,17 @@ void ParseJsonDelayed( base::TimeDelta::FromMilliseconds(kTestJsonParsingLatencyMs)); } -class VariationParams { - public: - VariationParams(const std::string& trial_name, - const std::map<std::string, std::string>& params) - : field_trial_list_(new metrics::SHA1EntropyProvider("foo")) { - variations::AssociateVariationParams(trial_name, "Group1", params); - base::FieldTrialList::CreateFieldTrial(trial_name, "Group1"); - } - - ~VariationParams() { variations::testing::ClearAllVariationParams(); } - - private: - base::FieldTrialList field_trial_list_; -}; - } // namespace class NTPSnippetsFetcherTest : public testing::Test { public: NTPSnippetsFetcherTest() - : NTPSnippetsFetcherTest( - GURL(base::StringPrintf(kTestChromeReaderUrlFormat, - google_apis::GetAPIKey().c_str())), - std::map<std::string, std::string>()) {} + : NTPSnippetsFetcherTest(GURL(kTestChromeReaderUrl), + std::map<std::string, std::string>()) {} NTPSnippetsFetcherTest(const GURL& gurl, const std::map<std::string, std::string>& params) - : params_(ntp_snippets::kStudyName, params), + : params_manager_(ntp_snippets::kStudyName, params), mock_task_runner_(new base::TestMockTimeTaskRunner()), mock_task_runner_handle_(mock_task_runner_), signin_client_(new TestSigninClient(nullptr)), @@ -158,26 +159,29 @@ class NTPSnippetsFetcherTest : public testing::Test { fake_signin_manager_(new FakeSigninManagerBase(signin_client_.get(), account_tracker_.get())), fake_token_service_(new FakeProfileOAuth2TokenService()), - snippets_fetcher_( - fake_signin_manager_.get(), - fake_token_service_.get(), - scoped_refptr<net::TestURLRequestContextGetter>( - new net::TestURLRequestContextGetter(mock_task_runner_.get())), - base::Bind(&ParseJsonDelayed), - /*is_stable_channel=*/true), + pref_service_(new TestingPrefServiceSimple()), test_lang_("en-US"), test_url_(gurl) { - snippets_fetcher_.SetCallback( + RequestThrottler::RegisterProfilePrefs(pref_service_->registry()); + + snippets_fetcher_ = base::MakeUnique<NTPSnippetsFetcher>( + fake_signin_manager_.get(), fake_token_service_.get(), + scoped_refptr<net::TestURLRequestContextGetter>( + new net::TestURLRequestContextGetter(mock_task_runner_.get())), + pref_service_.get(), &category_factory_, base::Bind(&ParseJsonDelayed), + kAPIKey); + + snippets_fetcher_->SetCallback( base::Bind(&MockSnippetsAvailableCallback::WrappedRun, base::Unretained(&mock_callback_))); - snippets_fetcher_.SetTickClockForTesting( + snippets_fetcher_->SetTickClockForTesting( mock_task_runner_->GetMockTickClock()); - test_hosts_.insert("www.somehost.com"); + test_excluded_.insert("1234567890"); // Increase initial time such that ticks are non-zero. mock_task_runner_->FastForwardBy(base::TimeDelta::FromMilliseconds(1234)); } - NTPSnippetsFetcher& snippets_fetcher() { return snippets_fetcher_; } + NTPSnippetsFetcher& snippets_fetcher() { return *snippets_fetcher_; } MockSnippetsAvailableCallback& mock_callback() { return mock_callback_; } void FastForwardUntilNoTasksRemain() { mock_task_runner_->FastForwardUntilNoTasksRemain(); @@ -185,6 +189,7 @@ class NTPSnippetsFetcherTest : public testing::Test { const std::string& test_lang() const { return test_lang_; } const GURL& test_url() { return test_url_; } const std::set<std::string>& test_hosts() const { return test_hosts_; } + const std::set<std::string>& test_excluded() const { return test_excluded_; } base::HistogramTester& histogram_tester() { return histogram_tester_; } void InitFakeURLFetcherFactory() { @@ -205,7 +210,7 @@ class NTPSnippetsFetcherTest : public testing::Test { } private: - VariationParams params_; + variations::testing::VariationParamsManager params_manager_; scoped_refptr<base::TestMockTimeTaskRunner> mock_task_runner_; base::ThreadTaskRunnerHandle mock_task_runner_handle_; FailingFakeURLFetcherFactory failing_url_fetcher_factory_; @@ -215,11 +220,14 @@ class NTPSnippetsFetcherTest : public testing::Test { std::unique_ptr<AccountTrackerService> account_tracker_; std::unique_ptr<SigninManagerBase> fake_signin_manager_; std::unique_ptr<OAuth2TokenService> fake_token_service_; - NTPSnippetsFetcher snippets_fetcher_; + std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher_; + std::unique_ptr<TestingPrefServiceSimple> pref_service_; + CategoryFactory category_factory_; MockSnippetsAvailableCallback mock_callback_; const std::string test_lang_; const GURL test_url_; std::set<std::string> test_hosts_; + std::set<std::string> test_excluded_; base::HistogramTester histogram_tester_; DISALLOW_COPY_AND_ASSIGN(NTPSnippetsFetcherTest); @@ -229,10 +237,8 @@ class NTPSnippetsContentSuggestionsFetcherTest : public NTPSnippetsFetcherTest { public: NTPSnippetsContentSuggestionsFetcherTest() : NTPSnippetsFetcherTest( - GURL(base::StringPrintf(kTestChromeContentSuggestionsUrlFormat, - google_apis::GetAPIKey().c_str())), - std::map<std::string, std::string>{ - {"content_suggestions_backend", kContentSuggestionsBackend}}) {} + GURL(kTestChromeContentSuggestionsUrl), + {{"content_suggestions_backend", kContentSuggestionsServer}}) {} }; TEST_F(NTPSnippetsFetcherTest, BuildRequestAuthenticated) { @@ -241,7 +247,9 @@ TEST_F(NTPSnippetsFetcherTest, BuildRequestAuthenticated) { params.only_return_personalized_results = true; params.user_locale = "en"; params.host_restricts = {"chromium.org"}; + params.excluded_ids = {"1234567890"}; params.count_to_fetch = 25; + params.interactive_request = false; params.fetch_api = NTPSnippetsFetcher::CHROME_READER_API; EXPECT_THAT(params.BuildRequest(), @@ -286,8 +294,12 @@ TEST_F(NTPSnippetsFetcherTest, BuildRequestAuthenticated) { EXPECT_THAT(params.BuildRequest(), EqualsJSON("{" " \"uiLanguage\": \"en\"," - " \"regularlyVisitedHostName\": [" + " \"priority\": \"BACKGROUND_PREFETCH\"," + " \"regularlyVisitedHostNames\": [" " \"chromium.org\"" + " ]," + " \"excludedSuggestionIds\": [" + " \"1234567890\"" " ]" "}")); } @@ -297,6 +309,9 @@ TEST_F(NTPSnippetsFetcherTest, BuildRequestUnauthenticated) { params.only_return_personalized_results = false; params.host_restricts = {}; params.count_to_fetch = 10; + params.excluded_ids = {}; + params.interactive_request = true; + params.fetch_api = NTPSnippetsFetcher::CHROME_READER_API; EXPECT_THAT(params.BuildRequest(), @@ -333,7 +348,51 @@ TEST_F(NTPSnippetsFetcherTest, BuildRequestUnauthenticated) { params.fetch_api = NTPSnippetsFetcher::CHROME_CONTENT_SUGGESTIONS_API; EXPECT_THAT(params.BuildRequest(), EqualsJSON("{" - " \"regularlyVisitedHostName\": []" + " \"regularlyVisitedHostNames\": []," + " \"priority\": \"USER_ACTION\"," + " \"excludedSuggestionIds\": []" + "}")); +} + +TEST_F(NTPSnippetsFetcherTest, BuildRequestExcludedIds) { + NTPSnippetsFetcher::RequestParams params; + params.only_return_personalized_results = false; + params.host_restricts = {}; + params.count_to_fetch = 10; + params.interactive_request = false; + for (int i = 0; i < 200; ++i) { + params.excluded_ids.insert(base::StringPrintf("%03d", i)); + } + + params.fetch_api = NTPSnippetsFetcher::CHROME_CONTENT_SUGGESTIONS_API; + EXPECT_THAT(params.BuildRequest(), + EqualsJSON("{" + " \"regularlyVisitedHostNames\": []," + " \"priority\": \"BACKGROUND_PREFETCH\"," + " \"excludedSuggestionIds\": [" + " \"000\", \"001\", \"002\", \"003\", \"004\"," + " \"005\", \"006\", \"007\", \"008\", \"009\"," + " \"010\", \"011\", \"012\", \"013\", \"014\"," + " \"015\", \"016\", \"017\", \"018\", \"019\"," + " \"020\", \"021\", \"022\", \"023\", \"024\"," + " \"025\", \"026\", \"027\", \"028\", \"029\"," + " \"030\", \"031\", \"032\", \"033\", \"034\"," + " \"035\", \"036\", \"037\", \"038\", \"039\"," + " \"040\", \"041\", \"042\", \"043\", \"044\"," + " \"045\", \"046\", \"047\", \"048\", \"049\"," + " \"050\", \"051\", \"052\", \"053\", \"054\"," + " \"055\", \"056\", \"057\", \"058\", \"059\"," + " \"060\", \"061\", \"062\", \"063\", \"064\"," + " \"065\", \"066\", \"067\", \"068\", \"069\"," + " \"070\", \"071\", \"072\", \"073\", \"074\"," + " \"075\", \"076\", \"077\", \"078\", \"079\"," + " \"080\", \"081\", \"082\", \"083\", \"084\"," + " \"085\", \"086\", \"087\", \"088\", \"089\"," + " \"090\", \"091\", \"092\", \"093\", \"094\"," + " \"095\", \"096\", \"097\", \"098\", \"099\"" + // Truncated to 100 entries. Currently, they happen to + // be those lexically first. + " ]" "}")); } @@ -360,11 +419,13 @@ TEST_F(NTPSnippetsFetcherTest, ShouldFetchSuccessfully) { " }]" " }" "}]}"; - SetFakeResponse(/*data=*/kJsonStr, net::HTTP_OK, + SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(/*snippets=*/PointeeSizeIs(1))).Times(1); + EXPECT_CALL(mock_callback(), Run(IsSingleArticle("http://localhost/foobar"))); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); EXPECT_THAT(snippets_fetcher().last_status(), Eq("OK")); EXPECT_THAT(snippets_fetcher().last_json(), Eq(kJsonStr)); @@ -378,24 +439,128 @@ TEST_F(NTPSnippetsFetcherTest, ShouldFetchSuccessfully) { TEST_F(NTPSnippetsContentSuggestionsFetcherTest, ShouldFetchSuccessfully) { const std::string kJsonStr = - "{\"snippet\" : [{" - " \"id\" : [\"http://localhost/foobar\"]," - " \"title\" : \"Foo Barred from Baz\"," - " \"summaryText\" : \"...\"," - " \"fullPageUrl\" : \"http://localhost/foobar\"," - " \"publishTime\" : \"2016-06-30T11:01:37.000Z\"," - " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," - " \"publisherName\" : \"Foo News\"," - " \"imageUrl\" : \"http://localhost/foobar.jpg\"," - " \"ampUrl\" : \"http://localhost/amp\"," - " \"faviconUrl\" : \"http://localhost/favicon.ico\" " + "{\"categories\" : [{" + " \"id\": 1," + " \"localizedTitle\": \"Articles for You\"," + " \"suggestions\" : [{" + " \"ids\" : [\"http://localhost/foobar\"]," + " \"title\" : \"Foo Barred from Baz\"," + " \"snippet\" : \"...\"," + " \"fullPageUrl\" : \"http://localhost/foobar\"," + " \"creationTime\" : \"2016-06-30T11:01:37.000Z\"," + " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," + " \"attribution\" : \"Foo News\"," + " \"imageUrl\" : \"http://localhost/foobar.jpg\"," + " \"ampUrl\" : \"http://localhost/amp\"," + " \"faviconUrl\" : \"http://localhost/favicon.ico\" " + " }]" + "}]}"; + SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, + net::URLRequestStatus::SUCCESS); + EXPECT_CALL(mock_callback(), Run(IsSingleArticle("http://localhost/foobar"))); + snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); + FastForwardUntilNoTasksRemain(); + EXPECT_THAT(snippets_fetcher().last_status(), Eq("OK")); + EXPECT_THAT(snippets_fetcher().last_json(), Eq(kJsonStr)); + EXPECT_THAT(histogram_tester().GetAllSamples( + "NewTabPage.Snippets.FetchHttpResponseOrErrorCode"), + ElementsAre(base::Bucket(/*min=*/200, /*count=*/1))); + EXPECT_THAT(histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchTime"), + ElementsAre(base::Bucket(/*min=*/kTestJsonParsingLatencyMs, + /*count=*/1))); +} + +TEST_F(NTPSnippetsContentSuggestionsFetcherTest, EmptyCategoryIsOK) { + const std::string kJsonStr = + "{\"categories\" : [{" + " \"id\": 1," + " \"localizedTitle\": \"Articles for You\"" + "}]}"; + SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, + net::URLRequestStatus::SUCCESS); + EXPECT_CALL(mock_callback(), Run(IsEmptyArticleList())); + snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); + FastForwardUntilNoTasksRemain(); + EXPECT_THAT(snippets_fetcher().last_status(), Eq("OK")); + EXPECT_THAT(snippets_fetcher().last_json(), Eq(kJsonStr)); + EXPECT_THAT(histogram_tester().GetAllSamples( + "NewTabPage.Snippets.FetchHttpResponseOrErrorCode"), + ElementsAre(base::Bucket(/*min=*/200, /*count=*/1))); + EXPECT_THAT(histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchTime"), + ElementsAre(base::Bucket(/*min=*/kTestJsonParsingLatencyMs, + /*count=*/1))); +} + +TEST_F(NTPSnippetsContentSuggestionsFetcherTest, ServerCategories) { + const std::string kJsonStr = + "{\"categories\" : [{" + " \"id\": 1," + " \"localizedTitle\": \"Articles for You\"," + " \"suggestions\" : [{" + " \"ids\" : [\"http://localhost/foobar\"]," + " \"title\" : \"Foo Barred from Baz\"," + " \"snippet\" : \"...\"," + " \"fullPageUrl\" : \"http://localhost/foobar\"," + " \"creationTime\" : \"2016-06-30T11:01:37.000Z\"," + " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," + " \"attribution\" : \"Foo News\"," + " \"imageUrl\" : \"http://localhost/foobar.jpg\"," + " \"ampUrl\" : \"http://localhost/amp\"," + " \"faviconUrl\" : \"http://localhost/favicon.ico\" " + " }]" + "}, {" + " \"id\": 2," + " \"localizedTitle\": \"Articles for Me\"," + " \"suggestions\" : [{" + " \"ids\" : [\"http://localhost/foo2\"]," + " \"title\" : \"Foo Barred from Baz\"," + " \"snippet\" : \"...\"," + " \"fullPageUrl\" : \"http://localhost/foo2\"," + " \"creationTime\" : \"2016-06-30T11:01:37.000Z\"," + " \"expirationTime\" : \"2016-07-01T11:01:37.000Z\"," + " \"attribution\" : \"Foo News\"," + " \"imageUrl\" : \"http://localhost/foo2.jpg\"," + " \"ampUrl\" : \"http://localhost/amp\"," + " \"faviconUrl\" : \"http://localhost/favicon.ico\" " + " }]" "}]}"; - SetFakeResponse(/*data=*/kJsonStr, net::HTTP_OK, + SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(/*snippets=*/PointeeSizeIs(1))).Times(1); + NTPSnippetsFetcher::OptionalFetchedCategories fetched_categories; + EXPECT_CALL(mock_callback(), Run(_)) + .WillOnce(WithArg<0>(MovePointeeTo(&fetched_categories))); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); + + ASSERT_TRUE(fetched_categories); + ASSERT_THAT(fetched_categories->size(), Eq(2u)); + for (const auto& category : *fetched_categories) { + const auto& articles = category.snippets; + switch (category.category.id()) { + case static_cast<int>(KnownCategories::ARTICLES): + ASSERT_THAT(articles.size(), Eq(1u)); + EXPECT_THAT(articles[0]->best_source().url.spec(), + Eq("http://localhost/foobar")); + break; + case static_cast<int>(KnownCategories::ARTICLES) + 1: + ASSERT_THAT(articles.size(), Eq(1u)); + EXPECT_THAT(articles[0]->best_source().url.spec(), + Eq("http://localhost/foo2")); + break; + default: + FAIL() << "unknown category ID " << category.category.id(); + } + } + EXPECT_THAT(snippets_fetcher().last_status(), Eq("OK")); EXPECT_THAT(snippets_fetcher().last_json(), Eq(kJsonStr)); EXPECT_THAT(histogram_tester().GetAllSamples( @@ -408,11 +573,13 @@ TEST_F(NTPSnippetsContentSuggestionsFetcherTest, ShouldFetchSuccessfully) { TEST_F(NTPSnippetsFetcherTest, ShouldFetchSuccessfullyEmptyList) { const std::string kJsonStr = "{\"recos\": []}"; - SetFakeResponse(/*data=*/kJsonStr, net::HTTP_OK, + SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(/*snippets=*/PointeeSizeIs(0))).Times(1); + EXPECT_CALL(mock_callback(), Run(IsEmptyArticleList())); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); EXPECT_THAT(snippets_fetcher().last_status(), Eq("OK")); EXPECT_THAT(snippets_fetcher().last_json(), Eq(kJsonStr)); @@ -424,16 +591,45 @@ TEST_F(NTPSnippetsFetcherTest, ShouldFetchSuccessfullyEmptyList) { ElementsAre(base::Bucket(/*min=*/200, /*count=*/1))); } -// TODO(jkrcal) Return the tests ShouldReportEmptyHostsError and -// ShouldRestrictToHosts once we have a way to change variation parameters from -// unittests. The tests were tailored to previous default value of the parameter -// fetching_host_restrict, which changed now. +TEST_F(NTPSnippetsFetcherTest, ShouldRestrictToHosts) { + net::TestURLFetcherFactory test_url_fetcher_factory; + snippets_fetcher().FetchSnippetsFromHosts( + {"www.somehost1.com", "www.somehost2.com"}, test_lang(), test_excluded(), + /*count=*/17, + /*interactive_request=*/true); + net::TestURLFetcher* fetcher = test_url_fetcher_factory.GetFetcherByID(0); + ASSERT_THAT(fetcher, NotNull()); + std::unique_ptr<base::Value> value = + base::JSONReader::Read(fetcher->upload_data()); + ASSERT_TRUE(value) << " failed to parse JSON: " + << PrintToString(fetcher->upload_data()); + const base::DictionaryValue* dict = nullptr; + ASSERT_TRUE(value->GetAsDictionary(&dict)); + const base::DictionaryValue* local_scoring_params = nullptr; + ASSERT_TRUE(dict->GetDictionary("advanced_options.local_scoring_params", + &local_scoring_params)); + const base::ListValue* content_selectors = nullptr; + ASSERT_TRUE( + local_scoring_params->GetList("content_selectors", &content_selectors)); + ASSERT_THAT(content_selectors->GetSize(), Eq(static_cast<size_t>(2))); + const base::DictionaryValue* content_selector = nullptr; + ASSERT_TRUE(content_selectors->GetDictionary(0, &content_selector)); + std::string content_selector_value; + EXPECT_TRUE(content_selector->GetString("value", &content_selector_value)); + EXPECT_THAT(content_selector_value, Eq("www.somehost1.com")); + ASSERT_TRUE(content_selectors->GetDictionary(1, &content_selector)); + EXPECT_TRUE(content_selector->GetString("value", &content_selector_value)); + EXPECT_THAT(content_selector_value, Eq("www.somehost2.com")); +} + TEST_F(NTPSnippetsFetcherTest, ShouldReportUrlStatusError) { - SetFakeResponse(/*data=*/std::string(), net::HTTP_NOT_FOUND, + SetFakeResponse(/*response_data=*/std::string(), net::HTTP_NOT_FOUND, net::URLRequestStatus::FAILED); EXPECT_CALL(mock_callback(), Run(/*snippets=*/Not(HasValue()))).Times(1); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); EXPECT_THAT(snippets_fetcher().last_status(), Eq("URLRequestStatus error -2")); @@ -449,11 +645,13 @@ TEST_F(NTPSnippetsFetcherTest, ShouldReportUrlStatusError) { } TEST_F(NTPSnippetsFetcherTest, ShouldReportHttpError) { - SetFakeResponse(/*data=*/std::string(), net::HTTP_NOT_FOUND, + SetFakeResponse(/*response_data=*/std::string(), net::HTTP_NOT_FOUND, net::URLRequestStatus::SUCCESS); EXPECT_CALL(mock_callback(), Run(/*snippets=*/Not(HasValue()))).Times(1); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); EXPECT_THAT(snippets_fetcher().last_json(), IsEmpty()); EXPECT_THAT( @@ -468,11 +666,13 @@ TEST_F(NTPSnippetsFetcherTest, ShouldReportHttpError) { TEST_F(NTPSnippetsFetcherTest, ShouldReportJsonError) { const std::string kInvalidJsonStr = "{ \"recos\": []"; - SetFakeResponse(/*data=*/kInvalidJsonStr, net::HTTP_OK, + SetFakeResponse(/*response_data=*/kInvalidJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); EXPECT_CALL(mock_callback(), Run(/*snippets=*/Not(HasValue()))).Times(1); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); EXPECT_THAT(snippets_fetcher().last_status(), StartsWith("Received invalid JSON (error ")); @@ -489,11 +689,13 @@ TEST_F(NTPSnippetsFetcherTest, ShouldReportJsonError) { } TEST_F(NTPSnippetsFetcherTest, ShouldReportJsonErrorForEmptyResponse) { - SetFakeResponse(/*data=*/std::string(), net::HTTP_OK, + SetFakeResponse(/*response_data=*/std::string(), net::HTTP_OK, net::URLRequestStatus::SUCCESS); EXPECT_CALL(mock_callback(), Run(/*snippets=*/Not(HasValue()))).Times(1); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); EXPECT_THAT(snippets_fetcher().last_json(), std::string()); EXPECT_THAT( @@ -507,11 +709,13 @@ TEST_F(NTPSnippetsFetcherTest, ShouldReportJsonErrorForEmptyResponse) { TEST_F(NTPSnippetsFetcherTest, ShouldReportInvalidListError) { const std::string kJsonStr = "{\"recos\": [{ \"contentInfo\": { \"foo\" : \"bar\" }}]}"; - SetFakeResponse(/*data=*/kJsonStr, net::HTTP_OK, + SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); EXPECT_CALL(mock_callback(), Run(/*snippets=*/Not(HasValue()))).Times(1); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); EXPECT_THAT(snippets_fetcher().last_json(), Eq(kJsonStr)); EXPECT_THAT( @@ -530,21 +734,27 @@ TEST_F(NTPSnippetsFetcherTest, ShouldReportHttpErrorForMissingBakedResponse) { InitFakeURLFetcherFactory(); EXPECT_CALL(mock_callback(), Run(/*snippets=*/Not(HasValue()))).Times(1); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); } TEST_F(NTPSnippetsFetcherTest, ShouldCancelOngoingFetch) { const std::string kJsonStr = "{ \"recos\": [] }"; - SetFakeResponse(/*data=*/kJsonStr, net::HTTP_OK, + SetFakeResponse(/*response_data=*/kJsonStr, net::HTTP_OK, net::URLRequestStatus::SUCCESS); - EXPECT_CALL(mock_callback(), Run(/*snippets=*/PointeeSizeIs(0))).Times(1); + EXPECT_CALL(mock_callback(), Run(IsEmptyArticleList())); snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); // Second call to FetchSnippetsFromHosts() overrides/cancels the previous. // Callback is expected to be called once. snippets_fetcher().FetchSnippetsFromHosts(test_hosts(), test_lang(), - /*count=*/1); + test_excluded(), + /*count=*/1, + /*interactive_request=*/true); FastForwardUntilNoTasksRemain(); EXPECT_THAT( histogram_tester().GetAllSamples("NewTabPage.Snippets.FetchResult"), @@ -559,14 +769,13 @@ TEST_F(NTPSnippetsFetcherTest, ShouldCancelOngoingFetch) { ::std::ostream& operator<<( ::std::ostream& os, - const NTPSnippetsFetcher::OptionalSnippets& snippets) { - if (snippets) { + const NTPSnippetsFetcher::OptionalFetchedCategories& fetched_categories) { + if (fetched_categories) { // Matchers above aren't any more precise than this, so this is sufficient // for test-failure diagnostics. - return os << "list with " << snippets->size() << " elements"; - } else { - return os << "null"; + return os << "list with " << fetched_categories->size() << " elements"; } + return os << "null"; } } // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/ntp_snippets_scheduler.h b/chromium/components/ntp_snippets/remote/ntp_snippets_scheduler.h new file mode 100644 index 00000000000..7a4fc1ef5c9 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_scheduler.h @@ -0,0 +1,36 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_SCHEDULER_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_SCHEDULER_H_ + +#include "base/macros.h" +#include "base/time/time.h" + +namespace ntp_snippets { + +// Interface to schedule the periodic fetching of snippets. +class NTPSnippetsScheduler { + public: + // Schedule periodic fetching of snippets, with different periods depending on + // network state. The concrete implementation should call + // NTPSnippetsService::FetchSnippets once per period. + // Any of the periods can be zero to indicate that the corresponding task + // should not be scheduled. + virtual bool Schedule(base::TimeDelta period_wifi, + base::TimeDelta period_fallback) = 0; + + // Cancel any scheduled tasks. + virtual bool Unschedule() = 0; + + protected: + NTPSnippetsScheduler() = default; + + private: + DISALLOW_COPY_AND_ASSIGN(NTPSnippetsScheduler); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_SCHEDULER_H_ diff --git a/chromium/components/ntp_snippets/remote/ntp_snippets_service.cc b/chromium/components/ntp_snippets/remote/ntp_snippets_service.cc new file mode 100644 index 00000000000..165790bc607 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_service.cc @@ -0,0 +1,1121 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/ntp_snippets_service.h" + +#include <algorithm> +#include <iterator> +#include <utility> + +#include "base/command_line.h" +#include "base/location.h" +#include "base/memory/ptr_util.h" +#include "base/metrics/histogram_macros.h" +#include "base/metrics/sparse_histogram.h" +#include "base/path_service.h" +#include "base/stl_util.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/utf_string_conversions.h" +#include "base/task_runner_util.h" +#include "base/time/time.h" +#include "base/values.h" +#include "components/data_use_measurement/core/data_use_user_data.h" +#include "components/history/core/browser/history_service.h" +#include "components/image_fetcher/image_decoder.h" +#include "components/image_fetcher/image_fetcher.h" +#include "components/ntp_snippets/features.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/ntp_snippets/remote/ntp_snippets_database.h" +#include "components/ntp_snippets/switches.h" +#include "components/ntp_snippets/user_classifier.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" +#include "components/variations/variations_associated_data.h" +#include "grit/components_strings.h" +#include "ui/base/l10n/l10n_util.h" +#include "ui/gfx/image/image.h" + +namespace ntp_snippets { + +namespace { + +// Number of snippets requested to the server. Consider replacing sparse UMA +// histograms with COUNTS() if this number increases beyond 50. +const int kMaxSnippetCount = 10; + +// Number of archived snippets we keep around in memory. +const int kMaxArchivedSnippetCount = 200; + +// Default values for fetching intervals, fallback and wifi. +const double kDefaultFetchingIntervalRareNtpUser[] = {48.0, 24.0}; +const double kDefaultFetchingIntervalActiveNtpUser[] = {24.0, 6.0}; +const double kDefaultFetchingIntervalActiveSuggestionsConsumer[] = {24.0, 6.0}; + +// Variation parameters than can override the default fetching intervals. +const char* kFetchingIntervalParamNameRareNtpUser[] = { + "fetching_interval_hours-fallback-rare_ntp_user", + "fetching_interval_hours-wifi-rare_ntp_user"}; +const char* kFetchingIntervalParamNameActiveNtpUser[] = { + "fetching_interval_hours-fallback-active_ntp_user", + "fetching_interval_hours-wifi-active_ntp_user"}; +const char* kFetchingIntervalParamNameActiveSuggestionsConsumer[] = { + "fetching_interval_hours-fallback-active_suggestions_consumer", + "fetching_interval_hours-wifi-active_suggestions_consumer"}; + +const int kDefaultExpiryTimeMins = 3 * 24 * 60; + +// Keys for storing CategoryContent info in prefs. +const char kCategoryContentId[] = "id"; +const char kCategoryContentTitle[] = "title"; +const char kCategoryContentProvidedByServer[] = "provided_by_server"; + +base::TimeDelta GetFetchingInterval(bool is_wifi, + UserClassifier::UserClass user_class) { + double value_hours = 0.0; + + const int index = is_wifi ? 1 : 0; + const char* param_name = ""; + switch (user_class) { + case UserClassifier::UserClass::RARE_NTP_USER: + value_hours = kDefaultFetchingIntervalRareNtpUser[index]; + param_name = kFetchingIntervalParamNameRareNtpUser[index]; + break; + case UserClassifier::UserClass::ACTIVE_NTP_USER: + value_hours = kDefaultFetchingIntervalActiveNtpUser[index]; + param_name = kFetchingIntervalParamNameActiveNtpUser[index]; + break; + case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER: + value_hours = kDefaultFetchingIntervalActiveSuggestionsConsumer[index]; + param_name = kFetchingIntervalParamNameActiveSuggestionsConsumer[index]; + break; + } + + // The default value can be overridden by a variation parameter. + std::string param_value_str = variations::GetVariationParamValueByFeature( + ntp_snippets::kArticleSuggestionsFeature, param_name); + if (!param_value_str.empty()) { + double param_value_hours = 0.0; + if (base::StringToDouble(param_value_str, ¶m_value_hours)) + value_hours = param_value_hours; + else + LOG(WARNING) << "Invalid value for variation parameter " << param_name; + } + + return base::TimeDelta::FromSecondsD(value_hours * 3600.0); +} + +std::set<std::string> GetAllIDs(const NTPSnippet::PtrVector& snippets) { + std::set<std::string> ids; + for (const std::unique_ptr<NTPSnippet>& snippet : snippets) { + ids.insert(snippet->id()); + for (const SnippetSource& source : snippet->sources()) + ids.insert(source.url.spec()); + } + return ids; +} + +std::set<std::string> GetSnippetIDSet(const NTPSnippet::PtrVector& snippets) { + std::set<std::string> ids; + for (const std::unique_ptr<NTPSnippet>& snippet : snippets) + ids.insert(snippet->id()); + return ids; +} + +std::unique_ptr<std::vector<std::string>> GetSnippetIDVector( + const NTPSnippet::PtrVector& snippets) { + auto result = base::MakeUnique<std::vector<std::string>>(); + for (const auto& snippet : snippets) { + result->push_back(snippet->id()); + } + return result; +} + +bool IsSnippetInSet(const std::unique_ptr<NTPSnippet>& snippet, + const std::set<std::string>& ids, + bool match_all_ids) { + if (ids.count(snippet->id())) + return true; + if (!match_all_ids) + return false; + for (const SnippetSource& source : snippet->sources()) { + if (ids.count(source.url.spec())) + return true; + } + return false; +} + +void EraseMatchingSnippets(NTPSnippet::PtrVector* snippets, + const std::set<std::string>& matching_ids, + bool match_all_ids) { + snippets->erase( + std::remove_if(snippets->begin(), snippets->end(), + [&matching_ids, match_all_ids]( + const std::unique_ptr<NTPSnippet>& snippet) { + return IsSnippetInSet(snippet, matching_ids, + match_all_ids); + }), + snippets->end()); +} + +void Compact(NTPSnippet::PtrVector* snippets) { + snippets->erase( + std::remove_if( + snippets->begin(), snippets->end(), + [](const std::unique_ptr<NTPSnippet>& snippet) { return !snippet; }), + snippets->end()); +} + +} // namespace + +NTPSnippetsService::NTPSnippetsService( + Observer* observer, + CategoryFactory* category_factory, + PrefService* pref_service, + const std::string& application_language_code, + const UserClassifier* user_classifier, + NTPSnippetsScheduler* scheduler, + std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher, + std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, + std::unique_ptr<image_fetcher::ImageDecoder> image_decoder, + std::unique_ptr<NTPSnippetsDatabase> database, + std::unique_ptr<NTPSnippetsStatusService> status_service) + : ContentSuggestionsProvider(observer, category_factory), + state_(State::NOT_INITED), + pref_service_(pref_service), + articles_category_( + category_factory->FromKnownCategory(KnownCategories::ARTICLES)), + application_language_code_(application_language_code), + user_classifier_(user_classifier), + scheduler_(scheduler), + snippets_fetcher_(std::move(snippets_fetcher)), + image_fetcher_(std::move(image_fetcher)), + image_decoder_(std::move(image_decoder)), + database_(std::move(database)), + snippets_status_service_(std::move(status_service)), + fetch_when_ready_(false), + nuke_when_initialized_(false), + thumbnail_requests_throttler_( + pref_service, + RequestThrottler::RequestType::CONTENT_SUGGESTION_THUMBNAIL) { + RestoreCategoriesFromPrefs(); + // The articles category always exists. Add it if we didn't get it from prefs. + // TODO(treib): Rethink this. + if (!base::ContainsKey(categories_, articles_category_)) { + categories_[articles_category_] = CategoryContent(); + categories_[articles_category_].localized_title = + l10n_util::GetStringUTF16(IDS_NTP_ARTICLE_SUGGESTIONS_SECTION_HEADER); + } + // Tell the observer about all the categories. + for (const auto& entry : categories_) { + observer->OnCategoryStatusChanged(this, entry.first, entry.second.status); + } + + if (database_->IsErrorState()) { + EnterState(State::ERROR_OCCURRED); + UpdateAllCategoryStatus(CategoryStatus::LOADING_ERROR); + return; + } + + database_->SetErrorCallback(base::Bind(&NTPSnippetsService::OnDatabaseError, + base::Unretained(this))); + + // We transition to other states while finalizing the initialization, when the + // database is done loading. + database_->LoadSnippets(base::Bind(&NTPSnippetsService::OnDatabaseLoaded, + base::Unretained(this))); +} + +NTPSnippetsService::~NTPSnippetsService() = default; + +// static +void NTPSnippetsService::RegisterProfilePrefs(PrefRegistrySimple* registry) { + // TODO(treib): Add cleanup logic for prefs::kSnippetHosts, then remove it + // completely after M56. + registry->RegisterListPref(prefs::kSnippetHosts); + registry->RegisterListPref(prefs::kRemoteSuggestionCategories); + registry->RegisterInt64Pref(prefs::kSnippetBackgroundFetchingIntervalWifi, 0); + registry->RegisterInt64Pref(prefs::kSnippetBackgroundFetchingIntervalFallback, + 0); + + NTPSnippetsStatusService::RegisterProfilePrefs(registry); +} + +void NTPSnippetsService::FetchSnippets(bool interactive_request) { + if (ready()) + FetchSnippetsFromHosts(std::set<std::string>(), interactive_request); + else + fetch_when_ready_ = true; +} + +void NTPSnippetsService::FetchSnippetsFromHosts( + const std::set<std::string>& hosts, + bool interactive_request) { + if (!ready()) + return; + + // Empty categories are marked as loading; others are unchanged. + for (const auto& item : categories_) { + Category category = item.first; + const CategoryContent& content = item.second; + if (content.snippets.empty()) + UpdateCategoryStatus(category, CategoryStatus::AVAILABLE_LOADING); + } + + std::set<std::string> excluded_ids; + for (const auto& item : categories_) { + const CategoryContent& content = item.second; + for (const auto& snippet : content.dismissed) + excluded_ids.insert(snippet->id()); + } + snippets_fetcher_->FetchSnippetsFromHosts(hosts, application_language_code_, + excluded_ids, kMaxSnippetCount, + interactive_request); +} + +void NTPSnippetsService::RescheduleFetching(bool force) { + // The scheduler only exists on Android so far, it's null on other platforms. + if (!scheduler_) + return; + + if (ready()) { + base::TimeDelta old_interval_wifi = + base::TimeDelta::FromInternalValue(pref_service_->GetInt64( + prefs::kSnippetBackgroundFetchingIntervalWifi)); + base::TimeDelta old_interval_fallback = + base::TimeDelta::FromInternalValue(pref_service_->GetInt64( + prefs::kSnippetBackgroundFetchingIntervalFallback)); + UserClassifier::UserClass user_class = user_classifier_->GetUserClass(); + base::TimeDelta interval_wifi = + GetFetchingInterval(/*is_wifi=*/true, user_class); + base::TimeDelta interval_fallback = + GetFetchingInterval(/*is_wifi=*/false, user_class); + if (force || interval_wifi != old_interval_wifi || + interval_fallback != old_interval_fallback) { + scheduler_->Schedule(interval_wifi, interval_fallback); + pref_service_->SetInt64(prefs::kSnippetBackgroundFetchingIntervalWifi, + interval_wifi.ToInternalValue()); + pref_service_->SetInt64( + prefs::kSnippetBackgroundFetchingIntervalFallback, + interval_fallback.ToInternalValue()); + } + } else { + // If we're NOT_INITED, we don't know whether to schedule or unschedule. + // If |force| is false, all is well: We'll reschedule on the next state + // change anyway. If it's true, then unschedule here, to make sure that the + // next reschedule actually happens. + if (state_ != State::NOT_INITED || force) { + scheduler_->Unschedule(); + pref_service_->ClearPref(prefs::kSnippetBackgroundFetchingIntervalWifi); + pref_service_->ClearPref( + prefs::kSnippetBackgroundFetchingIntervalFallback); + } + } +} + +CategoryStatus NTPSnippetsService::GetCategoryStatus(Category category) { + DCHECK(base::ContainsKey(categories_, category)); + return categories_[category].status; +} + +CategoryInfo NTPSnippetsService::GetCategoryInfo(Category category) { + DCHECK(base::ContainsKey(categories_, category)); + const CategoryContent& content = categories_[category]; + bool is_article = category == articles_category_; + return CategoryInfo(content.localized_title, + ContentSuggestionsCardLayout::FULL_CARD, + /*has_more_button=*/false, + /*show_if_empty=*/is_article); +} + +void NTPSnippetsService::DismissSuggestion( + const ContentSuggestion::ID& suggestion_id) { + if (!ready()) + return; + + DCHECK(base::ContainsKey(categories_, suggestion_id.category())); + + CategoryContent* content = &categories_[suggestion_id.category()]; + auto it = std::find_if( + content->snippets.begin(), content->snippets.end(), + [&suggestion_id](const std::unique_ptr<NTPSnippet>& snippet) { + return snippet->id() == suggestion_id.id_within_category(); + }); + if (it == content->snippets.end()) + return; + + (*it)->set_dismissed(true); + + database_->SaveSnippet(**it); + database_->DeleteImage(suggestion_id.id_within_category()); + + content->dismissed.push_back(std::move(*it)); + content->snippets.erase(it); +} + +void NTPSnippetsService::FetchSuggestionImage( + const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) { + database_->LoadImage( + suggestion_id.id_within_category(), + base::Bind(&NTPSnippetsService::OnSnippetImageFetchedFromDatabase, + base::Unretained(this), callback, suggestion_id)); +} + +void NTPSnippetsService::ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) { + // Both time range and the filter are ignored and all suggestions are removed, + // because it is not known which history entries were used for the suggestions + // personalization. + if (!ready()) + nuke_when_initialized_ = true; + else + NukeAllSnippets(); +} + +void NTPSnippetsService::ClearCachedSuggestions(Category category) { + if (!initialized()) + return; + + if (!base::ContainsKey(categories_, category)) + return; + CategoryContent* content = &categories_[category]; + if (content->snippets.empty()) + return; + + database_->DeleteSnippets(GetSnippetIDVector(content->snippets)); + database_->DeleteImages(GetSnippetIDVector(content->snippets)); + content->snippets.clear(); + + NotifyNewSuggestions(); +} + +void NTPSnippetsService::GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) { + DCHECK(base::ContainsKey(categories_, category)); + + std::vector<ContentSuggestion> result; + const CategoryContent& content = categories_[category]; + for (const std::unique_ptr<NTPSnippet>& snippet : content.dismissed) { + if (!snippet->is_complete()) + continue; + ContentSuggestion suggestion(category, snippet->id(), + snippet->best_source().url); + suggestion.set_amp_url(snippet->best_source().amp_url); + suggestion.set_title(base::UTF8ToUTF16(snippet->title())); + suggestion.set_snippet_text(base::UTF8ToUTF16(snippet->snippet())); + suggestion.set_publish_date(snippet->publish_date()); + suggestion.set_publisher_name( + base::UTF8ToUTF16(snippet->best_source().publisher_name)); + suggestion.set_score(snippet->score()); + result.emplace_back(std::move(suggestion)); + } + callback.Run(std::move(result)); +} + +void NTPSnippetsService::ClearDismissedSuggestionsForDebugging( + Category category) { + DCHECK(base::ContainsKey(categories_, category)); + + if (!initialized()) + return; + + CategoryContent* content = &categories_[category]; + if (content->dismissed.empty()) + return; + + database_->DeleteSnippets(GetSnippetIDVector(content->dismissed)); + // The image got already deleted when the suggestion was dismissed. + + content->dismissed.clear(); +} + +// static +int NTPSnippetsService::GetMaxSnippetCountForTesting() { + return kMaxSnippetCount; +} + +//////////////////////////////////////////////////////////////////////////////// +// Private methods + +GURL NTPSnippetsService::FindSnippetImageUrl( + const ContentSuggestion::ID& suggestion_id) const { + DCHECK(base::ContainsKey(categories_, suggestion_id.category())); + + const CategoryContent& content = categories_.at(suggestion_id.category()); + const NTPSnippet* snippet = + content.FindSnippet(suggestion_id.id_within_category()); + if (!snippet) + return GURL(); + return snippet->salient_image_url(); +} + +// image_fetcher::ImageFetcherDelegate implementation. +void NTPSnippetsService::OnImageDataFetched( + const std::string& id_within_category, + const std::string& image_data) { + if (image_data.empty()) + return; + + // Only save the image if the corresponding snippet still exists. + bool found = false; + for (const std::pair<const Category, CategoryContent>& entry : categories_) { + if (entry.second.FindSnippet(id_within_category)) { + found = true; + break; + } + } + if (!found) + return; + + // Only cache the data in the DB, the actual serving is done in the callback + // provided to |image_fetcher_| (OnSnippetImageDecodedFromNetwork()). + database_->SaveImage(id_within_category, image_data); +} + +void NTPSnippetsService::OnDatabaseLoaded(NTPSnippet::PtrVector snippets) { + if (state_ == State::ERROR_OCCURRED) + return; + DCHECK(state_ == State::NOT_INITED); + DCHECK(base::ContainsKey(categories_, articles_category_)); + + NTPSnippet::PtrVector to_delete; + for (std::unique_ptr<NTPSnippet>& snippet : snippets) { + Category snippet_category = + category_factory()->FromRemoteCategory(snippet->remote_category_id()); + // We should already know about the category. + if (!base::ContainsKey(categories_, snippet_category)) { + DLOG(WARNING) << "Loaded a suggestion for unknown category " + << snippet_category << " from the DB; deleting"; + to_delete.emplace_back(std::move(snippet)); + continue; + } + + CategoryContent* content = &categories_[snippet_category]; + if (snippet->is_dismissed()) + content->dismissed.emplace_back(std::move(snippet)); + else + content->snippets.emplace_back(std::move(snippet)); + } + if (!to_delete.empty()) { + database_->DeleteSnippets(GetSnippetIDVector(to_delete)); + database_->DeleteImages(GetSnippetIDVector(to_delete)); + } + + // Sort the suggestions in each category. + // TODO(treib): Persist the actual order in the DB somehow? crbug.com/654409 + for (auto& entry : categories_) { + CategoryContent* content = &entry.second; + std::sort(content->snippets.begin(), content->snippets.end(), + [](const std::unique_ptr<NTPSnippet>& lhs, + const std::unique_ptr<NTPSnippet>& rhs) { + return lhs->score() > rhs->score(); + }); + } + + // TODO(tschumann): If I move ClearExpiredDismisedSnippets() to the beginning + // of the function, it essentially does nothing but tests are still green. Fix + // this! + ClearExpiredDismissedSnippets(); + ClearOrphanedImages(); + FinishInitialization(); +} + +void NTPSnippetsService::OnDatabaseError() { + EnterState(State::ERROR_OCCURRED); + UpdateAllCategoryStatus(CategoryStatus::LOADING_ERROR); +} + +void NTPSnippetsService::OnFetchFinished( + NTPSnippetsFetcher::OptionalFetchedCategories fetched_categories) { + if (!ready()) + return; + + // Mark all categories as not provided by the server in the latest fetch. The + // ones we got will be marked again below. + for (auto& item : categories_) { + CategoryContent* content = &item.second; + content->provided_by_server = false; + } + + // Clear up expired dismissed snippets before we use them to filter new ones. + ClearExpiredDismissedSnippets(); + + // If snippets were fetched successfully, update our |categories_| from each + // category provided by the server. + if (fetched_categories) { + // TODO(treib): Reorder |categories_| to match the order we received from + // the server. crbug.com/653816 + // TODO(jkrcal): A bit hard to understand with so many variables called + // "*categor*". Isn't here some room for simplification? + for (NTPSnippetsFetcher::FetchedCategory& fetched_category : + *fetched_categories) { + Category category = fetched_category.category; + + // The ChromeReader backend doesn't provide category titles, so don't + // overwrite the existing title for ARTICLES if the new one is empty. + // TODO(treib): Remove this check after we fully switch to the content + // suggestions backend. + if (category != articles_category_ || + !fetched_category.localized_title.empty()) { + categories_[category].localized_title = + fetched_category.localized_title; + } + categories_[category].provided_by_server = true; + + // TODO(tschumann): Remove this histogram once we only talk to the content + // suggestions cloud backend. + if (category == articles_category_) { + UMA_HISTOGRAM_SPARSE_SLOWLY( + "NewTabPage.Snippets.NumArticlesFetched", + std::min(fetched_category.snippets.size(), + static_cast<size_t>(kMaxSnippetCount + 1))); + } + ReplaceSnippets(category, std::move(fetched_category.snippets)); + } + } + + // We might have gotten new categories (or updated the titles of existing + // ones), so update the pref. + StoreCategoriesToPrefs(); + + for (const auto& item : categories_) { + Category category = item.first; + UpdateCategoryStatus(category, CategoryStatus::AVAILABLE); + } + + // TODO(sfiera): equivalent metrics for non-articles. + const CategoryContent& content = categories_[articles_category_]; + UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticles", + content.snippets.size()); + if (content.snippets.empty() && !content.dismissed.empty()) { + UMA_HISTOGRAM_COUNTS("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded", + content.dismissed.size()); + } + + // TODO(sfiera): notify only when a category changed above. + NotifyNewSuggestions(); + + // Reschedule after a successful fetch. This resets all currently scheduled + // fetches, to make sure the fallback interval triggers only if no wifi fetch + // succeeded, and also that we don't do a background fetch immediately after + // a user-initiated one. + if (fetched_categories) + RescheduleFetching(true); +} + +void NTPSnippetsService::ArchiveSnippets(Category category, + NTPSnippet::PtrVector* to_archive) { + CategoryContent* content = &categories_[category]; + + database_->DeleteSnippets(GetSnippetIDVector(*to_archive)); + // Do not delete the thumbnail images as they are still handy on open NTPs. + + // Archive previous snippets - move them at the beginning of the list. + content->archived.insert(content->archived.begin(), + std::make_move_iterator(to_archive->begin()), + std::make_move_iterator(to_archive->end())); + Compact(to_archive); + + // If there are more archived snippets than we want to keep, delete the + // oldest ones by their fetch time (which are always in the back). + if (content->archived.size() > kMaxArchivedSnippetCount) { + NTPSnippet::PtrVector to_delete( + std::make_move_iterator(content->archived.begin() + + kMaxArchivedSnippetCount), + std::make_move_iterator(content->archived.end())); + content->archived.resize(kMaxArchivedSnippetCount); + database_->DeleteImages(GetSnippetIDVector(to_delete)); + } +} + +void NTPSnippetsService::ReplaceSnippets(Category category, + NTPSnippet::PtrVector new_snippets) { + DCHECK(ready()); + CategoryContent* content = &categories_[category]; + + // Remove new snippets that have been dismissed. + EraseMatchingSnippets(&new_snippets, GetAllIDs(content->dismissed), + /*match_all_ids=*/true); + + // Fill in default publish/expiry dates where required. + for (std::unique_ptr<NTPSnippet>& snippet : new_snippets) { + if (snippet->publish_date().is_null()) + snippet->set_publish_date(base::Time::Now()); + if (snippet->expiry_date().is_null()) { + snippet->set_expiry_date( + snippet->publish_date() + + base::TimeDelta::FromMinutes(kDefaultExpiryTimeMins)); + } + } + + if (!base::CommandLine::ForCurrentProcess()->HasSwitch( + switches::kAddIncompleteSnippets)) { + int num_new_snippets = new_snippets.size(); + // Remove snippets that do not have all the info we need to display it to + // the user. + new_snippets.erase( + std::remove_if(new_snippets.begin(), new_snippets.end(), + [](const std::unique_ptr<NTPSnippet>& snippet) { + return !snippet->is_complete(); + }), + new_snippets.end()); + int num_snippets_dismissed = num_new_snippets - new_snippets.size(); + UMA_HISTOGRAM_BOOLEAN("NewTabPage.Snippets.IncompleteSnippetsAfterFetch", + num_snippets_dismissed > 0); + if (num_snippets_dismissed > 0) { + UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumIncompleteSnippets", + num_snippets_dismissed); + } + } + + // Do not touch the current set of snippets if the newly fetched one is empty. + if (new_snippets.empty()) + return; + + // Remove current snippets that have been fetched again. We do not need to + // archive those as they will be in the new current set. + EraseMatchingSnippets(&content->snippets, GetSnippetIDSet(new_snippets), + /*match_all_ids=*/false); + + ArchiveSnippets(category, &content->snippets); + + // Save new articles to the DB. + database_->SaveSnippets(new_snippets); + + content->snippets = std::move(new_snippets); +} + +void NTPSnippetsService::ClearExpiredDismissedSnippets() { + std::vector<Category> categories_to_erase; + + const base::Time now = base::Time::Now(); + + for (auto& item : categories_) { + Category category = item.first; + CategoryContent* content = &item.second; + + NTPSnippet::PtrVector to_delete; + // Move expired dismissed snippets over into |to_delete|. + for (std::unique_ptr<NTPSnippet>& snippet : content->dismissed) { + if (snippet->expiry_date() <= now) + to_delete.emplace_back(std::move(snippet)); + } + Compact(&content->dismissed); + + // Delete the removed article suggestions from the DB. + database_->DeleteSnippets(GetSnippetIDVector(to_delete)); + // The image got already deleted when the suggestion was dismissed. + + if (content->snippets.empty() && content->dismissed.empty() && + category != articles_category_ && !content->provided_by_server) { + categories_to_erase.push_back(category); + } + } + + for (Category category : categories_to_erase) { + UpdateCategoryStatus(category, CategoryStatus::NOT_PROVIDED); + categories_.erase(category); + } + + StoreCategoriesToPrefs(); +} + +void NTPSnippetsService::ClearOrphanedImages() { + auto alive_snippets = base::MakeUnique<std::set<std::string>>(); + for (const auto& entry : categories_) { + const CategoryContent& content = entry.second; + for (const auto& snippet_ptr : content.snippets) { + alive_snippets->insert(snippet_ptr->id()); + } + for (const auto& snippet_ptr : content.dismissed) { + alive_snippets->insert(snippet_ptr->id()); + } + } + database_->GarbageCollectImages(std::move(alive_snippets)); +} + +void NTPSnippetsService::NukeAllSnippets() { + std::vector<Category> categories_to_erase; + + // Empty the ARTICLES category and remove all others, since they may or may + // not be personalized. + for (const auto& item : categories_) { + Category category = item.first; + + ClearCachedSuggestions(category); + ClearDismissedSuggestionsForDebugging(category); + + UpdateCategoryStatus(category, CategoryStatus::NOT_PROVIDED); + + // Remove the category entirely; it may or may not reappear. + if (category != articles_category_) + categories_to_erase.push_back(category); + } + + for (Category category : categories_to_erase) { + categories_.erase(category); + } + + StoreCategoriesToPrefs(); +} + +void NTPSnippetsService::OnSnippetImageFetchedFromDatabase( + const ImageFetchedCallback& callback, + const ContentSuggestion::ID& suggestion_id, + std::string data) { + // |image_decoder_| is null in tests. + if (image_decoder_ && !data.empty()) { + image_decoder_->DecodeImage( + data, base::Bind(&NTPSnippetsService::OnSnippetImageDecodedFromDatabase, + base::Unretained(this), callback, suggestion_id)); + return; + } + + // Fetching from the DB failed; start a network fetch. + FetchSnippetImageFromNetwork(suggestion_id, callback); +} + +void NTPSnippetsService::OnSnippetImageDecodedFromDatabase( + const ImageFetchedCallback& callback, + const ContentSuggestion::ID& suggestion_id, + const gfx::Image& image) { + if (!image.IsEmpty()) { + callback.Run(image); + return; + } + + // If decoding the image failed, delete the DB entry. + database_->DeleteImage(suggestion_id.id_within_category()); + + FetchSnippetImageFromNetwork(suggestion_id, callback); +} + +void NTPSnippetsService::FetchSnippetImageFromNetwork( + const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) { + if (!base::ContainsKey(categories_, suggestion_id.category())) { + OnSnippetImageDecodedFromNetwork( + callback, suggestion_id.id_within_category(), gfx::Image()); + return; + } + + GURL image_url = FindSnippetImageUrl(suggestion_id); + + if (image_url.is_empty() || + !thumbnail_requests_throttler_.DemandQuotaForRequest( + /*interactive_request=*/true)) { + // Return an empty image. Directly, this is never synchronous with the + // original FetchSuggestionImage() call - an asynchronous database query has + // happened in the meantime. + OnSnippetImageDecodedFromNetwork( + callback, suggestion_id.id_within_category(), gfx::Image()); + return; + } + + image_fetcher_->StartOrQueueNetworkRequest( + suggestion_id.id_within_category(), image_url, + base::Bind(&NTPSnippetsService::OnSnippetImageDecodedFromNetwork, + base::Unretained(this), callback)); +} + +void NTPSnippetsService::OnSnippetImageDecodedFromNetwork( + const ImageFetchedCallback& callback, + const std::string& id_within_category, + const gfx::Image& image) { + callback.Run(image); +} + +void NTPSnippetsService::EnterStateReady() { + if (nuke_when_initialized_) { + NukeAllSnippets(); + nuke_when_initialized_ = false; + } + + if (categories_[articles_category_].snippets.empty() || fetch_when_ready_) { + // TODO(jkrcal): Fetching snippets automatically upon creation of this + // lazily created service can cause troubles, e.g. in unit tests where + // network I/O is not allowed. + // Either add a DCHECK here that we actually are allowed to do network I/O + // or change the logic so that some explicit call is always needed for the + // network request. + FetchSnippets(/*interactive_request=*/false); + fetch_when_ready_ = false; + } + + for (const auto& item : categories_) { + Category category = item.first; + const CategoryContent& content = item.second; + // FetchSnippets has set the status to |AVAILABLE_LOADING| if relevant, + // otherwise we transition to |AVAILABLE| here. + if (content.status != CategoryStatus::AVAILABLE_LOADING) + UpdateCategoryStatus(category, CategoryStatus::AVAILABLE); + } +} + +void NTPSnippetsService::EnterStateDisabled() { + NukeAllSnippets(); +} + +void NTPSnippetsService::EnterStateError() { + snippets_status_service_.reset(); +} + +void NTPSnippetsService::FinishInitialization() { + if (nuke_when_initialized_) { + // We nuke here in addition to EnterStateReady, so that it happens even if + // we enter the DISABLED state below. + NukeAllSnippets(); + nuke_when_initialized_ = false; + } + + snippets_fetcher_->SetCallback( + base::Bind(&NTPSnippetsService::OnFetchFinished, base::Unretained(this))); + + // |image_fetcher_| can be null in tests. + if (image_fetcher_) { + image_fetcher_->SetImageFetcherDelegate(this); + image_fetcher_->SetDataUseServiceName( + data_use_measurement::DataUseUserData::NTP_SNIPPETS); + } + + // Note: Initializing the status service will run the callback right away with + // the current state. + snippets_status_service_->Init(base::Bind( + &NTPSnippetsService::OnSnippetsStatusChanged, base::Unretained(this))); + + // Always notify here even if we got nothing from the database, because we + // don't know how long the fetch will take or if it will even complete. + NotifyNewSuggestions(); +} + +void NTPSnippetsService::OnSnippetsStatusChanged( + SnippetsStatus old_snippets_status, + SnippetsStatus new_snippets_status) { + switch (new_snippets_status) { + case SnippetsStatus::ENABLED_AND_SIGNED_IN: + if (old_snippets_status == SnippetsStatus::ENABLED_AND_SIGNED_OUT) { + DCHECK(state_ == State::READY); + // Clear nonpersonalized suggestions. + NukeAllSnippets(); + // Fetch personalized ones. + FetchSnippets(/*interactive_request=*/true); + } else { + // Do not change the status. That will be done in EnterStateReady(). + EnterState(State::READY); + } + break; + + case SnippetsStatus::ENABLED_AND_SIGNED_OUT: + if (old_snippets_status == SnippetsStatus::ENABLED_AND_SIGNED_IN) { + DCHECK(state_ == State::READY); + // Clear personalized suggestions. + NukeAllSnippets(); + // Fetch nonpersonalized ones. + FetchSnippets(/*interactive_request=*/true); + } else { + // Do not change the status. That will be done in EnterStateReady(). + EnterState(State::READY); + } + break; + + case SnippetsStatus::EXPLICITLY_DISABLED: + EnterState(State::DISABLED); + UpdateAllCategoryStatus(CategoryStatus::CATEGORY_EXPLICITLY_DISABLED); + break; + + case SnippetsStatus::SIGNED_OUT_AND_DISABLED: + EnterState(State::DISABLED); + UpdateAllCategoryStatus(CategoryStatus::SIGNED_OUT); + break; + } +} + +void NTPSnippetsService::EnterState(State state) { + if (state == state_) + return; + + switch (state) { + case State::NOT_INITED: + // Initial state, it should not be possible to get back there. + NOTREACHED(); + break; + + case State::READY: + DCHECK(state_ == State::NOT_INITED || state_ == State::DISABLED); + + DVLOG(1) << "Entering state: READY"; + state_ = State::READY; + EnterStateReady(); + break; + + case State::DISABLED: + DCHECK(state_ == State::NOT_INITED || state_ == State::READY); + + DVLOG(1) << "Entering state: DISABLED"; + state_ = State::DISABLED; + EnterStateDisabled(); + break; + + case State::ERROR_OCCURRED: + DVLOG(1) << "Entering state: ERROR_OCCURRED"; + state_ = State::ERROR_OCCURRED; + EnterStateError(); + break; + } + + // Schedule or un-schedule background fetching after each state change. + RescheduleFetching(false); +} + +void NTPSnippetsService::NotifyNewSuggestions() { + for (const auto& item : categories_) { + Category category = item.first; + const CategoryContent& content = item.second; + + std::vector<ContentSuggestion> result; + for (const std::unique_ptr<NTPSnippet>& snippet : content.snippets) { + // TODO(sfiera): if a snippet is not going to be displayed, move it + // directly to content.dismissed on fetch. Otherwise, we might prune + // other snippets to get down to kMaxSnippetCount, only to hide one of the + // incomplete ones we kept. + if (!snippet->is_complete()) + continue; + ContentSuggestion suggestion(category, snippet->id(), + snippet->best_source().url); + suggestion.set_amp_url(snippet->best_source().amp_url); + suggestion.set_title(base::UTF8ToUTF16(snippet->title())); + suggestion.set_snippet_text(base::UTF8ToUTF16(snippet->snippet())); + suggestion.set_publish_date(snippet->publish_date()); + suggestion.set_publisher_name( + base::UTF8ToUTF16(snippet->best_source().publisher_name)); + suggestion.set_score(snippet->score()); + result.emplace_back(std::move(suggestion)); + } + + DVLOG(1) << "NotifyNewSuggestions(): " << result.size() + << " items in category " << category; + observer()->OnNewSuggestions(this, category, std::move(result)); + } +} + +void NTPSnippetsService::UpdateCategoryStatus(Category category, + CategoryStatus status) { + DCHECK(base::ContainsKey(categories_, category)); + CategoryContent& content = categories_[category]; + if (status == content.status) + return; + + DVLOG(1) << "UpdateCategoryStatus(): " << category.id() << ": " + << static_cast<int>(content.status) << " -> " + << static_cast<int>(status); + content.status = status; + observer()->OnCategoryStatusChanged(this, category, content.status); +} + +void NTPSnippetsService::UpdateAllCategoryStatus(CategoryStatus status) { + for (const auto& category : categories_) { + UpdateCategoryStatus(category.first, status); + } +} + +const NTPSnippet* NTPSnippetsService::CategoryContent::FindSnippet( + const std::string& id_within_category) const { + // Search for the snippet in current and archived snippets. + auto it = std::find_if( + snippets.begin(), snippets.end(), + [&id_within_category](const std::unique_ptr<NTPSnippet>& snippet) { + return snippet->id() == id_within_category; + }); + if (it != snippets.end()) + return it->get(); + + it = std::find_if( + archived.begin(), archived.end(), + [&id_within_category](const std::unique_ptr<NTPSnippet>& snippet) { + return snippet->id() == id_within_category; + }); + if (it != archived.end()) + return it->get(); + + return nullptr; +} + +void NTPSnippetsService::RestoreCategoriesFromPrefs() { + // This must only be called at startup, before there are any categories. + DCHECK(categories_.empty()); + + const base::ListValue* list = + pref_service_->GetList(prefs::kRemoteSuggestionCategories); + for (const std::unique_ptr<base::Value>& entry : *list) { + const base::DictionaryValue* dict = nullptr; + if (!entry->GetAsDictionary(&dict)) { + DLOG(WARNING) << "Invalid category pref value: " << *entry; + continue; + } + int id = 0; + if (!dict->GetInteger(kCategoryContentId, &id)) { + DLOG(WARNING) << "Invalid category pref value, missing '" + << kCategoryContentId << "': " << *entry; + continue; + } + base::string16 title; + if (!dict->GetString(kCategoryContentTitle, &title)) { + DLOG(WARNING) << "Invalid category pref value, missing '" + << kCategoryContentTitle << "': " << *entry; + continue; + } + bool provided_by_server = false; + if (!dict->GetBoolean(kCategoryContentProvidedByServer, + &provided_by_server)) { + DLOG(WARNING) << "Invalid category pref value, missing '" + << kCategoryContentProvidedByServer << "': " << *entry; + continue; + } + + Category category = category_factory()->FromIDValue(id); + categories_[category] = CategoryContent(); + categories_[category].localized_title = title; + categories_[category].provided_by_server = provided_by_server; + } +} + +void NTPSnippetsService::StoreCategoriesToPrefs() { + // Collect all the CategoryContents. + std::vector<std::pair<Category, const CategoryContent*>> to_store; + for (const auto& entry : categories_) + to_store.emplace_back(entry.first, &entry.second); + // Sort them into the proper category order. + std::sort(to_store.begin(), to_store.end(), + [this](const std::pair<Category, const CategoryContent*>& left, + const std::pair<Category, const CategoryContent*>& right) { + return category_factory()->CompareCategories(left.first, + right.first); + }); + // Convert the relevant info into a base::ListValue for storage. + base::ListValue list; + for (const auto& entry : to_store) { + Category category = entry.first; + const base::string16& title = entry.second->localized_title; + bool provided_by_server = entry.second->provided_by_server; + auto dict = base::MakeUnique<base::DictionaryValue>(); + dict->SetInteger(kCategoryContentId, category.id()); + dict->SetString(kCategoryContentTitle, title); + dict->SetBoolean(kCategoryContentProvidedByServer, provided_by_server); + list.Append(std::move(dict)); + } + // Finally, store the result in the pref service. + pref_service_->Set(prefs::kRemoteSuggestionCategories, list); +} + +NTPSnippetsService::CategoryContent::CategoryContent() = default; +NTPSnippetsService::CategoryContent::CategoryContent(CategoryContent&&) = + default; +NTPSnippetsService::CategoryContent::~CategoryContent() = default; +NTPSnippetsService::CategoryContent& NTPSnippetsService::CategoryContent:: +operator=(CategoryContent&&) = default; + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/ntp_snippets_service.h b/chromium/components/ntp_snippets/remote/ntp_snippets_service.h new file mode 100644 index 00000000000..af9c017bafb --- /dev/null +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_service.h @@ -0,0 +1,366 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_SERVICE_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_SERVICE_H_ + +#include <cstddef> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "base/callback_forward.h" +#include "base/gtest_prod_util.h" +#include "base/macros.h" +#include "components/image_fetcher/image_fetcher_delegate.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/category_status.h" +#include "components/ntp_snippets/content_suggestion.h" +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "components/ntp_snippets/remote/ntp_snippet.h" +#include "components/ntp_snippets/remote/ntp_snippets_fetcher.h" +#include "components/ntp_snippets/remote/ntp_snippets_scheduler.h" +#include "components/ntp_snippets/remote/ntp_snippets_status_service.h" +#include "components/ntp_snippets/remote/request_throttler.h" + +class PrefRegistrySimple; +class PrefService; + +namespace gfx { +class Image; +} // namespace gfx + +namespace image_fetcher { +class ImageDecoder; +class ImageFetcher; +} // namespace image_fetcher + +namespace ntp_snippets { + +class NTPSnippetsDatabase; +class UserClassifier; + +// Retrieves fresh content data (articles) from the server, stores them and +// provides them as content suggestions. +// This class is final because it does things in its constructor which make it +// unsafe to derive from it. +// TODO(treib): Introduce two-phase initialization and make the class not final? +// TODO(pke): Rename this service to ArticleSuggestionsProvider and move to +// a subdirectory. +// TODO(jkrcal): this class grows really, really large. The fact that +// NTPSnippetService also implements ImageFetcherDelegate adds unnecessary +// complexity (and after all the Service is conceptually not an +// ImagerFetcherDeletage ;-)). Instead, the cleaner solution would be to define +// a CachedImageFetcher class that handles the caching aspects and looks like an +// image fetcher to the NTPSnippetService. +class NTPSnippetsService final : public ContentSuggestionsProvider, + public image_fetcher::ImageFetcherDelegate { + public: + // |application_language_code| should be a ISO 639-1 compliant string, e.g. + // 'en' or 'en-US'. Note that this code should only specify the language, not + // the locale, so 'en_US' (English language with US locale) and 'en-GB_US' + // (British English person in the US) are not language codes. + NTPSnippetsService(Observer* observer, + CategoryFactory* category_factory, + PrefService* pref_service, + const std::string& application_language_code, + const UserClassifier* user_classifier, + NTPSnippetsScheduler* scheduler, + std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher, + std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, + std::unique_ptr<image_fetcher::ImageDecoder> image_decoder, + std::unique_ptr<NTPSnippetsDatabase> database, + std::unique_ptr<NTPSnippetsStatusService> status_service); + + ~NTPSnippetsService() override; + + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + // Returns whether the service is ready. While this is false, the list of + // snippets will be empty, and all modifications to it (fetch, dismiss, etc) + // will be ignored. + bool ready() const { return state_ == State::READY; } + + // Returns whether the service is initialized. While this is false, some + // calls may trigger DCHECKs. + bool initialized() const { return ready() || state_ == State::DISABLED; } + + // Fetches snippets from the server and replaces old snippets by the new ones. + // Requests can be marked more important by setting |interactive_request| to + // true (such request might circumvent the daily quota for requests, etc.) + // Useful for requests triggered by the user. + void FetchSnippets(bool interactive_request); + + // Fetches snippets from the server for specified hosts and adds them to the + // current ones. Only called from chrome://snippets-internals, DO NOT USE + // otherwise! Ignored while ready() is false. + void FetchSnippetsFromHosts(const std::set<std::string>& hosts, + bool interactive_request); + + const NTPSnippetsFetcher* snippets_fetcher() const { + return snippets_fetcher_.get(); + } + + // (Re)schedules the periodic fetching of snippets. If |force| is true, the + // tasks will be re-scheduled even if they already exist and have the correct + // periods. + void RescheduleFetching(bool force); + + // ContentSuggestionsProvider implementation + CategoryStatus GetCategoryStatus(Category category) override; + CategoryInfo GetCategoryInfo(Category category) override; + void DismissSuggestion(const ContentSuggestion::ID& suggestion_id) override; + void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) override; + void ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) override; + void ClearCachedSuggestions(Category category) override; + void GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) override; + void ClearDismissedSuggestionsForDebugging(Category category) override; + + // Returns the maximum number of snippets that will be shown at once. + static int GetMaxSnippetCountForTesting(); + + // Available snippets, only for unit tests. + // TODO(treib): Get rid of this. Tests should use a fake observer instead. + const NTPSnippet::PtrVector& GetSnippetsForTesting(Category category) const { + return categories_.find(category)->second.snippets; + } + + // Available snippets, only for unit tests. + const NTPSnippet::PtrVector& GetArchivedSnippetsForTesting( + Category category) const { + return categories_.find(category)->second.archived; + } + + // Dismissed snippets, only for unit tests. + const NTPSnippet::PtrVector& GetDismissedSnippetsForTesting( + Category category) const { + return categories_.find(category)->second.dismissed; + } + + private: + friend class NTPSnippetsServiceTest; + FRIEND_TEST_ALL_PREFIXES(NTPSnippetsServiceTest, + RemoveExpiredDismissedContent); + FRIEND_TEST_ALL_PREFIXES(NTPSnippetsServiceTest, RescheduleOnStateChange); + FRIEND_TEST_ALL_PREFIXES(NTPSnippetsServiceTest, StatusChanges); + FRIEND_TEST_ALL_PREFIXES(NTPSnippetsServiceTest, + SuggestionsFetchedOnSignInAndSignOut); + + // Possible state transitions: + // NOT_INITED --------+ + // / \ | + // v v | + // READY <--> DISABLED | + // \ / | + // v v | + // ERROR_OCCURRED <-----+ + enum class State { + // The service has just been created. Can change to states: + // - DISABLED: After the database is done loading, + // GetStateForDependenciesStatus can identify the next state to + // be DISABLED. + // - READY: if GetStateForDependenciesStatus returns it, after the database + // is done loading. + // - ERROR_OCCURRED: when an unrecoverable error occurred. + NOT_INITED, + + // The service registered observers, timers, etc. and is ready to answer to + // queries, fetch snippets... Can change to states: + // - DISABLED: when the global Chrome state changes, for example after + // |OnStateChanged| is called and sync is disabled. + // - ERROR_OCCURRED: when an unrecoverable error occurred. + READY, + + // The service is disabled and unregistered the related resources. + // Can change to states: + // - READY: when the global Chrome state changes, for example after + // |OnStateChanged| is called and sync is enabled. + // - ERROR_OCCURRED: when an unrecoverable error occurred. + DISABLED, + + // The service or one of its dependencies encountered an unrecoverable error + // and the service can't be used anymore. + ERROR_OCCURRED + }; + + // Returns the URL of the image of a snippet if it is among the current or + // among the archived snippets in the matching category. Returns an empty URL + // otherwise. + GURL FindSnippetImageUrl(const ContentSuggestion::ID& suggestion_id) const; + + // image_fetcher::ImageFetcherDelegate implementation. + void OnImageDataFetched(const std::string& id_within_category, + const std::string& image_data) override; + + // Callbacks for the NTPSnippetsDatabase. + void OnDatabaseLoaded(NTPSnippet::PtrVector snippets); + void OnDatabaseError(); + + // Callback for the NTPSnippetsFetcher. + void OnFetchFinished( + NTPSnippetsFetcher::OptionalFetchedCategories fetched_categories); + + // Moves all snippets from |to_archive| into the archive of the |category|. + // It also deletes the snippets from the DB and keeps the archive reasonably + // short. + void ArchiveSnippets(Category category, NTPSnippet::PtrVector* to_archive); + + // Replace old snippets in |category| by newly available snippets. + void ReplaceSnippets(Category category, NTPSnippet::PtrVector new_snippets); + + // Removes expired dismissed snippets from the service and the database. + void ClearExpiredDismissedSnippets(); + + // Removes images from the DB that are not referenced from any known snippet. + // Needs to iterate the whole snippet database -- so do it often enough to + // keep it small but not too often as it still iterates over the file system. + void ClearOrphanedImages(); + + // Clears all stored snippets and updates the observer. + void NukeAllSnippets(); + + // Completes the initialization phase of the service, registering the last + // observers. This is done after construction, once the database is loaded. + void FinishInitialization(); + + void OnSnippetImageFetchedFromDatabase( + const ImageFetchedCallback& callback, + const ContentSuggestion::ID& suggestion_id, + std::string data); + + void OnSnippetImageDecodedFromDatabase( + const ImageFetchedCallback& callback, + const ContentSuggestion::ID& suggestion_id, + const gfx::Image& image); + + void FetchSnippetImageFromNetwork(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback); + + void OnSnippetImageDecodedFromNetwork(const ImageFetchedCallback& callback, + const std::string& id_within_category, + const gfx::Image& image); + + // Triggers a state transition depending on the provided snippets status. This + // method is called when a change is detected by |snippets_status_service_|. + void OnSnippetsStatusChanged(SnippetsStatus old_snippets_status, + SnippetsStatus new_snippets_status); + + // Verifies state transitions (see |State|'s documentation) and applies them. + // Also updates the provider status. Does nothing except updating the provider + // status if called with the current state. + void EnterState(State state); + + // Enables the service. Do not call directly, use |EnterState| instead. + void EnterStateReady(); + + // Disables the service. Do not call directly, use |EnterState| instead. + void EnterStateDisabled(); + + // Disables the service permanently because an unrecoverable error occurred. + // Do not call directly, use |EnterState| instead. + void EnterStateError(); + + // Converts the cached snippets to article content suggestions and notifies + // the observers. + void NotifyNewSuggestions(); + + // Updates the internal status for |category| to |category_status_| and + // notifies the content suggestions observer if it changed. + void UpdateCategoryStatus(Category category, CategoryStatus status); + // Calls UpdateCategoryStatus() for all provided categories. + void UpdateAllCategoryStatus(CategoryStatus status); + + void RestoreCategoriesFromPrefs(); + void StoreCategoriesToPrefs(); + + State state_; + + PrefService* pref_service_; + + const Category articles_category_; + + // TODO(sfiera): Reduce duplication of CategoryContent with CategoryInfo. + struct CategoryContent { + CategoryStatus status = CategoryStatus::INITIALIZING; + + // The title of the section, localized to the running UI language. + base::string16 localized_title; + + // True iff the server returned results in this category in the last fetch. + // We never remove categories that the server still provides, but if the + // server stops providing a category, we won't yet report it as NOT_PROVIDED + // while we still have non-expired snippets in it. + bool provided_by_server = true; + + // All currently active suggestions (excl. the dismissed ones). + NTPSnippet::PtrVector snippets; + + // All previous suggestions that we keep around in memory because they can + // be on some open NTP. We do not persist this list so that on a new start + // of Chrome, this is empty. + NTPSnippet::PtrVector archived; + + // Suggestions that the user dismissed. We keep these around until they + // expire so we won't re-add them to |snippets| on the next fetch. + NTPSnippet::PtrVector dismissed; + + // Returns a non-dismissed snippet with the given |id_within_category|, or + // null if none exist. + const NTPSnippet* FindSnippet(const std::string& id_within_category) const; + + CategoryContent(); + CategoryContent(CategoryContent&&); + ~CategoryContent(); + CategoryContent& operator=(CategoryContent&&); + }; + std::map<Category, CategoryContent, Category::CompareByID> categories_; + + // The ISO 639-1 code of the language used by the application. + const std::string application_language_code_; + + // Classifier that tells us how active the user is. Not owned. + const UserClassifier* user_classifier_; + + // Scheduler for fetching snippets. Not owned. + NTPSnippetsScheduler* scheduler_; + + // The snippets fetcher. + std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher_; + + std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher_; + std::unique_ptr<image_fetcher::ImageDecoder> image_decoder_; + + // The database for persisting snippets. + std::unique_ptr<NTPSnippetsDatabase> database_; + + // The service that provides events and data about the signin and sync state. + std::unique_ptr<NTPSnippetsStatusService> snippets_status_service_; + + // Set to true if FetchSnippets is called while the service isn't ready. + // The fetch will be executed once the service enters the READY state. + bool fetch_when_ready_; + + // Set to true if NukeAllSnippets is called while the service isn't ready. + // The nuke will be executed once the service finishes initialization or + // enters the READY state. + bool nuke_when_initialized_; + + // Request throttler for limiting requests to thumbnail images. + RequestThrottler thumbnail_requests_throttler_; + + DISALLOW_COPY_AND_ASSIGN(NTPSnippetsService); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_SERVICE_H_ diff --git a/chromium/components/ntp_snippets/remote/ntp_snippets_service_unittest.cc b/chromium/components/ntp_snippets/remote/ntp_snippets_service_unittest.cc new file mode 100644 index 00000000000..5b6fb4fc45d --- /dev/null +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_service_unittest.cc @@ -0,0 +1,1305 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/ntp_snippets_service.h" + +#include <memory> +#include <utility> +#include <vector> + +#include "base/command_line.h" +#include "base/files/file_path.h" +#include "base/files/scoped_temp_dir.h" +#include "base/json/json_reader.h" +#include "base/macros.h" +#include "base/memory/ptr_util.h" +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_util.h" +#include "base/strings/stringprintf.h" +#include "base/test/histogram_tester.h" +#include "base/threading/thread_task_runner_handle.h" +#include "base/time/time.h" +#include "components/image_fetcher/image_decoder.h" +#include "components/image_fetcher/image_fetcher.h" +#include "components/image_fetcher/image_fetcher_delegate.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/ntp_snippets_constants.h" +#include "components/ntp_snippets/remote/ntp_snippet.h" +#include "components/ntp_snippets/remote/ntp_snippets_database.h" +#include "components/ntp_snippets/remote/ntp_snippets_fetcher.h" +#include "components/ntp_snippets/remote/ntp_snippets_scheduler.h" +#include "components/ntp_snippets/remote/ntp_snippets_test_utils.h" +#include "components/ntp_snippets/user_classifier.h" +#include "components/prefs/testing_pref_service.h" +#include "components/signin/core/browser/fake_profile_oauth2_token_service.h" +#include "components/signin/core/browser/fake_signin_manager.h" +#include "components/variations/variations_associated_data.h" +#include "net/url_request/test_url_fetcher_factory.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "ui/gfx/image/image.h" +#include "ui/gfx/image/image_unittest_util.h" + +using image_fetcher::ImageFetcher; +using image_fetcher::ImageFetcherDelegate; +using testing::ElementsAre; +using testing::Eq; +using testing::InSequence; +using testing::Invoke; +using testing::IsEmpty; +using testing::Mock; +using testing::MockFunction; +using testing::NiceMock; +using testing::SaveArg; +using testing::SizeIs; +using testing::StartsWith; +using testing::WithArgs; +using testing::_; + +namespace ntp_snippets { + +namespace { + +MATCHER_P(IdEq, value, "") { + return arg->id() == value; +} + +MATCHER_P(IsCategory, id, "") { + return arg.id() == static_cast<int>(id); +} + +const base::Time::Exploded kDefaultCreationTime = {2015, 11, 4, 25, 13, 46, 45}; +const char kTestContentSuggestionsServerEndpoint[] = + "https://localunittest-chromecontentsuggestions-pa.googleapis.com/v1/" + "suggestions/fetch"; +const char kAPIKey[] = "fakeAPIkey"; +const char kTestContentSuggestionsServerWithAPIKey[] = + "https://localunittest-chromecontentsuggestions-pa.googleapis.com/v1/" + "suggestions/fetch?key=fakeAPIkey"; + +const char kSnippetUrl[] = "http://localhost/foobar"; +const char kSnippetTitle[] = "Title"; +const char kSnippetText[] = "Snippet"; +const char kSnippetSalientImage[] = "http://localhost/salient_image"; +const char kSnippetPublisherName[] = "Foo News"; +const char kSnippetAmpUrl[] = "http://localhost/amp"; + +const char kSnippetUrl2[] = "http://foo.com/bar"; + +const char kTestJsonDefaultCategoryTitle[] = "Some title"; + +const int kUnknownRemoteCategoryId = 1234; + +base::Time GetDefaultCreationTime() { + base::Time out_time; + EXPECT_TRUE(base::Time::FromUTCExploded(kDefaultCreationTime, &out_time)); + return out_time; +} + +base::Time GetDefaultExpirationTime() { + return base::Time::Now() + base::TimeDelta::FromHours(1); +} + +std::string GetTestJson(const std::vector<std::string>& snippets, + const std::string& category_title) { + return base::StringPrintf( + "{\n" + " \"categories\": [{\n" + " \"id\": 1,\n" + " \"localizedTitle\": \"%s\",\n" + " \"suggestions\": [%s]\n" + " }]\n" + "}\n", + category_title.c_str(), + base::JoinString(snippets, ", ").c_str()); +} + +std::string GetTestJson(const std::vector<std::string>& snippets) { + return GetTestJson(snippets, kTestJsonDefaultCategoryTitle); +} + +std::string GetTestJsonWithoutTitle(const std::vector<std::string>& snippets) { + return GetTestJson(snippets, std::string()); +} + +std::string GetMultiCategoryJson(const std::vector<std::string>& articles, + const std::vector<std::string>& others, + int other_id = 2) { + return base::StringPrintf( + "{\n" + " \"categories\": [{\n" + " \"id\": 1,\n" + " \"localizedTitle\": \"Articles for You\",\n" + " \"suggestions\": [%s]\n" + " }, {\n" + " \"id\": %i,\n" + " \"localizedTitle\": \"Other Things\",\n" + " \"suggestions\": [%s]\n" + " }]\n" + "}\n", + base::JoinString(articles, ", ").c_str(), other_id, + base::JoinString(others, ", ").c_str()); +} + +std::string FormatTime(const base::Time& t) { + base::Time::Exploded x; + t.UTCExplode(&x); + return base::StringPrintf("%04d-%02d-%02dT%02d:%02d:%02dZ", x.year, x.month, + x.day_of_month, x.hour, x.minute, x.second); +} + +std::string GetSnippetWithUrlAndTimesAndSource( + const std::vector<std::string>& ids, + const std::string& url, + const base::Time& creation_time, + const base::Time& expiry_time, + const std::string& publisher, + const std::string& amp_url) { + const std::string ids_string = base::JoinString(ids, "\",\n \""); + return base::StringPrintf( + "{\n" + " \"ids\": [\n" + " \"%s\"\n" + " ],\n" + " \"title\": \"%s\",\n" + " \"snippet\": \"%s\",\n" + " \"fullPageUrl\": \"%s\",\n" + " \"creationTime\": \"%s\",\n" + " \"expirationTime\": \"%s\",\n" + " \"attribution\": \"%s\",\n" + " \"imageUrl\": \"%s\",\n" + " \"ampUrl\": \"%s\"\n" + " }", + ids_string.c_str(), kSnippetTitle, kSnippetText, url.c_str(), + FormatTime(creation_time).c_str(), FormatTime(expiry_time).c_str(), + publisher.c_str(), kSnippetSalientImage, amp_url.c_str()); +} + +std::string GetSnippetWithSources(const std::string& source_url, + const std::string& publisher, + const std::string& amp_url) { + return GetSnippetWithUrlAndTimesAndSource( + {kSnippetUrl}, source_url, GetDefaultCreationTime(), + GetDefaultExpirationTime(), publisher, amp_url); +} + +std::string GetSnippetWithUrlAndTimes(const std::string& url, + const base::Time& content_creation_time, + const base::Time& expiry_time) { + return GetSnippetWithUrlAndTimesAndSource({url}, url, content_creation_time, + expiry_time, kSnippetPublisherName, + kSnippetAmpUrl); +} + +std::string GetSnippetWithTimes(const base::Time& content_creation_time, + const base::Time& expiry_time) { + return GetSnippetWithUrlAndTimes(kSnippetUrl, content_creation_time, + expiry_time); +} + +std::string GetSnippetWithUrl(const std::string& url) { + return GetSnippetWithUrlAndTimes(url, GetDefaultCreationTime(), + GetDefaultExpirationTime()); +} + +std::string GetSnippet() { + return GetSnippetWithUrlAndTimes(kSnippetUrl, GetDefaultCreationTime(), + GetDefaultExpirationTime()); +} + +std::string GetSnippetN(int n) { + return GetSnippetWithUrlAndTimes(base::StringPrintf("%s/%d", kSnippetUrl, n), + GetDefaultCreationTime(), + GetDefaultExpirationTime()); +} + +std::string GetExpiredSnippet() { + return GetSnippetWithTimes(GetDefaultCreationTime(), base::Time::Now()); +} + +std::string GetInvalidSnippet() { + std::string json_str = GetSnippet(); + // Make the json invalid by removing the final closing brace. + return json_str.substr(0, json_str.size() - 1); +} + +std::string GetIncompleteSnippet() { + std::string json_str = GetSnippet(); + // Rename the "url" entry. The result is syntactically valid json that will + // fail to parse as snippets. + size_t pos = json_str.find("\"fullPageUrl\""); + if (pos == std::string::npos) { + NOTREACHED(); + return std::string(); + } + json_str[pos + 1] = 'x'; + return json_str; +} + +using ServeImageCallback = base::Callback<void( + const std::string&, + base::Callback<void(const std::string&, const gfx::Image&)>)>; + +void ServeOneByOneImage( + image_fetcher::ImageFetcherDelegate* notify, + const std::string& id, + base::Callback<void(const std::string&, const gfx::Image&)> callback) { + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::Bind(callback, id, gfx::test::CreateImage(1, 1))); + notify->OnImageDataFetched(id, "1-by-1-image-data"); +} + +void ParseJson( + const std::string& json, + const ntp_snippets::NTPSnippetsFetcher::SuccessCallback& success_callback, + const ntp_snippets::NTPSnippetsFetcher::ErrorCallback& error_callback) { + base::JSONReader json_reader; + std::unique_ptr<base::Value> value = json_reader.ReadToValue(json); + if (value) { + success_callback.Run(std::move(value)); + } else { + error_callback.Run(json_reader.GetErrorMessage()); + } +} + +// Factory for FakeURLFetcher objects that always generate errors. +class FailingFakeURLFetcherFactory : public net::URLFetcherFactory { + public: + std::unique_ptr<net::URLFetcher> CreateURLFetcher( + int id, const GURL& url, net::URLFetcher::RequestType request_type, + net::URLFetcherDelegate* d) override { + return base::MakeUnique<net::FakeURLFetcher>( + url, d, /*response_data=*/std::string(), net::HTTP_NOT_FOUND, + net::URLRequestStatus::FAILED); + } +}; + +class MockScheduler : public NTPSnippetsScheduler { + public: + MOCK_METHOD2(Schedule, + bool(base::TimeDelta period_wifi, + base::TimeDelta period_fallback)); + MOCK_METHOD0(Unschedule, bool()); +}; + +class MockImageFetcher : public ImageFetcher { + public: + MOCK_METHOD1(SetImageFetcherDelegate, void(ImageFetcherDelegate*)); + MOCK_METHOD1(SetDataUseServiceName, void(DataUseServiceName)); + MOCK_METHOD3( + StartOrQueueNetworkRequest, + void(const std::string&, + const GURL&, + base::Callback<void(const std::string&, const gfx::Image&)>)); +}; + +class FakeContentSuggestionsProviderObserver + : public ContentSuggestionsProvider::Observer { + public: + FakeContentSuggestionsProviderObserver() + : loaded_(base::WaitableEvent::ResetPolicy::MANUAL, + base::WaitableEvent::InitialState::NOT_SIGNALED) {} + + void OnNewSuggestions(ContentSuggestionsProvider* provider, + Category category, + std::vector<ContentSuggestion> suggestions) override { + suggestions_[category] = std::move(suggestions); + } + + void OnCategoryStatusChanged(ContentSuggestionsProvider* provider, + Category category, + CategoryStatus new_status) override { + if (category.IsKnownCategory(KnownCategories::ARTICLES) && + IsCategoryStatusAvailable(new_status)) { + loaded_.Signal(); + } + statuses_[category] = new_status; + } + + void OnSuggestionInvalidated( + ContentSuggestionsProvider* provider, + const ContentSuggestion::ID& suggestion_id) override {} + + const std::map<Category, CategoryStatus, Category::CompareByID>& statuses() + const { + return statuses_; + } + + CategoryStatus StatusForCategory(Category category) const { + auto it = statuses_.find(category); + if (it == statuses_.end()) { + return CategoryStatus::NOT_PROVIDED; + } + return it->second; + } + + const std::vector<ContentSuggestion>& SuggestionsForCategory( + Category category) { + return suggestions_[category]; + } + + void WaitForLoad() { loaded_.Wait(); } + bool Loaded() { return loaded_.IsSignaled(); } + + void Reset() { + loaded_.Reset(); + statuses_.clear(); + } + + private: + base::WaitableEvent loaded_; + std::map<Category, CategoryStatus, Category::CompareByID> statuses_; + std::map<Category, std::vector<ContentSuggestion>, Category::CompareByID> + suggestions_; + + DISALLOW_COPY_AND_ASSIGN(FakeContentSuggestionsProviderObserver); +}; + +class FakeImageDecoder : public image_fetcher::ImageDecoder { + public: + FakeImageDecoder() {} + ~FakeImageDecoder() override = default; + void DecodeImage( + const std::string& image_data, + const image_fetcher::ImageDecodedCallback& callback) override { + callback.Run(decoded_image_); + } + + void SetDecodedImage(const gfx::Image& image) { decoded_image_ = image; } + + private: + gfx::Image decoded_image_; +}; + +} // namespace + +class NTPSnippetsServiceTest : public ::testing::Test { + public: + NTPSnippetsServiceTest() + : params_manager_(ntp_snippets::kStudyName, + {{"content_suggestions_backend", + kTestContentSuggestionsServerEndpoint}}), + fake_url_fetcher_factory_( + /*default_factory=*/&failing_url_fetcher_factory_), + test_url_(kTestContentSuggestionsServerWithAPIKey), + user_classifier_(/*pref_service=*/nullptr), + image_fetcher_(nullptr), + image_decoder_(nullptr) { + NTPSnippetsService::RegisterProfilePrefs(utils_.pref_service()->registry()); + RequestThrottler::RegisterProfilePrefs(utils_.pref_service()->registry()); + + EXPECT_TRUE(database_dir_.CreateUniqueTempDir()); + } + + ~NTPSnippetsServiceTest() override { + // We need to run the message loop after deleting the database, because + // ProtoDatabaseImpl deletes the actual LevelDB asynchronously on the task + // runner. Without this, we'd get reports of memory leaks. + base::RunLoop().RunUntilIdle(); + } + + std::unique_ptr<NTPSnippetsService> MakeSnippetsService( + bool set_empty_response = true) { + auto service = MakeSnippetsServiceWithoutInitialization(); + WaitForSnippetsServiceInitialization(set_empty_response); + return service; + } + + std::unique_ptr<NTPSnippetsService> + MakeSnippetsServiceWithoutInitialization() { + scoped_refptr<base::SingleThreadTaskRunner> task_runner( + base::ThreadTaskRunnerHandle::Get()); + scoped_refptr<net::TestURLRequestContextGetter> request_context_getter = + new net::TestURLRequestContextGetter(task_runner.get()); + + utils_.ResetSigninManager(); + std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher = + base::MakeUnique<NTPSnippetsFetcher>( + utils_.fake_signin_manager(), fake_token_service_.get(), + std::move(request_context_getter), utils_.pref_service(), + &category_factory_, base::Bind(&ParseJson), kAPIKey); + + utils_.fake_signin_manager()->SignIn("foo@bar.com"); + snippets_fetcher->SetPersonalizationForTesting( + NTPSnippetsFetcher::Personalization::kNonPersonal); + + auto image_fetcher = base::MakeUnique<NiceMock<MockImageFetcher>>(); + + image_fetcher_ = image_fetcher.get(); + EXPECT_CALL(*image_fetcher, SetImageFetcherDelegate(_)); + auto image_decoder = base::MakeUnique<FakeImageDecoder>(); + image_decoder_ = image_decoder.get(); + EXPECT_FALSE(observer_); + observer_ = base::MakeUnique<FakeContentSuggestionsProviderObserver>(); + return base::MakeUnique<NTPSnippetsService>( + observer_.get(), &category_factory_, utils_.pref_service(), "fr", + &user_classifier_, &scheduler_, std::move(snippets_fetcher), + std::move(image_fetcher), std::move(image_decoder), + base::MakeUnique<NTPSnippetsDatabase>(database_dir_.GetPath(), + task_runner), + base::MakeUnique<NTPSnippetsStatusService>(utils_.fake_signin_manager(), + utils_.pref_service())); + } + + void WaitForSnippetsServiceInitialization(bool set_empty_response = true) { + EXPECT_TRUE(observer_); + EXPECT_FALSE(observer_->Loaded()); + + // Add an initial fetch response, as the service tries to fetch when there + // is nothing in the DB. + if (set_empty_response) + SetUpFetchResponse(GetTestJsonWithoutTitle(std::vector<std::string>())); + + base::RunLoop().RunUntilIdle(); + observer_->WaitForLoad(); + } + + void ResetSnippetsService(std::unique_ptr<NTPSnippetsService>* service) { + service->reset(); + observer_.reset(); + *service = MakeSnippetsService(); + } + + ContentSuggestion::ID MakeArticleID(const std::string& id_within_category) { + return ContentSuggestion::ID(articles_category(), id_within_category); + } + + Category articles_category() { + return category_factory_.FromKnownCategory(KnownCategories::ARTICLES); + } + + ContentSuggestion::ID MakeOtherID(const std::string& id_within_category) { + return ContentSuggestion::ID(other_category(), id_within_category); + } + + Category other_category() { return category_factory_.FromRemoteCategory(2); } + + Category unknown_category() { + return category_factory_.FromRemoteCategory(kUnknownRemoteCategoryId); + } + + protected: + const GURL& test_url() { return test_url_; } + FakeContentSuggestionsProviderObserver& observer() { return *observer_; } + MockScheduler& mock_scheduler() { return scheduler_; } + NiceMock<MockImageFetcher>* image_fetcher() { return image_fetcher_; } + FakeImageDecoder* image_decoder() { return image_decoder_; } + + // Provide the json to be returned by the fake fetcher. + void SetUpFetchResponse(const std::string& json) { + fake_url_fetcher_factory_.SetFakeResponse(test_url_, json, net::HTTP_OK, + net::URLRequestStatus::SUCCESS); + } + + void LoadFromJSONString(NTPSnippetsService* service, + const std::string& json) { + SetUpFetchResponse(json); + service->FetchSnippets(true); + base::RunLoop().RunUntilIdle(); + } + + private: + variations::testing::VariationParamsManager params_manager_; + test::NTPSnippetsTestUtils utils_; + base::MessageLoop message_loop_; + FailingFakeURLFetcherFactory failing_url_fetcher_factory_; + // Instantiation of factory automatically sets itself as URLFetcher's factory. + net::FakeURLFetcherFactory fake_url_fetcher_factory_; + const GURL test_url_; + std::unique_ptr<OAuth2TokenService> fake_token_service_; + UserClassifier user_classifier_; + NiceMock<MockScheduler> scheduler_; + std::unique_ptr<FakeContentSuggestionsProviderObserver> observer_; + CategoryFactory category_factory_; + NiceMock<MockImageFetcher>* image_fetcher_; + FakeImageDecoder* image_decoder_; + + base::ScopedTempDir database_dir_; + + DISALLOW_COPY_AND_ASSIGN(NTPSnippetsServiceTest); +}; + +TEST_F(NTPSnippetsServiceTest, ScheduleOnStart) { + // We should get two |Schedule| calls: The first when initialization + // completes, the second one after the automatic (since the service doesn't + // have any data yet) fetch finishes. + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(2); + EXPECT_CALL(mock_scheduler(), Unschedule()).Times(0); + auto service = MakeSnippetsService(); + + // When we have no snippets are all, loading the service initiates a fetch. + EXPECT_EQ("OK", service->snippets_fetcher()->last_status()); +} + +TEST_F(NTPSnippetsServiceTest, DontRescheduleOnStart) { + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(2); + EXPECT_CALL(mock_scheduler(), Unschedule()).Times(0); + SetUpFetchResponse(GetTestJson({GetSnippet()})); + auto service = MakeSnippetsService(/*set_empty_response=*/false); + + // When recreating the service, we should not get any |Schedule| calls: + // The tasks are already scheduled with the correct intervals, so nothing on + // initialization, and the service has data from the DB, so no automatic fetch + // should happen. + Mock::VerifyAndClearExpectations(&mock_scheduler()); + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(0); + EXPECT_CALL(mock_scheduler(), Unschedule()).Times(0); + ResetSnippetsService(&service); +} + +TEST_F(NTPSnippetsServiceTest, RescheduleAfterSuccessfulFetch) { + // We should get two |Schedule| calls: The first when initialization + // completes, the second one after the automatic (since the service doesn't + // have any data yet) fetch finishes. + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(2); + auto service = MakeSnippetsService(); + + // A successful fetch should trigger another |Schedule|. + EXPECT_CALL(mock_scheduler(), Schedule(_, _)); + LoadFromJSONString(service.get(), GetTestJson({GetSnippet()})); +} + +TEST_F(NTPSnippetsServiceTest, DontRescheduleAfterFailedFetch) { + // We should get two |Schedule| calls: The first when initialization + // completes, the second one after the automatic (since the service doesn't + // have any data yet) fetch finishes. + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(2); + auto service = MakeSnippetsService(); + + // A failed fetch should NOT trigger another |Schedule|. + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(0); + LoadFromJSONString(service.get(), GetTestJson({GetInvalidSnippet()})); +} + +TEST_F(NTPSnippetsServiceTest, IgnoreRescheduleBeforeInit) { + // We should get two |Schedule| calls: The first when initialization + // completes, the second one after the automatic (since the service doesn't + // have any data yet) fetch finishes. + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(2); + // The |RescheduleFetching| call shouldn't do anything (in particular not + // result in an |Unschedule|), since the service isn't initialized yet. + EXPECT_CALL(mock_scheduler(), Unschedule()).Times(0); + auto service = MakeSnippetsServiceWithoutInitialization(); + service->RescheduleFetching(false); + WaitForSnippetsServiceInitialization(); +} + +TEST_F(NTPSnippetsServiceTest, HandleForcedRescheduleBeforeInit) { + { + InSequence s; + // The |RescheduleFetching| call with force=true should result in an + // |Unschedule|, since the service isn't initialized yet. + EXPECT_CALL(mock_scheduler(), Unschedule()).Times(1); + // We should get two |Schedule| calls: The first when initialization + // completes, the second one after the automatic (since the service doesn't + // have any data yet) fetch finishes. + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(2); + } + auto service = MakeSnippetsServiceWithoutInitialization(); + service->RescheduleFetching(true); + WaitForSnippetsServiceInitialization(); +} + +TEST_F(NTPSnippetsServiceTest, RescheduleOnStateChange) { + { + InSequence s; + // Initial startup. + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(2); + // Service gets disabled. + EXPECT_CALL(mock_scheduler(), Unschedule()); + // Service gets enabled again. + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(2); + } + auto service = MakeSnippetsService(); + ASSERT_TRUE(service->ready()); + + service->OnSnippetsStatusChanged(SnippetsStatus::ENABLED_AND_SIGNED_IN, + SnippetsStatus::EXPLICITLY_DISABLED); + ASSERT_FALSE(service->ready()); + base::RunLoop().RunUntilIdle(); + + service->OnSnippetsStatusChanged(SnippetsStatus::EXPLICITLY_DISABLED, + SnippetsStatus::ENABLED_AND_SIGNED_OUT); + ASSERT_TRUE(service->ready()); + base::RunLoop().RunUntilIdle(); +} + +TEST_F(NTPSnippetsServiceTest, DontUnscheduleOnShutdown) { + EXPECT_CALL(mock_scheduler(), Schedule(_, _)).Times(2); + EXPECT_CALL(mock_scheduler(), Unschedule()).Times(0); + + auto service = MakeSnippetsService(); + + service.reset(); + base::RunLoop().RunUntilIdle(); +} + +TEST_F(NTPSnippetsServiceTest, Full) { + std::string json_str(GetTestJson({GetSnippet()})); + + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), json_str); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + + const ContentSuggestion& suggestion = + observer().SuggestionsForCategory(articles_category()).front(); + + EXPECT_EQ(MakeArticleID(kSnippetUrl), suggestion.id()); + EXPECT_EQ(kSnippetTitle, base::UTF16ToUTF8(suggestion.title())); + EXPECT_EQ(kSnippetText, base::UTF16ToUTF8(suggestion.snippet_text())); + EXPECT_EQ(GetDefaultCreationTime(), suggestion.publish_date()); + EXPECT_EQ(kSnippetPublisherName, + base::UTF16ToUTF8(suggestion.publisher_name())); + EXPECT_EQ(GURL(kSnippetAmpUrl), suggestion.amp_url()); +} + +TEST_F(NTPSnippetsServiceTest, CategoryTitle) { + const base::string16 response_title = + base::UTF8ToUTF16(kTestJsonDefaultCategoryTitle); + + auto service = MakeSnippetsService(); + + // The articles category should be there by default, and have a title. + CategoryInfo info_before = service->GetCategoryInfo(articles_category()); + ASSERT_FALSE(info_before.title().empty()); + ASSERT_NE(info_before.title(), response_title); + + std::string json_str_no_title(GetTestJsonWithoutTitle({GetSnippet()})); + LoadFromJSONString(service.get(), json_str_no_title); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + + // The response didn't contain a category title. Make sure we didn't touch + // the existing one. + CategoryInfo info_no_title = service->GetCategoryInfo(articles_category()); + EXPECT_EQ(info_before.title(), info_no_title.title()); + + std::string json_str_with_title(GetTestJson({GetSnippet()})); + LoadFromJSONString(service.get(), json_str_with_title); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + + // This time, the response contained a title, |kTestJsonDefaultCategoryTitle|. + // Make sure we updated the title in the CategoryInfo. + CategoryInfo info_with_title = service->GetCategoryInfo(articles_category()); + EXPECT_NE(info_before.title(), info_with_title.title()); + EXPECT_EQ(response_title, info_with_title.title()); +} + +TEST_F(NTPSnippetsServiceTest, MultipleCategories) { + std::string json_str( + GetMultiCategoryJson({GetSnippetN(0)}, {GetSnippetN(1)})); + + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), json_str); + + ASSERT_THAT(observer().statuses(), + Eq(std::map<Category, CategoryStatus, Category::CompareByID>{ + {articles_category(), CategoryStatus::AVAILABLE}, + {other_category(), CategoryStatus::AVAILABLE}, + })); + + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + EXPECT_THAT(service->GetSnippetsForTesting(other_category()), SizeIs(1)); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); + + ASSERT_THAT(observer().SuggestionsForCategory(other_category()), SizeIs(1)); + + { + const ContentSuggestion& suggestion = + observer().SuggestionsForCategory(articles_category()).front(); + EXPECT_EQ(MakeArticleID(std::string(kSnippetUrl) + "/0"), suggestion.id()); + EXPECT_EQ(kSnippetTitle, base::UTF16ToUTF8(suggestion.title())); + EXPECT_EQ(kSnippetText, base::UTF16ToUTF8(suggestion.snippet_text())); + EXPECT_EQ(GetDefaultCreationTime(), suggestion.publish_date()); + EXPECT_EQ(kSnippetPublisherName, + base::UTF16ToUTF8(suggestion.publisher_name())); + EXPECT_EQ(GURL(kSnippetAmpUrl), suggestion.amp_url()); + } + + { + const ContentSuggestion& suggestion = + observer().SuggestionsForCategory(other_category()).front(); + EXPECT_EQ(MakeOtherID(std::string(kSnippetUrl) + "/1"), suggestion.id()); + EXPECT_EQ(kSnippetTitle, base::UTF16ToUTF8(suggestion.title())); + EXPECT_EQ(kSnippetText, base::UTF16ToUTF8(suggestion.snippet_text())); + EXPECT_EQ(GetDefaultCreationTime(), suggestion.publish_date()); + EXPECT_EQ(kSnippetPublisherName, + base::UTF16ToUTF8(suggestion.publisher_name())); + EXPECT_EQ(GURL(kSnippetAmpUrl), suggestion.amp_url()); + } +} + +TEST_F(NTPSnippetsServiceTest, PersistCategoryInfos) { + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), + GetMultiCategoryJson({GetSnippetN(0)}, {GetSnippetN(1)}, + kUnknownRemoteCategoryId)); + + ASSERT_EQ(observer().StatusForCategory(articles_category()), + CategoryStatus::AVAILABLE); + ASSERT_EQ(observer().StatusForCategory(unknown_category()), + CategoryStatus::AVAILABLE); + + CategoryInfo info_articles_before = + service->GetCategoryInfo(articles_category()); + CategoryInfo info_unknown_before = + service->GetCategoryInfo(unknown_category()); + + // Recreate the service to simulate a Chrome restart. + ResetSnippetsService(&service); + + // The categories should have been restored. + ASSERT_NE(observer().StatusForCategory(articles_category()), + CategoryStatus::NOT_PROVIDED); + ASSERT_NE(observer().StatusForCategory(unknown_category()), + CategoryStatus::NOT_PROVIDED); + + EXPECT_EQ(observer().StatusForCategory(articles_category()), + CategoryStatus::AVAILABLE); + EXPECT_EQ(observer().StatusForCategory(unknown_category()), + CategoryStatus::AVAILABLE); + + CategoryInfo info_articles_after = + service->GetCategoryInfo(articles_category()); + CategoryInfo info_unknown_after = + service->GetCategoryInfo(unknown_category()); + + EXPECT_EQ(info_articles_before.title(), info_articles_after.title()); + EXPECT_EQ(info_unknown_before.title(), info_unknown_after.title()); +} + +TEST_F(NTPSnippetsServiceTest, PersistSuggestions) { + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), + GetMultiCategoryJson({GetSnippetN(0)}, {GetSnippetN(1)})); + + ASSERT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); + ASSERT_THAT(observer().SuggestionsForCategory(other_category()), SizeIs(1)); + + // Recreate the service to simulate a Chrome restart. + ResetSnippetsService(&service); + + // The suggestions in both categories should have been restored. + EXPECT_THAT(observer().SuggestionsForCategory(articles_category()), + SizeIs(1)); + EXPECT_THAT(observer().SuggestionsForCategory(other_category()), SizeIs(1)); +} + +TEST_F(NTPSnippetsServiceTest, Clear) { + auto service = MakeSnippetsService(); + + std::string json_str(GetTestJson({GetSnippet()})); + + LoadFromJSONString(service.get(), json_str); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + + service->ClearCachedSuggestions(articles_category()); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, ReplaceSnippets) { + auto service = MakeSnippetsService(); + + std::string first("http://first"); + LoadFromJSONString(service.get(), GetTestJson({GetSnippetWithUrl(first)})); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), + ElementsAre(IdEq(first))); + + std::string second("http://second"); + LoadFromJSONString(service.get(), GetTestJson({GetSnippetWithUrl(second)})); + // The snippets loaded last replace all that was loaded previously. + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), + ElementsAre(IdEq(second))); +} + +TEST_F(NTPSnippetsServiceTest, LoadInvalidJson) { + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), GetTestJson({GetInvalidSnippet()})); + EXPECT_THAT(service->snippets_fetcher()->last_status(), + StartsWith("Received invalid JSON")); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, LoadInvalidJsonWithExistingSnippets) { + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), GetTestJson({GetSnippet()})); + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + ASSERT_EQ("OK", service->snippets_fetcher()->last_status()); + + LoadFromJSONString(service.get(), GetTestJson({GetInvalidSnippet()})); + EXPECT_THAT(service->snippets_fetcher()->last_status(), + StartsWith("Received invalid JSON")); + // This should not have changed the existing snippets. + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); +} + +TEST_F(NTPSnippetsServiceTest, LoadIncompleteJson) { + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), GetTestJson({GetIncompleteSnippet()})); + EXPECT_EQ("Invalid / empty list.", + service->snippets_fetcher()->last_status()); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, LoadIncompleteJsonWithExistingSnippets) { + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), GetTestJson({GetSnippet()})); + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + + LoadFromJSONString(service.get(), GetTestJson({GetIncompleteSnippet()})); + EXPECT_EQ("Invalid / empty list.", + service->snippets_fetcher()->last_status()); + // This should not have changed the existing snippets. + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); +} + +TEST_F(NTPSnippetsServiceTest, Dismiss) { + auto service = MakeSnippetsService(); + + std::string json_str( + GetTestJson({GetSnippetWithSources("http://site.com", "Source 1", "")})); + + LoadFromJSONString(service.get(), json_str); + + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + + // Dismissing a non-existent snippet shouldn't do anything. + service->DismissSuggestion(MakeArticleID("http://othersite.com")); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + + // Dismiss the snippet. + service->DismissSuggestion(MakeArticleID(kSnippetUrl)); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); + + // Make sure that fetching the same snippet again does not re-add it. + LoadFromJSONString(service.get(), json_str); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); + + // The snippet should stay dismissed even after re-creating the service. + ResetSnippetsService(&service); + LoadFromJSONString(service.get(), json_str); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); + + // The snippet can be added again after clearing dismissed snippets. + service->ClearDismissedSuggestionsForDebugging(articles_category()); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); + LoadFromJSONString(service.get(), json_str); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); +} + +TEST_F(NTPSnippetsServiceTest, GetDismissed) { + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), GetTestJson({GetSnippet()})); + + service->DismissSuggestion(MakeArticleID(kSnippetUrl)); + + service->GetDismissedSuggestionsForDebugging( + articles_category(), + base::Bind( + [](NTPSnippetsService* service, NTPSnippetsServiceTest* test, + std::vector<ContentSuggestion> dismissed_suggestions) { + EXPECT_EQ(1u, dismissed_suggestions.size()); + for (auto& suggestion : dismissed_suggestions) { + EXPECT_EQ(test->MakeArticleID(kSnippetUrl), suggestion.id()); + } + }, + service.get(), this)); + base::RunLoop().RunUntilIdle(); + + // There should be no dismissed snippet after clearing the list. + service->ClearDismissedSuggestionsForDebugging(articles_category()); + service->GetDismissedSuggestionsForDebugging( + articles_category(), + base::Bind( + [](NTPSnippetsService* service, NTPSnippetsServiceTest* test, + std::vector<ContentSuggestion> dismissed_suggestions) { + EXPECT_EQ(0u, dismissed_suggestions.size()); + }, + service.get(), this)); + base::RunLoop().RunUntilIdle(); +} + +TEST_F(NTPSnippetsServiceTest, CreationTimestampParseFail) { + auto service = MakeSnippetsService(); + + std::string json = + GetSnippetWithTimes(GetDefaultCreationTime(), GetDefaultExpirationTime()); + base::ReplaceFirstSubstringAfterOffset( + &json, 0, FormatTime(GetDefaultCreationTime()), "aaa1448459205"); + std::string json_str(GetTestJson({json})); + + LoadFromJSONString(service.get(), json_str); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, RemoveExpiredDismissedContent) { + auto service = MakeSnippetsService(); + + std::string json_str1(GetTestJson({GetExpiredSnippet()})); + // Load it. + LoadFromJSONString(service.get(), json_str1); + // Dismiss the suggestion + service->DismissSuggestion( + ContentSuggestion::ID(articles_category(), kSnippetUrl)); + + // Load a different snippet - this will clear the expired dismissed ones. + std::string json_str2(GetTestJson({GetSnippetWithUrl(kSnippetUrl2)})); + LoadFromJSONString(service.get(), json_str2); + + EXPECT_THAT(service->GetDismissedSnippetsForTesting(articles_category()), + IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, ExpiredContentNotRemoved) { + auto service = MakeSnippetsService(); + + std::string json_str(GetTestJson({GetExpiredSnippet()})); + + LoadFromJSONString(service.get(), json_str); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); +} + +TEST_F(NTPSnippetsServiceTest, TestSingleSource) { + auto service = MakeSnippetsService(); + + std::string json_str(GetTestJson({GetSnippetWithSources( + "http://source1.com", "Source 1", "http://source1.amp.com")})); + + LoadFromJSONString(service.get(), json_str); + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + const NTPSnippet& snippet = + *service->GetSnippetsForTesting(articles_category()).front(); + EXPECT_EQ(snippet.sources().size(), 1u); + EXPECT_EQ(snippet.id(), kSnippetUrl); + EXPECT_EQ(snippet.best_source().url, GURL("http://source1.com")); + EXPECT_EQ(snippet.best_source().publisher_name, std::string("Source 1")); + EXPECT_EQ(snippet.best_source().amp_url, GURL("http://source1.amp.com")); +} + +TEST_F(NTPSnippetsServiceTest, TestSingleSourceWithMalformedUrl) { + auto service = MakeSnippetsService(); + + std::string json_str(GetTestJson({GetSnippetWithSources( + "ceci n'est pas un url", "Source 1", "http://source1.amp.com")})); + + LoadFromJSONString(service.get(), json_str); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, TestSingleSourceWithMissingData) { + auto service = MakeSnippetsService(); + + std::string json_str( + GetTestJson({GetSnippetWithSources("http://source1.com", "", "")})); + + LoadFromJSONString(service.get(), json_str); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, LogNumArticlesHistogram) { + auto service = MakeSnippetsService(); + + base::HistogramTester tester; + LoadFromJSONString(service.get(), GetTestJson({GetInvalidSnippet()})); + + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); + + // Invalid JSON shouldn't contribute to NumArticlesFetched. + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), + IsEmpty()); + + // Valid JSON with empty list. + LoadFromJSONString(service.get(), GetTestJson(std::vector<std::string>())); + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/2))); + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1))); + + // Snippet list should be populated with size 1. + LoadFromJSONString(service.get(), GetTestJson({GetSnippet()})); + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/2), + base::Bucket(/*min=*/1, /*count=*/1))); + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1), + base::Bucket(/*min=*/1, /*count=*/1))); + + // Duplicate snippet shouldn't increase the list size. + LoadFromJSONString(service.get(), GetTestJson({GetSnippet()})); + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/2), + base::Bucket(/*min=*/1, /*count=*/2))); + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1), + base::Bucket(/*min=*/1, /*count=*/2))); + EXPECT_THAT( + tester.GetAllSamples("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded"), + IsEmpty()); + + // Dismissing a snippet should decrease the list size. This will only be + // logged after the next fetch. + service->DismissSuggestion(MakeArticleID(kSnippetUrl)); + LoadFromJSONString(service.get(), GetTestJson({GetSnippet()})); + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticles"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/3), + base::Bucket(/*min=*/1, /*count=*/2))); + // Dismissed snippets shouldn't influence NumArticlesFetched. + EXPECT_THAT(tester.GetAllSamples("NewTabPage.Snippets.NumArticlesFetched"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1), + base::Bucket(/*min=*/1, /*count=*/3))); + EXPECT_THAT( + tester.GetAllSamples("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded"), + ElementsAre(base::Bucket(/*min=*/1, /*count=*/1))); + + // There is only a single, dismissed snippet in the database, so recreating + // the service will require us to re-fetch. + tester.ExpectTotalCount("NewTabPage.Snippets.NumArticlesFetched", 4); + ResetSnippetsService(&service); + EXPECT_EQ(observer().StatusForCategory(articles_category()), + CategoryStatus::AVAILABLE); + tester.ExpectTotalCount("NewTabPage.Snippets.NumArticlesFetched", 5); + EXPECT_THAT( + tester.GetAllSamples("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded"), + ElementsAre(base::Bucket(/*min=*/1, /*count=*/2))); + + // But if there's a non-dismissed snippet in the database, recreating it + // shouldn't trigger a fetch. + LoadFromJSONString( + service.get(), + GetTestJson({GetSnippetWithUrl("http://not-dismissed.com")})); + tester.ExpectTotalCount("NewTabPage.Snippets.NumArticlesFetched", 6); + ResetSnippetsService(&service); + tester.ExpectTotalCount("NewTabPage.Snippets.NumArticlesFetched", 6); +} + +TEST_F(NTPSnippetsServiceTest, DismissShouldRespectAllKnownUrls) { + auto service = MakeSnippetsService(); + + const base::Time creation = GetDefaultCreationTime(); + const base::Time expiry = GetDefaultExpirationTime(); + const std::vector<std::string> source_urls = { + "http://mashable.com/2016/05/11/stolen", + "http://www.aol.com/article/2016/05/stolen-doggie"}; + const std::vector<std::string> publishers = {"Mashable", "AOL"}; + const std::vector<std::string> amp_urls = { + "http://mashable-amphtml.googleusercontent.com/1", + "http://t2.gstatic.com/images?q=tbn:3"}; + + // Add the snippet from the mashable domain. + LoadFromJSONString(service.get(), + GetTestJson({GetSnippetWithUrlAndTimesAndSource( + source_urls, source_urls[0], creation, expiry, + publishers[0], amp_urls[0])})); + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + // Dismiss the snippet via the mashable source corpus ID. + service->DismissSuggestion(MakeArticleID(source_urls[0])); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); + + // The same article from the AOL domain should now be detected as dismissed. + LoadFromJSONString(service.get(), + GetTestJson({GetSnippetWithUrlAndTimesAndSource( + source_urls, source_urls[1], creation, expiry, + publishers[1], amp_urls[1])})); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, StatusChanges) { + auto service = MakeSnippetsService(); + + // Simulate user signed out + SetUpFetchResponse(GetTestJson({GetSnippet()})); + service->OnSnippetsStatusChanged(SnippetsStatus::ENABLED_AND_SIGNED_IN, + SnippetsStatus::SIGNED_OUT_AND_DISABLED); + + base::RunLoop().RunUntilIdle(); + EXPECT_THAT(observer().StatusForCategory(articles_category()), + Eq(CategoryStatus::SIGNED_OUT)); + EXPECT_THAT(NTPSnippetsService::State::DISABLED, Eq(service->state_)); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), + IsEmpty()); // No fetch should be made. + + // Simulate user sign in. The service should be ready again and load snippets. + SetUpFetchResponse(GetTestJson({GetSnippet()})); + service->OnSnippetsStatusChanged(SnippetsStatus::SIGNED_OUT_AND_DISABLED, + SnippetsStatus::ENABLED_AND_SIGNED_IN); + EXPECT_THAT(observer().StatusForCategory(articles_category()), + Eq(CategoryStatus::AVAILABLE_LOADING)); + + base::RunLoop().RunUntilIdle(); + EXPECT_THAT(observer().StatusForCategory(articles_category()), + Eq(CategoryStatus::AVAILABLE)); + EXPECT_THAT(NTPSnippetsService::State::READY, Eq(service->state_)); + EXPECT_FALSE(service->GetSnippetsForTesting(articles_category()).empty()); +} + +TEST_F(NTPSnippetsServiceTest, ImageReturnedWithTheSameId) { + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), GetTestJson({GetSnippet()})); + + gfx::Image image; + MockFunction<void(const gfx::Image&)> image_fetched; + ServeImageCallback cb = base::Bind(&ServeOneByOneImage, service.get()); + { + InSequence s; + EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _)) + .WillOnce(WithArgs<0, 2>(Invoke(&cb, &ServeImageCallback::Run))); + EXPECT_CALL(image_fetched, Call(_)).WillOnce(SaveArg<0>(&image)); + } + + service->FetchSuggestionImage( + MakeArticleID(kSnippetUrl), + base::Bind(&MockFunction<void(const gfx::Image&)>::Call, + base::Unretained(&image_fetched))); + base::RunLoop().RunUntilIdle(); + // Check that the image by ServeOneByOneImage is really served. + EXPECT_EQ(1, image.Width()); +} + +TEST_F(NTPSnippetsServiceTest, EmptyImageReturnedForNonExistentId) { + auto service = MakeSnippetsService(); + + // Create a non-empty image so that we can test the image gets updated. + gfx::Image image = gfx::test::CreateImage(1, 1); + MockFunction<void(const gfx::Image&)> image_fetched; + EXPECT_CALL(image_fetched, Call(_)).WillOnce(SaveArg<0>(&image)); + + service->FetchSuggestionImage( + MakeArticleID(kSnippetUrl2), + base::Bind(&MockFunction<void(const gfx::Image&)>::Call, + base::Unretained(&image_fetched))); + + base::RunLoop().RunUntilIdle(); + EXPECT_TRUE(image.IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, ClearHistoryRemovesAllSuggestions) { + auto service = MakeSnippetsService(); + + std::string first_snippet = GetSnippetWithUrl("http://url1.com"); + std::string second_snippet = GetSnippetWithUrl("http://url2.com"); + std::string json_str = GetTestJson({first_snippet, second_snippet}); + LoadFromJSONString(service.get(), json_str); + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(2)); + + service->DismissSuggestion(MakeArticleID("http://url1.com")); + ASSERT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + ASSERT_THAT(service->GetDismissedSnippetsForTesting(articles_category()), + SizeIs(1)); + + base::Time begin = base::Time::FromTimeT(123), + end = base::Time::FromTimeT(456); + base::Callback<bool(const GURL& url)> filter; + service->ClearHistory(begin, end, filter); + + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), IsEmpty()); + EXPECT_THAT(service->GetDismissedSnippetsForTesting(articles_category()), + IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, SuggestionsFetchedOnSignInAndSignOut) { + auto service = MakeSnippetsService(); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(0)); + + // |MakeSnippetsService()| creates a service where user is signed in already, + // so we start by signing out. + SetUpFetchResponse(GetTestJson({GetSnippetN(1)})); + service->OnSnippetsStatusChanged(SnippetsStatus::ENABLED_AND_SIGNED_IN, + SnippetsStatus::ENABLED_AND_SIGNED_OUT); + base::RunLoop().RunUntilIdle(); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(1)); + + // Sign in to check a transition from signed out to signed in. + SetUpFetchResponse(GetTestJson({GetSnippetN(1), GetSnippetN(2)})); + service->OnSnippetsStatusChanged(SnippetsStatus::ENABLED_AND_SIGNED_OUT, + SnippetsStatus::ENABLED_AND_SIGNED_IN); + base::RunLoop().RunUntilIdle(); + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), SizeIs(2)); +} + +namespace { + +gfx::Image FetchImage(NTPSnippetsService* service, + const ContentSuggestion::ID& suggestion_id) { + gfx::Image result; + base::RunLoop run_loop; + service->FetchSuggestionImage(suggestion_id, + base::Bind( + [](base::Closure signal, gfx::Image* output, + const gfx::Image& loaded) { + *output = loaded; + signal.Run(); + }, + run_loop.QuitClosure(), &result)); + run_loop.Run(); + return result; +} + +} // namespace + +TEST_F(NTPSnippetsServiceTest, ShouldClearOrphanedImagesOnRestart) { + auto service = MakeSnippetsService(); + + LoadFromJSONString(service.get(), GetTestJson({GetSnippet()})); + ServeImageCallback cb = base::Bind(&ServeOneByOneImage, service.get()); + + EXPECT_CALL(*image_fetcher(), StartOrQueueNetworkRequest(_, _, _)) + .WillOnce(WithArgs<0, 2>(Invoke(&cb, &ServeImageCallback::Run))); + image_decoder()->SetDecodedImage(gfx::test::CreateImage(1, 1)); + + gfx::Image image = FetchImage(service.get(), MakeArticleID(kSnippetUrl)); + EXPECT_EQ(1, image.Width()); + EXPECT_FALSE(image.IsEmpty()); + + // Send new suggestion which don't include the snippet referencing the image. + LoadFromJSONString(service.get(), + GetTestJson({GetSnippetWithUrl( + "http://something.com/pletely/unrelated")})); + // The image should still be available until a restart happens. + EXPECT_FALSE(FetchImage(service.get(), MakeArticleID(kSnippetUrl)).IsEmpty()); + ResetSnippetsService(&service); + // After the restart, the image should be garbage collected. + EXPECT_TRUE(FetchImage(service.get(), MakeArticleID(kSnippetUrl)).IsEmpty()); +} + +TEST_F(NTPSnippetsServiceTest, ShouldHandleMoreThanMaxSnippetsInResponse) { + auto service = MakeSnippetsService(); + + std::vector<std::string> suggestions; + for (int i = 0 ; i < service->GetMaxSnippetCountForTesting() + 1; ++i) { + suggestions.push_back(GetSnippetWithUrl( + base::StringPrintf("http://localhost/snippet-id-%d", i))); + } + LoadFromJSONString(service.get(), GetTestJson(suggestions)); + // TODO(tschumann): We should probably trim out any additional results and + // only serve the MaxSnippetCount items. + EXPECT_THAT(service->GetSnippetsForTesting(articles_category()), + SizeIs(service->GetMaxSnippetCountForTesting() + 1)); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/ntp_snippets_status_service.cc b/chromium/components/ntp_snippets/remote/ntp_snippets_status_service.cc new file mode 100644 index 00000000000..62514d270b1 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_status_service.cc @@ -0,0 +1,119 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/ntp_snippets_status_service.h" + +#include <string> + +#include "components/ntp_snippets/features.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" +#include "components/variations/variations_associated_data.h" + +namespace ntp_snippets { + +namespace { + +const char kFetchingRequiresSignin[] = "fetching_requires_signin"; +const char kFetchingRequiresSigninEnabled[] = "true"; +const char kFetchingRequiresSigninDisabled[] = "false"; + +} // namespace + +NTPSnippetsStatusService::NTPSnippetsStatusService( + SigninManagerBase* signin_manager, + PrefService* pref_service) + : snippets_status_(SnippetsStatus::EXPLICITLY_DISABLED), + require_signin_(false), + signin_manager_(signin_manager), + pref_service_(pref_service), + signin_observer_(this) { + std::string param_value_str = variations::GetVariationParamValueByFeature( + kArticleSuggestionsFeature, kFetchingRequiresSignin); + if (param_value_str == kFetchingRequiresSigninEnabled) { + require_signin_ = true; + } else if (!param_value_str.empty() && + param_value_str != kFetchingRequiresSigninDisabled) { + DLOG(WARNING) << "Unknow value for the variations parameter " + << kFetchingRequiresSignin << ": " << param_value_str; + } +} + +NTPSnippetsStatusService::~NTPSnippetsStatusService() = default; + +// static +void NTPSnippetsStatusService::RegisterProfilePrefs( + PrefRegistrySimple* registry) { + registry->RegisterBooleanPref(prefs::kEnableSnippets, true); +} + +void NTPSnippetsStatusService::Init( + const SnippetsStatusChangeCallback& callback) { + DCHECK(snippets_status_change_callback_.is_null()); + + snippets_status_change_callback_ = callback; + + // Notify about the current state before registering the observer, to make + // sure we don't get a double notification due to an undefined start state. + SnippetsStatus old_snippets_status = snippets_status_; + snippets_status_ = GetSnippetsStatusFromDeps(); + snippets_status_change_callback_.Run(old_snippets_status, snippets_status_); + + signin_observer_.Add(signin_manager_); + + pref_change_registrar_.Init(pref_service_); + pref_change_registrar_.Add( + prefs::kEnableSnippets, + base::Bind(&NTPSnippetsStatusService::OnSnippetsEnabledChanged, + base::Unretained(this))); +} + +void NTPSnippetsStatusService::OnSnippetsEnabledChanged() { + OnStateChanged(GetSnippetsStatusFromDeps()); +} + +void NTPSnippetsStatusService::OnStateChanged( + SnippetsStatus new_snippets_status) { + if (new_snippets_status == snippets_status_) + return; + + snippets_status_change_callback_.Run(snippets_status_, new_snippets_status); + snippets_status_ = new_snippets_status; +} + +bool NTPSnippetsStatusService::IsSignedIn() const { + return signin_manager_ && signin_manager_->IsAuthenticated(); +} + +void NTPSnippetsStatusService::GoogleSigninSucceeded( + const std::string& account_id, + const std::string& username, + const std::string& password) { + OnStateChanged(GetSnippetsStatusFromDeps()); +} + +void NTPSnippetsStatusService::GoogleSignedOut(const std::string& account_id, + const std::string& username) { + OnStateChanged(GetSnippetsStatusFromDeps()); +} + +SnippetsStatus NTPSnippetsStatusService::GetSnippetsStatusFromDeps() const { + if (!pref_service_->GetBoolean(prefs::kEnableSnippets)) { + DVLOG(1) << "[GetNewSnippetsStatus] Disabled via pref"; + return SnippetsStatus::EXPLICITLY_DISABLED; + } + + if (require_signin_ && !IsSignedIn()) { + DVLOG(1) << "[GetNewSnippetsStatus] Signed out and disabled due to this."; + return SnippetsStatus::SIGNED_OUT_AND_DISABLED; + } + + DVLOG(1) << "[GetNewSnippetsStatus] Enabled, signed " + << (IsSignedIn() ? "in" : "out"); + return IsSignedIn() ? SnippetsStatus::ENABLED_AND_SIGNED_IN + : SnippetsStatus::ENABLED_AND_SIGNED_OUT; +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/ntp_snippets_status_service.h b/chromium/components/ntp_snippets/remote/ntp_snippets_status_service.h new file mode 100644 index 00000000000..2f8f47a7635 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_status_service.h @@ -0,0 +1,86 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_STATUS_SERVICE_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_STATUS_SERVICE_H_ + +#include "base/callback.h" +#include "base/gtest_prod_util.h" +#include "base/scoped_observer.h" +#include "components/prefs/pref_change_registrar.h" +#include "components/signin/core/browser/signin_manager.h" + +class PrefRegistrySimple; +class PrefService; + +namespace ntp_snippets { + +enum class SnippetsStatus : int { + // Snippets are enabled and the user is signed in. + ENABLED_AND_SIGNED_IN, + // Snippets are enabled and the user is signed out (sign in is not required). + ENABLED_AND_SIGNED_OUT, + // Snippets have been disabled as part of the service configuration. + EXPLICITLY_DISABLED, + // The user is not signed in, and the service requires it to be enabled. + SIGNED_OUT_AND_DISABLED, +}; + +// Aggregates data from preferences and signin to notify the snippet service of +// relevant changes in their states. +class NTPSnippetsStatusService : public SigninManagerBase::Observer { + public: + using SnippetsStatusChangeCallback = + base::Callback<void(SnippetsStatus /*old_status*/, + SnippetsStatus /*new_status*/)>; + + NTPSnippetsStatusService(SigninManagerBase* signin_manager, + PrefService* pref_service); + + ~NTPSnippetsStatusService() override; + + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + // Starts listening for changes from the dependencies. |callback| will be + // called when a significant change in state is detected. + void Init(const SnippetsStatusChangeCallback& callback); + + private: + FRIEND_TEST_ALL_PREFIXES(NTPSnippetsStatusServiceTest, DisabledViaPref); + + // SigninManagerBase::Observer implementation + void GoogleSigninSucceeded(const std::string& account_id, + const std::string& username, + const std::string& password) override; + void GoogleSignedOut(const std::string& account_id, + const std::string& username) override; + + // Callback for the PrefChangeRegistrar. + void OnSnippetsEnabledChanged(); + + void OnStateChanged(SnippetsStatus new_snippets_status); + + bool IsSignedIn() const; + + SnippetsStatus GetSnippetsStatusFromDeps() const; + + SnippetsStatus snippets_status_; + SnippetsStatusChangeCallback snippets_status_change_callback_; + + bool require_signin_; + SigninManagerBase* signin_manager_; + PrefService* pref_service_; + + PrefChangeRegistrar pref_change_registrar_; + + // The observer for the SigninManager. + ScopedObserver<SigninManagerBase, SigninManagerBase::Observer> + signin_observer_; + + DISALLOW_COPY_AND_ASSIGN(NTPSnippetsStatusService); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_STATUS_SERVICE_H_ diff --git a/chromium/components/ntp_snippets/remote/ntp_snippets_status_service_unittest.cc b/chromium/components/ntp_snippets/remote/ntp_snippets_status_service_unittest.cc new file mode 100644 index 00000000000..827f7de45ff --- /dev/null +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_status_service_unittest.cc @@ -0,0 +1,59 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/ntp_snippets_status_service.h" + +#include <memory> + +#include "components/ntp_snippets/pref_names.h" +#include "components/ntp_snippets/remote/ntp_snippets_test_utils.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/testing_pref_service.h" +#include "components/signin/core/browser/account_tracker_service.h" +#include "components/signin/core/browser/fake_signin_manager.h" +#include "components/signin/core/browser/test_signin_client.h" +#include "components/signin/core/common/signin_pref_names.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace ntp_snippets { + +class NTPSnippetsStatusServiceTest : public ::testing::Test { + public: + NTPSnippetsStatusServiceTest() { + NTPSnippetsStatusService::RegisterProfilePrefs( + utils_.pref_service()->registry()); + } + + std::unique_ptr<NTPSnippetsStatusService> MakeService() { + return base::MakeUnique<NTPSnippetsStatusService>( + utils_.fake_signin_manager(), utils_.pref_service()); + } + + protected: + test::NTPSnippetsTestUtils utils_; +}; + +// TODO(jkrcal): Extend the ways to override variation parameters in unit-test +// (bug 645447), and recover the SigninStateCompatibility test that sign-in is +// required when the parameter is overriden. +TEST_F(NTPSnippetsStatusServiceTest, DisabledViaPref) { + auto service = MakeService(); + + // The default test setup is signed out. The service is enabled. + ASSERT_EQ(SnippetsStatus::ENABLED_AND_SIGNED_OUT, + service->GetSnippetsStatusFromDeps()); + + // Once the enabled pref is set to false, we should be disabled. + utils_.pref_service()->SetBoolean(prefs::kEnableSnippets, false); + EXPECT_EQ(SnippetsStatus::EXPLICITLY_DISABLED, + service->GetSnippetsStatusFromDeps()); + + // Signing-in shouldn't matter anymore. + utils_.fake_signin_manager()->SignIn("foo@bar.com"); + EXPECT_EQ(SnippetsStatus::EXPLICITLY_DISABLED, + service->GetSnippetsStatusFromDeps()); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/ntp_snippets_test_utils.cc b/chromium/components/ntp_snippets/remote/ntp_snippets_test_utils.cc index c00629c7632..cdcb4501b14 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_test_utils.cc +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_test_utils.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/ntp_snippets/ntp_snippets_test_utils.h" +#include "components/ntp_snippets/remote/ntp_snippets_test_utils.h" #include <memory> @@ -12,41 +12,41 @@ #include "components/signin/core/browser/fake_signin_manager.h" #include "components/signin/core/browser/test_signin_client.h" #include "components/signin/core/common/signin_pref_names.h" -#include "components/sync_driver/fake_sync_service.h" +#include "components/sync/driver/fake_sync_service.h" namespace ntp_snippets { namespace test { -MockSyncService::MockSyncService() +FakeSyncService::FakeSyncService() : can_sync_start_(true), is_sync_active_(true), configuration_done_(true), is_encrypt_everything_enabled_(false), active_data_types_(syncer::HISTORY_DELETE_DIRECTIVES) {} -MockSyncService::~MockSyncService() {} +FakeSyncService::~FakeSyncService() = default; -bool MockSyncService::CanSyncStart() const { +bool FakeSyncService::CanSyncStart() const { return can_sync_start_; } -bool MockSyncService::IsSyncActive() const { +bool FakeSyncService::IsSyncActive() const { return is_sync_active_; } -bool MockSyncService::ConfigurationDone() const { +bool FakeSyncService::ConfigurationDone() const { return configuration_done_; } -bool MockSyncService::IsEncryptEverythingEnabled() const { +bool FakeSyncService::IsEncryptEverythingEnabled() const { return is_encrypt_everything_enabled_; } -syncer::ModelTypeSet MockSyncService::GetActiveDataTypes() const { +syncer::ModelTypeSet FakeSyncService::GetActiveDataTypes() const { return active_data_types_; } -NTPSnippetsTestBase::NTPSnippetsTestBase() +NTPSnippetsTestUtils::NTPSnippetsTestUtils() : pref_service_(new TestingPrefServiceSimple()) { pref_service_->registry()->RegisterStringPref(prefs::kGoogleServicesAccountId, std::string()); @@ -54,18 +54,15 @@ NTPSnippetsTestBase::NTPSnippetsTestBase() prefs::kGoogleServicesLastAccountId, std::string()); pref_service_->registry()->RegisterStringPref( prefs::kGoogleServicesLastUsername, std::string()); -} - -NTPSnippetsTestBase::~NTPSnippetsTestBase() {} - -void NTPSnippetsTestBase::SetUp() { signin_client_.reset(new TestSigninClient(pref_service_.get())); account_tracker_.reset(new AccountTrackerService()); - mock_sync_service_.reset(new MockSyncService()); + fake_sync_service_.reset(new FakeSyncService()); ResetSigninManager(); } -void NTPSnippetsTestBase::ResetSigninManager() { +NTPSnippetsTestUtils::~NTPSnippetsTestUtils() = default; + +void NTPSnippetsTestUtils::ResetSigninManager() { fake_signin_manager_.reset( new FakeSigninManagerBase(signin_client_.get(), account_tracker_.get())); } diff --git a/chromium/components/ntp_snippets/ntp_snippets_test_utils.h b/chromium/components/ntp_snippets/remote/ntp_snippets_test_utils.h index 20e9c854b61..2a61b3d037e 100644 --- a/chromium/components/ntp_snippets/ntp_snippets_test_utils.h +++ b/chromium/components/ntp_snippets/remote/ntp_snippets_test_utils.h @@ -2,12 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_TEST_UTILS_H_ -#define COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_TEST_UTILS_H_ +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_TEST_UTILS_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_TEST_UTILS_H_ #include <memory> -#include "components/sync_driver/fake_sync_service.h" +#include "components/sync/driver/fake_sync_service.h" #include "testing/gtest/include/gtest/gtest.h" class AccountTrackerService; @@ -19,10 +19,10 @@ class TestSigninClient; namespace ntp_snippets { namespace test { -class MockSyncService : public sync_driver::FakeSyncService { +class FakeSyncService : public syncer::FakeSyncService { public: - MockSyncService(); - ~MockSyncService() override; + FakeSyncService(); + ~FakeSyncService() override; bool CanSyncStart() const override; bool IsSyncActive() const override; @@ -37,18 +37,16 @@ class MockSyncService : public sync_driver::FakeSyncService { syncer::ModelTypeSet active_data_types_; }; -// Common base for snippet tests, handles initializing mocks for sync and -// signin. |SetUp()| should be called if a subclass overrides it. -class NTPSnippetsTestBase : public testing::Test { +// Common utilities for snippet tests, handles initializing fakes for sync and +// signin. +class NTPSnippetsTestUtils { public: - void SetUp() override; + NTPSnippetsTestUtils(); + ~NTPSnippetsTestUtils(); - protected: - NTPSnippetsTestBase(); - ~NTPSnippetsTestBase() override; void ResetSigninManager(); - MockSyncService* mock_sync_service() { return mock_sync_service_.get(); } + FakeSyncService* fake_sync_service() { return fake_sync_service_.get(); } FakeSigninManagerBase* fake_signin_manager() { return fake_signin_manager_.get(); } @@ -56,7 +54,7 @@ class NTPSnippetsTestBase : public testing::Test { private: std::unique_ptr<FakeSigninManagerBase> fake_signin_manager_; - std::unique_ptr<MockSyncService> mock_sync_service_; + std::unique_ptr<FakeSyncService> fake_sync_service_; std::unique_ptr<TestingPrefServiceSimple> pref_service_; std::unique_ptr<TestSigninClient> signin_client_; std::unique_ptr<AccountTrackerService> account_tracker_; @@ -65,4 +63,4 @@ class NTPSnippetsTestBase : public testing::Test { } // namespace test } // namespace ntp_snippets -#endif // COMPONENTS_NTP_SNIPPETS_NTP_SNIPPETS_TEST_UTILS_H_ +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_NTP_SNIPPETS_TEST_UTILS_H_ diff --git a/chromium/components/ntp_snippets/proto/BUILD.gn b/chromium/components/ntp_snippets/remote/proto/BUILD.gn index 95ba1796625..95ba1796625 100644 --- a/chromium/components/ntp_snippets/proto/BUILD.gn +++ b/chromium/components/ntp_snippets/remote/proto/BUILD.gn diff --git a/chromium/components/ntp_snippets/proto/ntp_snippets.proto b/chromium/components/ntp_snippets/remote/proto/ntp_snippets.proto index 0225dc69dba..d51c774810f 100644 --- a/chromium/components/ntp_snippets/proto/ntp_snippets.proto +++ b/chromium/components/ntp_snippets/remote/proto/ntp_snippets.proto @@ -23,7 +23,8 @@ message SnippetProto { optional int64 expiry_date = 6; optional float score = 7; repeated SnippetSourceProto sources = 8; - optional bool discarded = 9; + optional bool dismissed = 9; + optional int32 remote_category_id = 10; } message SnippetImageProto { diff --git a/chromium/components/ntp_snippets/remote/request_throttler.cc b/chromium/components/ntp_snippets/remote/request_throttler.cc new file mode 100644 index 00000000000..b6d3bbbf536 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/request_throttler.cc @@ -0,0 +1,192 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/request_throttler.h" + +#include <climits> +#include <vector> + +#include "base/metrics/histogram.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_split.h" +#include "base/strings/stringprintf.h" +#include "base/time/time.h" +#include "components/ntp_snippets/ntp_snippets_constants.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" +#include "components/variations/variations_associated_data.h" + +namespace ntp_snippets { + +namespace { + +// Enumeration listing all possible outcomes for fetch attempts. Used for UMA +// histogram, so do not change existing values. Insert new values at the end, +// and update the histogram definition. +enum class RequestStatus { + INTERACTIVE_QUOTA_GRANTED, + BACKGROUND_QUOTA_GRANTED, + BACKGROUND_QUOTA_EXCEEDED, + INTERACTIVE_QUOTA_EXCEEDED, + REQUEST_STATUS_COUNT +}; + +// Quota value to use if no quota should be applied (by default). +const int kUnlimitedQuota = INT_MAX; + +} // namespace + +struct RequestThrottler::RequestTypeInfo { + const char* name; + const char* count_pref; + const char* interactive_count_pref; + const char* day_pref; + const int default_quota; + const int default_interactive_quota; +}; + +// When adding a new type here, extend also the "RequestThrottlerTypes" +// <histogram_suffixes> in histograms.xml with the |name| string. +const RequestThrottler::RequestTypeInfo RequestThrottler::kRequestTypeInfo[] = { + // RequestCounter::RequestType::CONTENT_SUGGESTION_FETCHER, + {"SuggestionFetcher", prefs::kSnippetFetcherRequestCount, + prefs::kSnippetFetcherInteractiveRequestCount, + prefs::kSnippetFetcherRequestsDay, 50, kUnlimitedQuota}, + // RequestCounter::RequestType::CONTENT_SUGGESTION_THUMBNAIL, + {"SuggestionThumbnailFetcher", prefs::kSnippetThumbnailsRequestCount, + prefs::kSnippetThumbnailsInteractiveRequestCount, + prefs::kSnippetThumbnailsRequestsDay, kUnlimitedQuota, kUnlimitedQuota}}; + +RequestThrottler::RequestThrottler(PrefService* pref_service, RequestType type) + : pref_service_(pref_service), + type_info_(kRequestTypeInfo[static_cast<int>(type)]) { + DCHECK(pref_service); + + std::string quota = variations::GetVariationParamValue( + ntp_snippets::kStudyName, + base::StringPrintf("quota_%s", GetRequestTypeName())); + if (!base::StringToInt(quota, "a_)) { + LOG_IF(WARNING, !quota.empty()) + << "Invalid variation parameter for quota for " + << GetRequestTypeName(); + quota_ = type_info_.default_quota; + } + + std::string interactive_quota = variations::GetVariationParamValue( + ntp_snippets::kStudyName, + base::StringPrintf("interactive_quota_%s", GetRequestTypeName())); + if (!base::StringToInt(interactive_quota, &interactive_quota_)) { + LOG_IF(WARNING, !interactive_quota.empty()) + << "Invalid variation parameter for interactive quota for " + << GetRequestTypeName(); + interactive_quota_ = type_info_.default_interactive_quota; + } + + // Since the histogram names are dynamic, we cannot use the standard macros + // and we need to lookup the histograms, instead. + int status_count = static_cast<int>(RequestStatus::REQUEST_STATUS_COUNT); + // Corresponds to UMA_HISTOGRAM_ENUMERATION(name, sample, |status_count|). + histogram_request_status_ = base::LinearHistogram::FactoryGet( + base::StringPrintf("NewTabPage.RequestThrottler.RequestStatus_%s", + GetRequestTypeName()), + 1, status_count, status_count + 1, + base::HistogramBase::kUmaTargetedHistogramFlag); + // Corresponds to UMA_HISTOGRAM_COUNTS_100(name, sample). + histogram_per_day_background_ = base::Histogram::FactoryGet( + base::StringPrintf("NewTabPage.RequestThrottler.PerDay_%s", + GetRequestTypeName()), + 1, 100, 50, base::HistogramBase::kUmaTargetedHistogramFlag); + // Corresponds to UMA_HISTOGRAM_COUNTS_100(name, sample). + histogram_per_day_interactive_ = base::Histogram::FactoryGet( + base::StringPrintf("NewTabPage.RequestThrottler.PerDayInteractive_%s", + GetRequestTypeName()), + 1, 100, 50, base::HistogramBase::kUmaTargetedHistogramFlag); +} + +// static +void RequestThrottler::RegisterProfilePrefs(PrefRegistrySimple* registry) { + for (const RequestTypeInfo& info : kRequestTypeInfo) { + registry->RegisterIntegerPref(info.count_pref, 0); + registry->RegisterIntegerPref(info.interactive_count_pref, 0); + registry->RegisterIntegerPref(info.day_pref, 0); + } +} + +bool RequestThrottler::DemandQuotaForRequest(bool interactive_request) { + ResetCounterIfDayChanged(); + + int new_count = GetCount(interactive_request) + 1; + SetCount(interactive_request, new_count); + bool available = (new_count <= GetQuota(interactive_request)); + + if (interactive_request) { + histogram_request_status_->Add(static_cast<int>( + available ? RequestStatus::INTERACTIVE_QUOTA_GRANTED + : RequestStatus::INTERACTIVE_QUOTA_EXCEEDED)); + } else { + histogram_request_status_->Add( + static_cast<int>(available ? RequestStatus::BACKGROUND_QUOTA_GRANTED + : RequestStatus::BACKGROUND_QUOTA_EXCEEDED)); + } + return available; +} + +void RequestThrottler::ResetCounterIfDayChanged() { + // Get the date, "concatenated" into an int in "YYYYMMDD" format. + base::Time::Exploded now_exploded; + base::Time::Now().LocalExplode(&now_exploded); + int now_day = 10000 * now_exploded.year + 100 * now_exploded.month + + now_exploded.day_of_month; + + if (!HasDay()) { + // The counter is used for the first time in this profile. + SetDay(now_day); + } else if (now_day != GetDay()) { + // Day has changed - report the number of requests from the previous day. + histogram_per_day_background_->Add(GetCount(/*interactive_request=*/false)); + histogram_per_day_interactive_->Add(GetCount(/*interactive_request=*/true)); + // Reset the counters. + SetCount(/*interactive_request=*/false, 0); + SetCount(/*interactive_request=*/true, 0); + SetDay(now_day); + } +} + +const char* RequestThrottler::GetRequestTypeName() const { + return type_info_.name; +} + +// TODO(jkrcal): turn RequestTypeInfo into a proper class, move those methods +// onto the class and hide the members. +int RequestThrottler::GetQuota(bool interactive_request) const { + return interactive_request ? interactive_quota_ : quota_; +} + +int RequestThrottler::GetCount(bool interactive_request) const { + return pref_service_->GetInteger(interactive_request + ? type_info_.interactive_count_pref + : type_info_.count_pref); +} + +void RequestThrottler::SetCount(bool interactive_request, int count) { + pref_service_->SetInteger(interactive_request + ? type_info_.interactive_count_pref + : type_info_.count_pref, + count); +} + +int RequestThrottler::GetDay() const { + return pref_service_->GetInteger(type_info_.day_pref); +} + +void RequestThrottler::SetDay(int day) { + pref_service_->SetInteger(type_info_.day_pref, day); +} + +bool RequestThrottler::HasDay() const { + return pref_service_->HasPrefPath(type_info_.day_pref); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/remote/request_throttler.h b/chromium/components/ntp_snippets/remote/request_throttler.h new file mode 100644 index 00000000000..30c2dd557c5 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/request_throttler.h @@ -0,0 +1,96 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_REQUEST_THROTTLER_H_ +#define COMPONENTS_NTP_SNIPPETS_REMOTE_REQUEST_THROTTLER_H_ + +#include <string> + +#include "base/macros.h" + +class PrefRegistrySimple; +class PrefService; + +namespace base { +class HistogramBase; +} // namespace base + +namespace ntp_snippets { + +// Counts requests to external services, compares them to a daily quota, reports +// them to UMA. In the application code, create one local instance for each type +// of requests, identified by the RequestType. The request counter is based on: +// - daily quota from a variation param "quota_|type|" in the NTPSnippets trial +// - pref "ntp.request_throttler.|type|.count" to store the current counter, +// - pref "ntp.request_throttler.|type|.day" to store current day to which the +// current counter value applies. +// Furthermore the counter reports to histograms: +// - "NewTabPage.RequestThrottler.RequestStatus_|type|" - status of each +// request; +// - "NewTabPage.RequestThrottler.PerDay_|type|" - the daily count of requests. +// +// Implementation notes: When extending this class for a new RequestType, please +// 1) define in request_counter.cc in kRequestTypeInfo +// a) the string value for your |type| and +// b) constants for day/count prefs; +// 2) define a new RequestThrottlerTypes histogram suffix in histogram.xml +// (with the same string value as in 1a)). +class RequestThrottler { + public: + // Enumeration listing all current applications of the request counter. + enum class RequestType { + CONTENT_SUGGESTION_FETCHER, + CONTENT_SUGGESTION_THUMBNAIL, + }; + + RequestThrottler(PrefService* pref_service, RequestType type); + + // Registers profile prefs for all RequestTypes. Called from browser_prefs.cc. + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + // Returns whether quota is available for another request and reports this + // information to UMA. Interactive requests should be always granted (upon + // standard conditions) and should be only used for requests initiated by the + // user (if it is safe to assume that all users cannot generate an amount of + // requests we cannot handle). + bool DemandQuotaForRequest(bool interactive_request); + + private: + friend class RequestThrottlerTest; + // Used internally for working with a RequestType. + struct RequestTypeInfo; + + // The array of info entries - one per each RequestType. + static const RequestTypeInfo kRequestTypeInfo[]; + + // Also emits the PerDay histogram if the day changed. + void ResetCounterIfDayChanged(); + + const char* GetRequestTypeName() const; + + int GetQuota(bool interactive_request) const; + int GetCount(bool interactive_request) const; + void SetCount(bool interactive_request, int count); + int GetDay() const; + void SetDay(int day); + bool HasDay() const; + + PrefService* pref_service_; + const RequestTypeInfo& type_info_; + + // The quotas are hardcoded, but can be overridden by variation params. + int quota_; + int interactive_quota_; + + // The histograms for reporting the requests of the given |type_|. + base::HistogramBase* histogram_request_status_; + base::HistogramBase* histogram_per_day_background_; + base::HistogramBase* histogram_per_day_interactive_; + + DISALLOW_COPY_AND_ASSIGN(RequestThrottler); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_REMOTE_REQUEST_THROTTLER_H_ diff --git a/chromium/components/ntp_snippets/remote/request_throttler_unittest.cc b/chromium/components/ntp_snippets/remote/request_throttler_unittest.cc new file mode 100644 index 00000000000..79eca3b0b88 --- /dev/null +++ b/chromium/components/ntp_snippets/remote/request_throttler_unittest.cc @@ -0,0 +1,72 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/remote/request_throttler.h" + +#include <memory> + +#include "base/strings/stringprintf.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/testing_pref_service.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace { +const int kCounterQuota = 2; +} // namespace + +namespace ntp_snippets { + +class RequestThrottlerTest : public testing::Test { + public: + RequestThrottlerTest() { + RequestThrottler::RegisterProfilePrefs(test_prefs_.registry()); + // Use any arbitrary RequestType for this unittest. + throttler_.reset(new RequestThrottler( + &test_prefs_, + RequestThrottler::RequestType::CONTENT_SUGGESTION_FETCHER)); + throttler_->quota_ = kCounterQuota; + } + + protected: + TestingPrefServiceSimple test_prefs_; + std::unique_ptr<RequestThrottler> throttler_; + + private: + DISALLOW_COPY_AND_ASSIGN(RequestThrottlerTest); +}; + +TEST_F(RequestThrottlerTest, QuotaExceeded) { + EXPECT_TRUE(throttler_->DemandQuotaForRequest(false)); + EXPECT_TRUE(throttler_->DemandQuotaForRequest(false)); + EXPECT_FALSE(throttler_->DemandQuotaForRequest(false)); +} + +TEST_F(RequestThrottlerTest, ForcedDoesNotCountInQuota) { + EXPECT_TRUE(throttler_->DemandQuotaForRequest(false)); + EXPECT_TRUE(throttler_->DemandQuotaForRequest(true)); + EXPECT_TRUE(throttler_->DemandQuotaForRequest(false)); +} + +TEST_F(RequestThrottlerTest, ForcedWorksRegardlessOfQuota) { + EXPECT_TRUE(throttler_->DemandQuotaForRequest(false)); + EXPECT_TRUE(throttler_->DemandQuotaForRequest(false)); + EXPECT_TRUE(throttler_->DemandQuotaForRequest(true)); +} + +TEST_F(RequestThrottlerTest, QuotaIsPerDay) { + EXPECT_TRUE(throttler_->DemandQuotaForRequest(false)); + EXPECT_TRUE(throttler_->DemandQuotaForRequest(false)); + + // Now fake the day pref so that the counter believes the count comes from + // yesterday. + int now_day = (base::Time::Now() - base::Time::UnixEpoch()).InDays(); + test_prefs_.SetInteger(ntp_snippets::prefs::kSnippetFetcherRequestsDay, + now_day - 1); + + // The quota should get reset as the day has changed. + EXPECT_TRUE(throttler_->DemandQuotaForRequest(false)); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/sessions/DEPS b/chromium/components/ntp_snippets/sessions/DEPS new file mode 100644 index 00000000000..e279a07903d --- /dev/null +++ b/chromium/components/ntp_snippets/sessions/DEPS @@ -0,0 +1,4 @@ +include_rules = [ + "+components/sessions", + "+components/sync_sessions", +] diff --git a/chromium/components/ntp_snippets/sessions/foreign_sessions_suggestions_provider.cc b/chromium/components/ntp_snippets/sessions/foreign_sessions_suggestions_provider.cc new file mode 100644 index 00000000000..f913d660fca --- /dev/null +++ b/chromium/components/ntp_snippets/sessions/foreign_sessions_suggestions_provider.cc @@ -0,0 +1,314 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/sessions/foreign_sessions_suggestions_provider.h" + +#include <algorithm> +#include <map> +#include <tuple> +#include <utility> + +#include "base/strings/string_piece.h" +#include "base/strings/utf_string_conversions.h" +#include "base/time/time.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/category_info.h" +#include "components/ntp_snippets/content_suggestion.h" +#include "components/ntp_snippets/features.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/ntp_snippets/pref_util.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" +#include "components/sessions/core/session_types.h" +#include "components/sync_sessions/synced_session.h" +#include "grit/components_strings.h" +#include "ui/base/l10n/l10n_util.h" +#include "ui/gfx/image/image.h" +#include "url/gurl.h" + +using base::TimeDelta; +using sessions::SerializedNavigationEntry; +using sessions::SessionTab; +using sessions::SessionWindow; +using sync_sessions::SyncedSession; + +namespace ntp_snippets { +namespace { + +const int kMaxForeignTabsTotal = 10; +const int kMaxForeignTabsPerDevice = 3; +const int kMaxForeignTabAgeInMinutes = 180; + +const char* kMaxForeignTabsTotalParamName = "max_foreign_tabs_total"; +const char* kMaxForeignTabsPerDeviceParamName = "max_foreign_tabs_per_device"; +const char* kMaxForeignTabAgeInMinutesParamName = + "max_foreign_tabs_age_in_minutes"; + +int GetMaxForeignTabsTotal() { + return GetParamAsInt(ntp_snippets::kForeignSessionsSuggestionsFeature, + kMaxForeignTabsTotalParamName, kMaxForeignTabsTotal); +} + +int GetMaxForeignTabsPerDevice() { + return GetParamAsInt(ntp_snippets::kForeignSessionsSuggestionsFeature, + kMaxForeignTabsPerDeviceParamName, + kMaxForeignTabsPerDevice); +} + +TimeDelta GetMaxForeignTabAge() { + return TimeDelta::FromMinutes(GetParamAsInt( + ntp_snippets::kForeignSessionsSuggestionsFeature, + kMaxForeignTabAgeInMinutesParamName, kMaxForeignTabAgeInMinutes)); +} + +} // namespace + +// Collection of pointers to various sessions objects that contain a superset of +// the information needed to create a single suggestion. +struct ForeignSessionsSuggestionsProvider::SessionData { + const sync_sessions::SyncedSession* session; + const sessions::SessionTab* tab; + const sessions::SerializedNavigationEntry* navigation; + bool operator<(const SessionData& other) const { + // Note that SerializedNavigationEntry::timestamp() is never set to a + // value, so always use SessionTab::timestamp() instead. + // TODO(skym): It might be better if we sorted by recency of session, and + // only then by recency of the tab. Right now this causes a single + // device's tabs to be interleaved with another devices' tabs. + return tab->timestamp > other.tab->timestamp; + } +}; + +ForeignSessionsSuggestionsProvider::ForeignSessionsSuggestionsProvider( + ContentSuggestionsProvider::Observer* observer, + CategoryFactory* category_factory, + std::unique_ptr<ForeignSessionsProvider> foreign_sessions_provider, + PrefService* pref_service) + : ContentSuggestionsProvider(observer, category_factory), + category_status_(CategoryStatus::INITIALIZING), + provided_category_( + category_factory->FromKnownCategory(KnownCategories::FOREIGN_TABS)), + foreign_sessions_provider_(std::move(foreign_sessions_provider)), + pref_service_(pref_service) { + foreign_sessions_provider_->SubscribeForForeignTabChange( + base::Bind(&ForeignSessionsSuggestionsProvider::OnForeignTabChange, + base::Unretained(this))); + + // If sync is already initialzed, try suggesting now, though this is unlikely. + OnForeignTabChange(); +} + +ForeignSessionsSuggestionsProvider::~ForeignSessionsSuggestionsProvider() = + default; + +// static +void ForeignSessionsSuggestionsProvider::RegisterProfilePrefs( + PrefRegistrySimple* registry) { + registry->RegisterListPref(prefs::kDismissedForeignSessionsSuggestions); +} + +CategoryStatus ForeignSessionsSuggestionsProvider::GetCategoryStatus( + Category category) { + DCHECK_EQ(category, provided_category_); + return category_status_; +} + +CategoryInfo ForeignSessionsSuggestionsProvider::GetCategoryInfo( + Category category) { + DCHECK_EQ(category, provided_category_); + return CategoryInfo(l10n_util::GetStringUTF16( + IDS_NTP_FOREIGN_SESSIONS_SUGGESTIONS_SECTION_HEADER), + ContentSuggestionsCardLayout::MINIMAL_CARD, + /*has_more_button=*/true, + /*show_if_empty=*/false); +} + +void ForeignSessionsSuggestionsProvider::DismissSuggestion( + const ContentSuggestion::ID& suggestion_id) { + // TODO(skym): Right now this continuously grows, without clearing out old and + // irrelevant entries. Could either use a timestamp and expire after a + // threshold, or compare with current foreign tabs and remove anything that + // isn't actively blockign a foreign_sessions tab. + std::set<std::string> dismissed_ids = prefs::ReadDismissedIDsFromPrefs( + *pref_service_, prefs::kDismissedForeignSessionsSuggestions); + dismissed_ids.insert(suggestion_id.id_within_category()); + prefs::StoreDismissedIDsToPrefs(pref_service_, + prefs::kDismissedForeignSessionsSuggestions, + dismissed_ids); +} + +void ForeignSessionsSuggestionsProvider::FetchSuggestionImage( + const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) { + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::Bind(callback, gfx::Image())); +} + +void ForeignSessionsSuggestionsProvider::ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) { + std::set<std::string> dismissed_ids = prefs::ReadDismissedIDsFromPrefs( + *pref_service_, prefs::kDismissedForeignSessionsSuggestions); + for (auto iter = dismissed_ids.begin(); iter != dismissed_ids.end();) { + if (filter.Run(GURL(*iter))) { + iter = dismissed_ids.erase(iter); + } else { + ++iter; + } + } + prefs::StoreDismissedIDsToPrefs(pref_service_, + prefs::kDismissedForeignSessionsSuggestions, + dismissed_ids); +} + +void ForeignSessionsSuggestionsProvider::ClearCachedSuggestions( + Category category) { + DCHECK_EQ(category, provided_category_); + // Ignored. +} + +void ForeignSessionsSuggestionsProvider::GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) { + DCHECK_EQ(category, provided_category_); + callback.Run(std::vector<ContentSuggestion>()); +} + +void ForeignSessionsSuggestionsProvider::ClearDismissedSuggestionsForDebugging( + Category category) { + DCHECK_EQ(category, provided_category_); + pref_service_->ClearPref(prefs::kDismissedForeignSessionsSuggestions); +} + +void ForeignSessionsSuggestionsProvider::OnForeignTabChange() { + if (!foreign_sessions_provider_->HasSessionsData()) { + if (category_status_ == CategoryStatus::AVAILABLE) { + // This is to handle the case where the user disabled sync [sessions] or + // logs out after we've already provided actual suggestions. + category_status_ = CategoryStatus::NOT_PROVIDED; + observer()->OnCategoryStatusChanged(this, provided_category_, + category_status_); + } + return; + } + + if (category_status_ != CategoryStatus::AVAILABLE) { + // The further below logic will overwrite any error state. This is + // currently okay because no where in the current implementation does the + // status get set to an error state. Should this change, reconsider the + // overwriting logic. + DCHECK(category_status_ == CategoryStatus::INITIALIZING || + category_status_ == CategoryStatus::NOT_PROVIDED); + + // It is difficult to tell if sync simply has not initialized yet or there + // will never be data because the user is signed out or has disabled the + // sessions data type. Because this provider is hidden when there are no + // results, always just update to AVAILABLE once we might have results. + category_status_ = CategoryStatus::AVAILABLE; + observer()->OnCategoryStatusChanged(this, provided_category_, + category_status_); + } + + // observer()->OnNewSuggestions must be called even when we have no + // suggestions to remove previous suggestions that are now filtered out. + observer()->OnNewSuggestions(this, provided_category_, BuildSuggestions()); +} + +std::vector<ContentSuggestion> +ForeignSessionsSuggestionsProvider::BuildSuggestions() { + const int max_foreign_tabs_total = GetMaxForeignTabsTotal(); + const int max_foreign_tabs_per_device = GetMaxForeignTabsPerDevice(); + + std::vector<SessionData> suggestion_candidates = GetSuggestionCandidates(); + // This sorts by recency so that we keep the most recent entries and they + // appear as suggestions in reverse chronological order. + std::sort(suggestion_candidates.begin(), suggestion_candidates.end()); + + std::vector<ContentSuggestion> suggestions; + std::set<std::string> included_urls; + std::map<std::string, int> suggestions_per_session; + for (const SessionData& candidate : suggestion_candidates) { + const std::string& session_tag = candidate.session->session_tag; + auto duplicates_iter = + included_urls.find(candidate.navigation->virtual_url().spec()); + auto count_iter = suggestions_per_session.find(session_tag); + int count = + count_iter == suggestions_per_session.end() ? 0 : count_iter->second; + + // Pick up to max (total and per device) tabs, and ensure no duplicates + // are selected. This filtering must be done in a second pass because + // this can cause newer tabs occluding less recent tabs, requiring more + // than |max_foreign_tabs_per_device| to be considered per device. + if (static_cast<int>(suggestions.size()) >= max_foreign_tabs_total || + duplicates_iter != included_urls.end() || + count >= max_foreign_tabs_per_device) { + continue; + } + included_urls.insert(candidate.navigation->virtual_url().spec()); + suggestions_per_session[session_tag] = count + 1; + suggestions.push_back(BuildSuggestion(candidate)); + } + + return suggestions; +} + +std::vector<ForeignSessionsSuggestionsProvider::SessionData> +ForeignSessionsSuggestionsProvider::GetSuggestionCandidates() { + // TODO(skym): If a tab was previously dismissed, but was since updated, + // should it be resurrected and removed from the dismissed list? This would + // likely require a change to the dismissed ids. + // TODO(skym): No sense in keeping around dismissals for urls that no longer + // exist on any current foreign devices. Should prune and save the pref back. + const std::vector<const SyncedSession*>& foreign_sessions = + foreign_sessions_provider_->GetAllForeignSessions(); + std::set<std::string> dismissed_ids = prefs::ReadDismissedIDsFromPrefs( + *pref_service_, prefs::kDismissedForeignSessionsSuggestions); + const TimeDelta max_foreign_tab_age = GetMaxForeignTabAge(); + std::vector<SessionData> suggestion_candidates; + + for (const SyncedSession* session : foreign_sessions) { + for (const std::pair<const SessionID::id_type, + std::unique_ptr<sessions::SessionWindow>>& key_value : + session->windows) { + for (const std::unique_ptr<SessionTab>& tab : key_value.second->tabs) { + if (tab->navigations.empty()) + continue; + + const SerializedNavigationEntry& navigation = tab->navigations.back(); + const std::string id = navigation.virtual_url().spec(); + // TODO(skym): Filter out internal pages. Tabs that contain only + // non-syncable content should never reach the local client, but + // sometimes the most recent navigation may be internal while one + // of the previous ones was more valid. + if (dismissed_ids.find(id) == dismissed_ids.end() && + (base::Time::Now() - tab->timestamp) < max_foreign_tab_age) { + suggestion_candidates.push_back( + SessionData{session, tab.get(), &navigation}); + } + } + } + } + return suggestion_candidates; +} + +ContentSuggestion ForeignSessionsSuggestionsProvider::BuildSuggestion( + const SessionData& data) { + ContentSuggestion suggestion(provided_category_, + data.navigation->virtual_url().spec(), + data.navigation->virtual_url()); + suggestion.set_title(data.navigation->title()); + suggestion.set_publish_date(data.tab->timestamp); + // TODO(skym): It's unclear if this simple approach is sufficient for + // right-to-left languages. + // This field is sandwiched between the url's favicon, which is on the left, + // and the |publish_date|, which is to the right. The domain should always + // appear next to the favicon. + suggestion.set_publisher_name( + base::UTF8ToUTF16(data.navigation->virtual_url().host() + " - " + + data.session->session_name)); + return suggestion; +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/sessions/foreign_sessions_suggestions_provider.h b/chromium/components/ntp_snippets/sessions/foreign_sessions_suggestions_provider.h new file mode 100644 index 00000000000..4d21a3f9157 --- /dev/null +++ b/chromium/components/ntp_snippets/sessions/foreign_sessions_suggestions_provider.h @@ -0,0 +1,85 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_SESSIONS_FOREIGN_SESSIONS_SUGGESTIONS_PROVIDER_H_ +#define COMPONENTS_NTP_SNIPPETS_SESSIONS_FOREIGN_SESSIONS_SUGGESTIONS_PROVIDER_H_ + +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "base/callback_forward.h" +#include "base/macros.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_status.h" +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "components/sessions/core/session_types.h" +#include "components/sync_sessions/synced_session.h" + +class PrefRegistrySimple; +class PrefService; + +namespace ntp_snippets { + +// Simple interface to get foreign tab data on demand and on change. +class ForeignSessionsProvider { + public: + virtual ~ForeignSessionsProvider() = default; + virtual bool HasSessionsData() = 0; + virtual std::vector<const sync_sessions::SyncedSession*> + GetAllForeignSessions() = 0; + // Should only be called at most once. + virtual void SubscribeForForeignTabChange( + const base::Closure& change_callback) = 0; +}; + +// Provides content suggestions from foreign sessions. +class ForeignSessionsSuggestionsProvider : public ContentSuggestionsProvider { + public: + ForeignSessionsSuggestionsProvider( + ContentSuggestionsProvider::Observer* observer, + CategoryFactory* category_factory, + std::unique_ptr<ForeignSessionsProvider> foreign_sessions_provider, + PrefService* pref_service); + ~ForeignSessionsSuggestionsProvider() override; + + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + private: + friend class ForeignSessionsSuggestionsProviderTest; + struct SessionData; + + // ContentSuggestionsProvider implementation. + CategoryStatus GetCategoryStatus(Category category) override; + CategoryInfo GetCategoryInfo(Category category) override; + void DismissSuggestion(const ContentSuggestion::ID& suggestion_id) override; + void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id, + const ImageFetchedCallback& callback) override; + void ClearHistory( + base::Time begin, + base::Time end, + const base::Callback<bool(const GURL& url)>& filter) override; + void ClearCachedSuggestions(Category category) override; + void GetDismissedSuggestionsForDebugging( + Category category, + const DismissedSuggestionsCallback& callback) override; + void ClearDismissedSuggestionsForDebugging(Category category) override; + + void OnForeignTabChange(); + std::vector<ContentSuggestion> BuildSuggestions(); + std::vector<SessionData> GetSuggestionCandidates(); + ContentSuggestion BuildSuggestion(const SessionData& data); + + CategoryStatus category_status_; + const Category provided_category_; + std::unique_ptr<ForeignSessionsProvider> foreign_sessions_provider_; + PrefService* pref_service_; + + DISALLOW_COPY_AND_ASSIGN(ForeignSessionsSuggestionsProvider); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_SESSIONS_FOREIGN_SESSIONS_SUGGESTIONS_PROVIDER_H_ diff --git a/chromium/components/ntp_snippets/sessions/foreign_sessions_suggestions_provider_unittest.cc b/chromium/components/ntp_snippets/sessions/foreign_sessions_suggestions_provider_unittest.cc new file mode 100644 index 00000000000..e36cf2351d8 --- /dev/null +++ b/chromium/components/ntp_snippets/sessions/foreign_sessions_suggestions_provider_unittest.cc @@ -0,0 +1,326 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/sessions/foreign_sessions_suggestions_provider.h" + +#include <map> +#include <utility> + +#include "base/callback_forward.h" +#include "base/memory/ptr_util.h" +#include "base/strings/string_number_conversions.h" +#include "components/ntp_snippets/category.h" +#include "components/ntp_snippets/category_factory.h" +#include "components/ntp_snippets/content_suggestions_provider.h" +#include "components/ntp_snippets/mock_content_suggestions_provider_observer.h" +#include "components/prefs/testing_pref_service.h" +#include "components/sessions/core/serialized_navigation_entry.h" +#include "components/sessions/core/serialized_navigation_entry_test_helper.h" +#include "components/sessions/core/session_types.h" +#include "components/sync_sessions/synced_session.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using base::Time; +using base::TimeDelta; +using sessions::SerializedNavigationEntry; +using sessions::SessionTab; +using sessions::SessionWindow; +using sync_sessions::SyncedSession; +using testing::ElementsAre; +using testing::IsEmpty; +using testing::Property; +using testing::Test; +using testing::_; + +namespace ntp_snippets { +namespace { + +const char kUrl1[] = "http://www.fake1.com/"; +const char kUrl2[] = "http://www.fake2.com/"; +const char kUrl3[] = "http://www.fake3.com/"; +const char kUrl4[] = "http://www.fake4.com/"; +const char kUrl5[] = "http://www.fake5.com/"; +const char kUrl6[] = "http://www.fake6.com/"; +const char kUrl7[] = "http://www.fake7.com/"; +const char kUrl8[] = "http://www.fake8.com/"; +const char kUrl9[] = "http://www.fake9.com/"; +const char kUrl10[] = "http://www.fake10.com/"; +const char kUrl11[] = "http://www.fake11.com/"; +const char kTitle[] = "title is ignored"; + +SessionWindow* GetOrCreateWindow(SyncedSession* session, int window_id) { + if (session->windows.find(window_id) == session->windows.end()) + session->windows[window_id] = base::MakeUnique<SessionWindow>(); + + return session->windows[window_id].get(); +} + +void AddTabToSession(SyncedSession* session, + int window_id, + const std::string& url, + TimeDelta age) { + SerializedNavigationEntry navigation = + sessions::SerializedNavigationEntryTestHelper::CreateNavigation(url, + kTitle); + + std::unique_ptr<SessionTab> tab = base::MakeUnique<SessionTab>(); + tab->timestamp = Time::Now() - age; + tab->navigations.push_back(navigation); + + SessionWindow* window = GetOrCreateWindow(session, window_id); + // The window deletes the tabs it points at upon destruction. + window->tabs.push_back(std::move(tab)); +} + +class FakeForeignSessionsProvider : public ForeignSessionsProvider { + public: + ~FakeForeignSessionsProvider() override = default; + void SetAllForeignSessions(std::vector<const SyncedSession*> sessions) { + sessions_ = std::move(sessions); + change_callback_.Run(); + } + + // ForeignSessionsProvider implementation. + void SubscribeForForeignTabChange( + const base::Closure& change_callback) override { + change_callback_ = change_callback; + } + bool HasSessionsData() override { return true; } + std::vector<const sync_sessions::SyncedSession*> GetAllForeignSessions() + override { + return sessions_; + } + + private: + std::vector<const SyncedSession*> sessions_; + base::Closure change_callback_; +}; +} // namespace + +class ForeignSessionsSuggestionsProviderTest : public Test { + public: + ForeignSessionsSuggestionsProviderTest() { + ForeignSessionsSuggestionsProvider::RegisterProfilePrefs( + pref_service_.registry()); + + std::unique_ptr<FakeForeignSessionsProvider> + fake_foreign_sessions_provider = + base::MakeUnique<FakeForeignSessionsProvider>(); + fake_foreign_sessions_provider_ = fake_foreign_sessions_provider.get(); + + // During the provider's construction the following mock calls occur. + EXPECT_CALL(*observer(), OnNewSuggestions(_, category(), IsEmpty())); + EXPECT_CALL(*observer(), OnCategoryStatusChanged( + _, category(), CategoryStatus::AVAILABLE)); + + provider_ = base::MakeUnique<ForeignSessionsSuggestionsProvider>( + &observer_, &category_factory_, + std::move(fake_foreign_sessions_provider), &pref_service_); + } + + protected: + SyncedSession* GetOrCreateSession(int session_id) { + if (sessions_map_.find(session_id) == sessions_map_.end()) { + std::string id_as_string = base::IntToString(session_id); + std::unique_ptr<SyncedSession> owned_session = + base::MakeUnique<SyncedSession>(); + owned_session->session_tag = id_as_string; + owned_session->session_name = id_as_string; + sessions_map_[session_id] = std::move(owned_session); + } + return sessions_map_[session_id].get(); + } + + void AddTab(int session_id, + int window_id, + const std::string& url, + TimeDelta age) { + AddTabToSession(GetOrCreateSession(session_id), window_id, url, age); + } + + void TriggerOnChange() { + std::vector<const SyncedSession*> sessions; + for (const auto& kv : sessions_map_) { + sessions.push_back(kv.second.get()); + } + fake_foreign_sessions_provider_->SetAllForeignSessions(std::move(sessions)); + } + + void Dismiss(const std::string& url) { + // The url of a given suggestion is used as the |id_within_category|. + provider_->DismissSuggestion(ContentSuggestion::ID(category(), url)); + } + + Category category() { + return category_factory_.FromKnownCategory(KnownCategories::FOREIGN_TABS); + } + + MockContentSuggestionsProviderObserver* observer() { return &observer_; } + + private: + FakeForeignSessionsProvider* fake_foreign_sessions_provider_; + MockContentSuggestionsProviderObserver observer_; + CategoryFactory category_factory_; + TestingPrefServiceSimple pref_service_; + std::unique_ptr<ForeignSessionsSuggestionsProvider> provider_; + std::map<int, std::unique_ptr<SyncedSession>> sessions_map_; + + DISALLOW_COPY_AND_ASSIGN(ForeignSessionsSuggestionsProviderTest); +}; + +TEST_F(ForeignSessionsSuggestionsProviderTest, Empty) { + // When no sessions data is added, expect no suggestions. + EXPECT_CALL(*observer(), OnNewSuggestions(_, category(), IsEmpty())); + TriggerOnChange(); +} + +TEST_F(ForeignSessionsSuggestionsProviderTest, Single) { + // Expect a single valid tab because that is what has been added. + EXPECT_CALL(*observer(), + OnNewSuggestions( + _, category(), + ElementsAre(Property(&ContentSuggestion::url, GURL(kUrl1))))); + AddTab(0, 0, kUrl1, TimeDelta::FromMinutes(1)); + TriggerOnChange(); +} + +TEST_F(ForeignSessionsSuggestionsProviderTest, Old) { + // The only sessions data is too old to be suggested, so expect empty. + EXPECT_CALL(*observer(), OnNewSuggestions(_, category(), IsEmpty())); + AddTab(0, 0, kUrl1, TimeDelta::FromHours(4)); + TriggerOnChange(); +} + +TEST_F(ForeignSessionsSuggestionsProviderTest, Ordered) { + // Suggestions ordering should be in reverse chronological order, or youngest + // first. + EXPECT_CALL(*observer(), + OnNewSuggestions( + _, category(), + ElementsAre(Property(&ContentSuggestion::url, GURL(kUrl1)), + Property(&ContentSuggestion::url, GURL(kUrl2)), + Property(&ContentSuggestion::url, GURL(kUrl3)), + Property(&ContentSuggestion::url, GURL(kUrl4))))); + AddTab(0, 0, kUrl2, TimeDelta::FromMinutes(2)); + AddTab(0, 0, kUrl4, TimeDelta::FromMinutes(4)); + AddTab(0, 1, kUrl3, TimeDelta::FromMinutes(3)); + AddTab(1, 0, kUrl1, TimeDelta::FromMinutes(1)); + TriggerOnChange(); +} + +TEST_F(ForeignSessionsSuggestionsProviderTest, MaxPerDevice) { + // Each device, which is to equivalent a unique |session_tag|, has a limit to + // the number of suggestions it is allowed to contribute. Here all four + // suggestions are within the recency threshold, but only three are allowed + // per device. As such, expect that the oldest of the four will not be + // suggested. + EXPECT_CALL(*observer(), + OnNewSuggestions( + _, category(), + ElementsAre(Property(&ContentSuggestion::url, GURL(kUrl1)), + Property(&ContentSuggestion::url, GURL(kUrl2)), + Property(&ContentSuggestion::url, GURL(kUrl3))))); + AddTab(0, 0, kUrl1, TimeDelta::FromMinutes(1)); + AddTab(0, 0, kUrl2, TimeDelta::FromMinutes(2)); + AddTab(0, 0, kUrl3, TimeDelta::FromMinutes(3)); + AddTab(0, 0, kUrl4, TimeDelta::FromMinutes(4)); + TriggerOnChange(); +} + +TEST_F(ForeignSessionsSuggestionsProviderTest, MaxTotal) { + // There's a limit to the total nubmer of suggestions that the provider will + // ever return, which should be ten. Here there are eleven valid suggestion + // entries, spread out over multiple devices/sessions to avoid the per device + // cutoff. Expect that the least recent of the eleven to be dropped. + EXPECT_CALL( + *observer(), + OnNewSuggestions( + _, category(), + ElementsAre(Property(&ContentSuggestion::url, GURL(kUrl1)), + Property(&ContentSuggestion::url, GURL(kUrl2)), + Property(&ContentSuggestion::url, GURL(kUrl3)), + Property(&ContentSuggestion::url, GURL(kUrl4)), + Property(&ContentSuggestion::url, GURL(kUrl5)), + Property(&ContentSuggestion::url, GURL(kUrl6)), + Property(&ContentSuggestion::url, GURL(kUrl7)), + Property(&ContentSuggestion::url, GURL(kUrl8)), + Property(&ContentSuggestion::url, GURL(kUrl9)), + Property(&ContentSuggestion::url, GURL(kUrl10))))); + AddTab(0, 0, kUrl1, TimeDelta::FromMinutes(1)); + AddTab(0, 0, kUrl2, TimeDelta::FromMinutes(2)); + AddTab(0, 0, kUrl3, TimeDelta::FromMinutes(3)); + AddTab(1, 0, kUrl4, TimeDelta::FromMinutes(4)); + AddTab(1, 0, kUrl5, TimeDelta::FromMinutes(5)); + AddTab(1, 0, kUrl6, TimeDelta::FromMinutes(6)); + AddTab(2, 0, kUrl7, TimeDelta::FromMinutes(7)); + AddTab(2, 0, kUrl8, TimeDelta::FromMinutes(8)); + AddTab(2, 0, kUrl9, TimeDelta::FromMinutes(9)); + AddTab(3, 0, kUrl10, TimeDelta::FromMinutes(10)); + AddTab(3, 0, kUrl11, TimeDelta::FromMinutes(11)); + TriggerOnChange(); +} + +TEST_F(ForeignSessionsSuggestionsProviderTest, Duplicates) { + // The same url is never suggested more than once at a time. All the session + // data has the same url so only expect a single suggestion. + EXPECT_CALL(*observer(), + OnNewSuggestions( + _, category(), + ElementsAre(Property(&ContentSuggestion::url, GURL(kUrl1))))); + AddTab(0, 0, kUrl1, TimeDelta::FromMinutes(1)); + AddTab(0, 1, kUrl1, TimeDelta::FromMinutes(2)); + AddTab(1, 1, kUrl1, TimeDelta::FromMinutes(3)); + TriggerOnChange(); +} + +TEST_F(ForeignSessionsSuggestionsProviderTest, DuplicatesChangingOtherSession) { + // Normally |kUrl4| wouldn't show up, because session_id=0 already has 3 + // younger tabs, but session_id=1 has a younger |kUrl3| which gives |kUrl4| a + // spot. + EXPECT_CALL(*observer(), + OnNewSuggestions( + _, category(), + ElementsAre(Property(&ContentSuggestion::url, GURL(kUrl3)), + Property(&ContentSuggestion::url, GURL(kUrl1)), + Property(&ContentSuggestion::url, GURL(kUrl2)), + Property(&ContentSuggestion::url, GURL(kUrl4))))); + AddTab(0, 0, kUrl1, TimeDelta::FromMinutes(1)); + AddTab(0, 0, kUrl2, TimeDelta::FromMinutes(2)); + AddTab(0, 0, kUrl3, TimeDelta::FromMinutes(3)); + AddTab(0, 0, kUrl4, TimeDelta::FromMinutes(4)); + AddTab(1, 0, kUrl3, TimeDelta::FromMinutes(0)); + TriggerOnChange(); +} + +TEST_F(ForeignSessionsSuggestionsProviderTest, Dismissed) { + // Dimissed urls should not be suggested. + EXPECT_CALL(*observer(), OnNewSuggestions(_, category(), IsEmpty())); + Dismiss(kUrl1); + AddTab(0, 0, kUrl1, TimeDelta::FromMinutes(1)); + TriggerOnChange(); +} + +TEST_F(ForeignSessionsSuggestionsProviderTest, DismissedChangingOwnSession) { + // Similar to DuplicatesChangingOtherSession, without dismissals we would + // expect urls 1-3. However, because of dismissals we reach all the down to + // |kUrl5| before the per device cutoff is hit. + EXPECT_CALL(*observer(), + OnNewSuggestions( + _, category(), + ElementsAre(Property(&ContentSuggestion::url, GURL(kUrl2)), + Property(&ContentSuggestion::url, GURL(kUrl3)), + Property(&ContentSuggestion::url, GURL(kUrl5))))); + Dismiss(kUrl1); + Dismiss(kUrl4); + AddTab(0, 0, kUrl1, TimeDelta::FromMinutes(1)); + AddTab(0, 0, kUrl2, TimeDelta::FromMinutes(2)); + AddTab(0, 0, kUrl3, TimeDelta::FromMinutes(3)); + AddTab(0, 0, kUrl4, TimeDelta::FromMinutes(4)); + AddTab(0, 0, kUrl5, TimeDelta::FromMinutes(5)); + AddTab(0, 0, kUrl6, TimeDelta::FromMinutes(6)); + TriggerOnChange(); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/sessions/tab_delegate_sync_adapter.cc b/chromium/components/ntp_snippets/sessions/tab_delegate_sync_adapter.cc new file mode 100644 index 00000000000..aa0626cbfd4 --- /dev/null +++ b/chromium/components/ntp_snippets/sessions/tab_delegate_sync_adapter.cc @@ -0,0 +1,64 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/sessions/tab_delegate_sync_adapter.h" + +#include "components/sync/driver/sync_service.h" +#include "components/sync_sessions/open_tabs_ui_delegate.h" + +using syncer::SyncService; +using sync_sessions::OpenTabsUIDelegate; + +namespace ntp_snippets { + +TabDelegateSyncAdapter::TabDelegateSyncAdapter(SyncService* sync_service) + : sync_service_(sync_service) { + sync_service_->AddObserver(this); +} + +TabDelegateSyncAdapter::~TabDelegateSyncAdapter() { + sync_service_->RemoveObserver(this); +} + +bool TabDelegateSyncAdapter::HasSessionsData() { + // GetOpenTabsUIDelegate will be a nullptr if sync has not started, or if the + // sessions data type is not enabled. + return sync_service_->GetOpenTabsUIDelegate() != nullptr; +} + +std::vector<const sync_sessions::SyncedSession*> +TabDelegateSyncAdapter::GetAllForeignSessions() { + std::vector<const sync_sessions::SyncedSession*> sessions; + OpenTabsUIDelegate* delegate = sync_service_->GetOpenTabsUIDelegate(); + if (delegate != nullptr) { + // The return bool from GetAllForeignSessions(...) is ignored. + delegate->GetAllForeignSessions(&sessions); + } + return sessions; +} + +void TabDelegateSyncAdapter::SubscribeForForeignTabChange( + const base::Closure& change_callback) { + DCHECK(change_callback_.is_null()); + change_callback_ = change_callback; +} + +void TabDelegateSyncAdapter::OnStateChanged() { + // Ignored. +} + +void TabDelegateSyncAdapter::OnSyncConfigurationCompleted() { + InvokeCallback(); +} + +void TabDelegateSyncAdapter::OnForeignSessionUpdated() { + InvokeCallback(); +} + +void TabDelegateSyncAdapter::InvokeCallback() { + if (!change_callback_.is_null()) + change_callback_.Run(); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/sessions/tab_delegate_sync_adapter.h b/chromium/components/ntp_snippets/sessions/tab_delegate_sync_adapter.h new file mode 100644 index 00000000000..8796444c6fe --- /dev/null +++ b/chromium/components/ntp_snippets/sessions/tab_delegate_sync_adapter.h @@ -0,0 +1,53 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_SESSIONS_TAB_DELEGATE_SYNC_ADAPTER_H_ +#define COMPONENTS_NTP_SNIPPETS_SESSIONS_TAB_DELEGATE_SYNC_ADAPTER_H_ + +#include <vector> + +#include "base/callback.h" +#include "base/callback_forward.h" +#include "base/macros.h" +#include "components/ntp_snippets/sessions/foreign_sessions_suggestions_provider.h" +#include "components/sync/driver/sync_service_observer.h" + +namespace syncer { +class SyncService; +} // namespace syncer + +namespace ntp_snippets { + +// Adapter that sits on top of SyncService and OpenTabsUIDelegate and provides +// simplified notifications and accessors for foreign tabs data. +class TabDelegateSyncAdapter : public syncer::SyncServiceObserver, + public ForeignSessionsProvider { + public: + explicit TabDelegateSyncAdapter(syncer::SyncService* sync_service); + ~TabDelegateSyncAdapter() override; + + // ForeignSessionsProvider implementation. + bool HasSessionsData() override; + std::vector<const sync_sessions::SyncedSession*> GetAllForeignSessions() + override; + void SubscribeForForeignTabChange( + const base::Closure& change_callback) override; + + private: + // syncer::SyncServiceObserver implementation. + void OnStateChanged() override; + void OnSyncConfigurationCompleted() override; + void OnForeignSessionUpdated() override; + + void InvokeCallback(); + + syncer::SyncService* sync_service_; + base::Closure change_callback_; + + DISALLOW_COPY_AND_ASSIGN(TabDelegateSyncAdapter); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_SESSIONS_TAB_DELEGATE_SYNC_ADAPTER_H_ diff --git a/chromium/components/ntp_snippets/switches.cc b/chromium/components/ntp_snippets/switches.cc index ee5f979e112..e34431c934b 100644 --- a/chromium/components/ntp_snippets/switches.cc +++ b/chromium/components/ntp_snippets/switches.cc @@ -7,17 +7,6 @@ namespace ntp_snippets { namespace switches { -const char kFetchingIntervalWifiChargingSeconds[] = - "ntp-snippets-fetching-interval-wifi-charging"; -const char kFetchingIntervalWifiSeconds[] = - "ntp-snippets-fetching-interval-wifi"; -const char kFetchingIntervalFallbackSeconds[] = - "ntp-snippets-fetching-interval-fallback"; - -// If this flag is set, the snippets won't be restricted to the user's NTP -// suggestions. -const char kDontRestrict[] = "ntp-snippets-dont-restrict"; - // If this flag is set, we will add downloaded snippets that are missing some // critical data to the list. const char kAddIncompleteSnippets[] = "ntp-snippets-add-incomplete"; diff --git a/chromium/components/ntp_snippets/switches.h b/chromium/components/ntp_snippets/switches.h index 4897f136ec8..a48e8aff680 100644 --- a/chromium/components/ntp_snippets/switches.h +++ b/chromium/components/ntp_snippets/switches.h @@ -8,11 +8,6 @@ namespace ntp_snippets { namespace switches { -extern const char kFetchingIntervalWifiChargingSeconds[]; -extern const char kFetchingIntervalWifiSeconds[]; -extern const char kFetchingIntervalFallbackSeconds[]; - -extern const char kDontRestrict[]; extern const char kAddIncompleteSnippets[]; } // namespace switches diff --git a/chromium/components/ntp_snippets/user_classifier.cc b/chromium/components/ntp_snippets/user_classifier.cc new file mode 100644 index 00000000000..a9122343a1e --- /dev/null +++ b/chromium/components/ntp_snippets/user_classifier.cc @@ -0,0 +1,373 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/ntp_snippets/user_classifier.h" + +#include <algorithm> +#include <cfloat> +#include <string> + +#include "base/metrics/histogram_macros.h" +#include "base/strings/string_number_conversions.h" +#include "components/ntp_snippets/features.h" +#include "components/ntp_snippets/pref_names.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" +#include "components/variations/variations_associated_data.h" + +namespace ntp_snippets { + +namespace { + +// The discount rate for computing the discounted-average metrics. Must be +// strictly larger than 0 and strictly smaller than 1! +const double kDiscountRatePerDay = 0.25; +const char kDiscountRatePerDayParam[] = + "user_classifier_discount_rate_per_day"; + +// Never consider any larger interval than this (so that extreme situations such +// as losing your phone or going for a long offline vacation do not skew the +// average too much). +// When everriding via variation parameters, it is better to use smaller values +// than |kMaxHours| as this it the maximum value reported in the histograms. +const double kMaxHours = 7 * 24; +const char kMaxHoursParam[] = "user_classifier_max_hours"; + +// Ignore events within |kMinHours| hours since the last event (|kMinHours| is +// the length of the browsing session where subsequent events of the same type +// do not count again). +const double kMinHours = 0.5; +const char kMinHoursParam[] = "user_classifier_min_hours"; + +// Classification constants. +const double kActiveConsumerScrollsAtLeastOncePerHours = 24; +const char kActiveConsumerScrollsAtLeastOncePerHoursParam[] = + "user_classifier_active_consumer_scrolls_at_least_once_per_hours"; + +const double kRareUserOpensNTPAtMostOncePerHours = 72; +const char kRareUserOpensNTPAtMostOncePerHoursParam[] = + "user_classifier_rare_user_opens_ntp_at_most_once_per_hours"; + +// Histograms for logging the estimated average hours to next event. +const char kHistogramAverageHoursToOpenNTP[] = + "NewTabPage.UserClassifier.AverageHoursToOpenNTP"; +const char kHistogramAverageHoursToShowSuggestions[] = + "NewTabPage.UserClassifier.AverageHoursToShowSuggestions"; +const char kHistogramAverageHoursToUseSuggestions[] = + "NewTabPage.UserClassifier.AverageHoursToUseSuggestions"; + +// The enum used for iteration. +const UserClassifier::Metric kMetrics[] = { + UserClassifier::Metric::NTP_OPENED, + UserClassifier::Metric::SUGGESTIONS_SHOWN, + UserClassifier::Metric::SUGGESTIONS_USED}; + +// The summary of the prefs. +const char* kMetricKeys[] = { + prefs::kUserClassifierAverageNTPOpenedPerHour, + prefs::kUserClassifierAverageSuggestionsShownPerHour, + prefs::kUserClassifierAverageSuggestionsUsedPerHour}; +const char* kLastTimeKeys[] = {prefs::kUserClassifierLastTimeToOpenNTP, + prefs::kUserClassifierLastTimeToShowSuggestions, + prefs::kUserClassifierLastTimeToUseSuggestions}; + +// Default lengths of the intervals for new users for the metrics. +const double kInitialHoursBetweenEvents[] = {24, 36, 48}; +const char* kInitialHoursBetweenEventsParams[] = { + "user_classifier_default_interval_ntp_opened", + "user_classifier_default_interval_suggestions_shown", + "user_classifier_default_interval_suggestions_used"}; + +static_assert(arraysize(kMetrics) == + static_cast<int>(UserClassifier::Metric::COUNT) && + arraysize(kMetricKeys) == + static_cast<int>(UserClassifier::Metric::COUNT) && + arraysize(kLastTimeKeys) == + static_cast<int>(UserClassifier::Metric::COUNT) && + arraysize(kInitialHoursBetweenEvents) == + static_cast<int>(UserClassifier::Metric::COUNT) && + arraysize(kInitialHoursBetweenEventsParams) == + static_cast<int>(UserClassifier::Metric::COUNT), + "Fill in info for all metrics."); + +double GetParamValue(const char* param_name, double default_value) { + std::string param_value_str = variations::GetVariationParamValueByFeature( + kArticleSuggestionsFeature, param_name); + double param_value = 0; + if (!base::StringToDouble(param_value_str, ¶m_value)) { + LOG_IF(WARNING, !param_value_str.empty()) + << "Invalid variation parameter for " << param_name; + return default_value; + } + return param_value; +} + +// Computes the discount rate. +double GetDiscountRatePerHour() { + double discount_rate_per_day = + GetParamValue(kDiscountRatePerDayParam, kDiscountRatePerDay); + // Check for illegal values. + if (discount_rate_per_day <= 0 || discount_rate_per_day >= 1) { + DLOG(WARNING) << "Illegal value " << discount_rate_per_day + << " for the parameter " << kDiscountRatePerDayParam + << " (must be strictly between 0 and 1; the default " + << kDiscountRatePerDay << " is used, instead)."; + discount_rate_per_day = kDiscountRatePerDay; + } + // Compute discount_rate_per_hour such that + // discount_rate_per_day = 1 - e^{-discount_rate_per_hour * 24}. + return std::log(1.0 / (1.0 - discount_rate_per_day)) / 24.0; +} + +double GetInitialHoursBetweenEvents(UserClassifier::Metric metric) { + return GetParamValue( + kInitialHoursBetweenEventsParams[static_cast<int>(metric)], + kInitialHoursBetweenEvents[static_cast<int>(metric)]); +} + +double GetMinHours() { + return GetParamValue(kMinHoursParam, kMinHours); +} + +double GetMaxHours() { + return GetParamValue(kMaxHoursParam, kMaxHours); +} + +// Returns the new value of the metric using its |old_value|, assuming +// |hours_since_last_time| hours have passed since it was last discounted. +double DiscountMetric(double old_value, + double hours_since_last_time, + double discount_rate_per_hour) { + // Compute the new discounted average according to the formula + // avg_events := e^{-discount_rate_per_hour * hours_since} * avg_events + return std::exp(-discount_rate_per_hour * hours_since_last_time) * old_value; +} + +// Compute the number of hours between two events for the given metric value +// assuming the events were equally distributed. +double GetEstimateHoursBetweenEvents(double metric_value, + double discount_rate_per_hour, + double min_hours, + double max_hours) { + // The computation below is well-defined only for |metric_value| > 1 (log of + // negative value or division by zero). When |metric_value| -> 1, the estimate + // below -> infinity, so max_hours is a natural result, here. + if (metric_value <= 1) + return max_hours; + + // This is the estimate with the assumption that last event happened right + // now and the system is in the steady-state. Solve estimate_hours in the + // steady-state equation: + // metric_value = 1 + e^{-discount_rate * estimate_hours} * metric_value, + // i.e. + // -discount_rate * estimate_hours = log((metric_value - 1) / metric_value), + // discount_rate * estimate_hours = log(metric_value / (metric_value - 1)), + // estimate_hours = log(metric_value / (metric_value - 1)) / discount_rate. + double estimate_hours = + std::log(metric_value / (metric_value - 1)) / discount_rate_per_hour; + return std::max(min_hours, std::min(max_hours, estimate_hours)); +} + +// The inverse of GetEstimateHoursBetweenEvents(). +double GetMetricValueForEstimateHoursBetweenEvents( + double estimate_hours, + double discount_rate_per_hour, + double min_hours, + double max_hours) { + // Keep the input value within [min_hours, max_hours]. + estimate_hours = std::max(min_hours, std::min(max_hours, estimate_hours)); + // Return |metric_value| such that GetEstimateHoursBetweenEvents for + // |metric_value| returns |estimate_hours|. Thus, solve |metric_value| in + // metric_value = 1 + e^{-discount_rate * estimate_hours} * metric_value, + // i.e. + // metric_value * (1 - e^{-discount_rate * estimate_hours}) = 1, + // metric_value = 1 / (1 - e^{-discount_rate * estimate_hours}). + return 1.0 / (1.0 - std::exp(-discount_rate_per_hour * estimate_hours)); +} + +} // namespace + +UserClassifier::UserClassifier(PrefService* pref_service) + : pref_service_(pref_service), + discount_rate_per_hour_(GetDiscountRatePerHour()), + min_hours_(GetMinHours()), + max_hours_(GetMaxHours()), + active_consumer_scrolls_at_least_once_per_hours_( + GetParamValue(kActiveConsumerScrollsAtLeastOncePerHoursParam, + kActiveConsumerScrollsAtLeastOncePerHours)), + rare_user_opens_ntp_at_most_once_per_hours_( + GetParamValue(kRareUserOpensNTPAtMostOncePerHoursParam, + kRareUserOpensNTPAtMostOncePerHours)) { + // The pref_service_ can be null in tests. + if (!pref_service_) + return; + + // TODO(jkrcal): Store the current discount rate per hour into prefs. If it + // differs from the previous value, rescale the metric values so that the + // expectation does not change abruptly! + + // Initialize the prefs storing the last time: the counter has just started! + for (const Metric metric : kMetrics) { + if (!HasLastTime(metric)) + SetLastTimeToNow(metric); + } +} + +UserClassifier::~UserClassifier() = default; + +// static +void UserClassifier::RegisterProfilePrefs(PrefRegistrySimple* registry) { + double discount_rate = GetDiscountRatePerHour(); + double min_hours = GetMinHours(); + double max_hours = GetMaxHours(); + + for (Metric metric : kMetrics) { + double default_metric_value = GetMetricValueForEstimateHoursBetweenEvents( + GetInitialHoursBetweenEvents(metric), discount_rate, min_hours, + max_hours); + registry->RegisterDoublePref(kMetricKeys[static_cast<int>(metric)], + default_metric_value); + registry->RegisterInt64Pref(kLastTimeKeys[static_cast<int>(metric)], 0); + } +} + +void UserClassifier::OnEvent(Metric metric) { + DCHECK_NE(metric, Metric::COUNT); + double metric_value = UpdateMetricOnEvent(metric); + + double avg = GetEstimateHoursBetweenEvents( + metric_value, discount_rate_per_hour_, min_hours_, max_hours_); + // We use kMaxHours as the max value below as the maximum value for the + // histograms must be constant. + switch (metric) { + case Metric::NTP_OPENED: + UMA_HISTOGRAM_CUSTOM_COUNTS(kHistogramAverageHoursToOpenNTP, avg, 1, + kMaxHours, 50); + break; + case Metric::SUGGESTIONS_SHOWN: + UMA_HISTOGRAM_CUSTOM_COUNTS(kHistogramAverageHoursToShowSuggestions, avg, + 1, kMaxHours, 50); + break; + case Metric::SUGGESTIONS_USED: + UMA_HISTOGRAM_CUSTOM_COUNTS(kHistogramAverageHoursToUseSuggestions, avg, + 1, kMaxHours, 50); + break; + case Metric::COUNT: + NOTREACHED(); + break; + } +} + +double UserClassifier::GetEstimatedAvgTime(Metric metric) const { + DCHECK_NE(metric, Metric::COUNT); + double metric_value = GetUpToDateMetricValue(metric); + return GetEstimateHoursBetweenEvents(metric_value, discount_rate_per_hour_, + min_hours_, max_hours_); +} + +UserClassifier::UserClass UserClassifier::GetUserClass() const { + if (GetEstimatedAvgTime(Metric::NTP_OPENED) >= + rare_user_opens_ntp_at_most_once_per_hours_) { + return UserClass::RARE_NTP_USER; + } + + if (GetEstimatedAvgTime(Metric::SUGGESTIONS_SHOWN) <= + active_consumer_scrolls_at_least_once_per_hours_) { + return UserClass::ACTIVE_SUGGESTIONS_CONSUMER; + } + + return UserClass::ACTIVE_NTP_USER; +} + +std::string UserClassifier::GetUserClassDescriptionForDebugging() const { + switch (GetUserClass()) { + case UserClass::RARE_NTP_USER: + return "Rare user of the NTP"; + case UserClass::ACTIVE_NTP_USER: + return "Active user of the NTP"; + case UserClass::ACTIVE_SUGGESTIONS_CONSUMER: + return "Active consumer of NTP suggestions"; + } + NOTREACHED(); + return std::string(); +} + +void UserClassifier::ClearClassificationForDebugging() { + // The pref_service_ can be null in tests. + if (!pref_service_) + return; + + for (const Metric& metric : kMetrics) { + ClearMetricValue(metric); + SetLastTimeToNow(metric); + } +} + +double UserClassifier::UpdateMetricOnEvent(Metric metric) { + // The pref_service_ can be null in tests. + if (!pref_service_) + return 0; + + double hours_since_last_time = + std::min(max_hours_, GetHoursSinceLastTime(metric)); + // Ignore events within the same "browsing session". + if (hours_since_last_time < min_hours_) + return GetUpToDateMetricValue(metric); + + SetLastTimeToNow(metric); + + double metric_value = GetMetricValue(metric); + // Add 1 to the discounted metric as the event has happened right now. + double new_metric_value = + 1 + DiscountMetric(metric_value, hours_since_last_time, + discount_rate_per_hour_); + SetMetricValue(metric, new_metric_value); + return new_metric_value; +} + +double UserClassifier::GetUpToDateMetricValue(Metric metric) const { + // The pref_service_ can be null in tests. + if (!pref_service_) + return 0; + + double hours_since_last_time = + std::min(max_hours_, GetHoursSinceLastTime(metric)); + + double metric_value = GetMetricValue(metric); + return DiscountMetric(metric_value, hours_since_last_time, + discount_rate_per_hour_); +} + +double UserClassifier::GetHoursSinceLastTime(Metric metric) const { + if (!HasLastTime(metric)) + return 0; + + base::TimeDelta since_last_time = + base::Time::Now() - base::Time::FromInternalValue(pref_service_->GetInt64( + kLastTimeKeys[static_cast<int>(metric)])); + return since_last_time.InSecondsF() / 3600; +} + +bool UserClassifier::HasLastTime(Metric metric) const { + return pref_service_->HasPrefPath(kLastTimeKeys[static_cast<int>(metric)]); +} + +void UserClassifier::SetLastTimeToNow(Metric metric) { + pref_service_->SetInt64(kLastTimeKeys[static_cast<int>(metric)], + base::Time::Now().ToInternalValue()); +} + +double UserClassifier::GetMetricValue(Metric metric) const { + return pref_service_->GetDouble(kMetricKeys[static_cast<int>(metric)]); +} + +void UserClassifier::SetMetricValue(Metric metric, double metric_value) { + pref_service_->SetDouble(kMetricKeys[static_cast<int>(metric)], metric_value); +} + +void UserClassifier::ClearMetricValue(Metric metric) { + pref_service_->ClearPref(kMetricKeys[static_cast<int>(metric)]); +} + +} // namespace ntp_snippets diff --git a/chromium/components/ntp_snippets/user_classifier.h b/chromium/components/ntp_snippets/user_classifier.h new file mode 100644 index 00000000000..4f8c9848d62 --- /dev/null +++ b/chromium/components/ntp_snippets/user_classifier.h @@ -0,0 +1,109 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_NTP_SNIPPETS_USER_CLASSIFIER_H_ +#define COMPONENTS_NTP_SNIPPETS_USER_CLASSIFIER_H_ + +#include <string> + +#include "base/macros.h" +#include "base/time/time.h" + +class PrefRegistrySimple; +class PrefService; + +namespace ntp_snippets { + +// Collects data about user usage patterns of content suggestions, computes +// long-term user metrics locally using pref, and reports the metrics to UMA. +// Based on these long-term user metrics, it classifies the user in a UserClass. +class UserClassifier { + public: + // Enumeration listing user classes + enum class UserClass { + RARE_NTP_USER, + ACTIVE_NTP_USER, + ACTIVE_SUGGESTIONS_CONSUMER, + }; + + // For estimating the average length of the intervals between two successive + // events, we keep a simple frequency model, a single value that we call + // "metric" below. + // We track exponentially-discounted rate of the given event per hour where + // the continuous utility function between two successive events (e.g. opening + // a NTP) at times t1 < t2 is 1 / (t2-t1), i.e. intuitively the rate of this + // event in this time interval. + // See https://en.wikipedia.org/wiki/Exponential_discounting for more details. + // We keep track of the following events. + // NOTE: if you add any element, add it also in the static arrays in .cc and + // create another histogram. + enum class Metric { + NTP_OPENED, // When the user opens a new NTP - this indicates potential + // use of content suggestions. + SUGGESTIONS_SHOWN, // When the content suggestions are shown to the user - + // in the current implementation when the user scrolls + // below the fold. + SUGGESTIONS_USED, // When the user clicks on some suggestions or on some + // "More" button. + COUNT // Keep this as the last element. + }; + + // The provided |pref_service| may be nullptr in unit-tests. + explicit UserClassifier(PrefService* pref_service); + ~UserClassifier(); + + // Registers profile prefs for all metrics. Called from browser_prefs.cc. + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + // Informs the UserClassifier about a new event for |metric|. The + // classification is based on these calls. + void OnEvent(Metric metric); + + // Get the estimate average length of the interval between two successive + // events of the given type. + double GetEstimatedAvgTime(Metric metric) const; + + // Return the classification of the current user. + UserClass GetUserClass() const; + std::string GetUserClassDescriptionForDebugging() const; + + // Resets the classification (emulates a fresh upgrade / install). + void ClearClassificationForDebugging(); + + private: + // The event has happened, recompute the metric accordingly. Then store and + // return the new value. + double UpdateMetricOnEvent(Metric metric); + // No event has happened but we need to get up-to-date metric, recompute and + // return the new value. This function does not store the recomputed metric. + double GetUpToDateMetricValue(Metric metric) const; + + // Returns the number of hours since the last event of the same type. + // If there is no last event of that type, assume it happened just now and + // return 0. + double GetHoursSinceLastTime(Metric metric) const; + bool HasLastTime(Metric metric) const; + void SetLastTimeToNow(Metric metric); + + double GetMetricValue(Metric metric) const; + void SetMetricValue(Metric metric, double metric_value); + void ClearMetricValue(Metric metric); + + PrefService* pref_service_; + + // Params of the metric. + const double discount_rate_per_hour_; + const double min_hours_; + const double max_hours_; + + // Params of the classification. + const double active_consumer_scrolls_at_least_once_per_hours_; + const double rare_user_opens_ntp_at_most_once_per_hours_; + + DISALLOW_COPY_AND_ASSIGN(UserClassifier); +}; + +} // namespace ntp_snippets + +#endif // COMPONENTS_NTP_SNIPPETS_USER_CLASSIFIER_H_ |