diff options
Diffstat (limited to 'chromium/chromeos/services/assistant')
5 files changed, 61 insertions, 352 deletions
diff --git a/chromium/chromeos/services/assistant/BUILD.gn b/chromium/chromeos/services/assistant/BUILD.gn index dc1eb5af16b..52e4387f54b 100644 --- a/chromium/chromeos/services/assistant/BUILD.gn +++ b/chromium/chromeos/services/assistant/BUILD.gn @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/buildflag_header.gni") import("//chromeos/assistant/assistant.gni") assert(is_chromeos) @@ -17,6 +18,8 @@ component("lib") { defines = [ "IS_ASSISTANT_SERVICE_IMPL" ] sources = [ + "assistant_interaction_logger.cc", + "assistant_interaction_logger.h", "assistant_manager_service.h", "fake_assistant_manager_service_impl.cc", "fake_assistant_manager_service_impl.h", @@ -41,6 +44,7 @@ component("lib") { "//components/prefs", "//components/signin/public/identity_manager", "//components/user_manager", + "//media", "//services/device/public/mojom", "//services/network/public/cpp:cpp", "//ui/accessibility:ax_assistant", @@ -101,7 +105,15 @@ component("lib") { "utils.h", ] + if (enable_fake_assistant_microphone) { + sources += [ + "platform/fake_input_device.cc", + "platform/fake_input_device.h", + ] + } + deps += [ + ":buildflags", "//chromeos/assistant/internal", "//chromeos/assistant/internal:libassistant", "//chromeos/assistant/internal/proto/google3", @@ -128,6 +140,7 @@ source_set("tests") { testonly = true deps = [ ":lib", + ":test_support", "//ash/public/cpp/assistant/test_support", "//ash/public/mojom", "//base", @@ -175,8 +188,6 @@ source_set("tests") { "test_support/fake_platform_api.h", "test_support/fake_service_context.cc", "test_support/fake_service_context.h", - "test_support/mock_assistant_interaction_subscriber.cc", - "test_support/mock_assistant_interaction_subscriber.h", "test_support/mock_media_manager.cc", "test_support/mock_media_manager.h", ] @@ -190,6 +201,7 @@ source_set("tests") { "//chromeos/services/network_config/public/mojom", "//services/audio/public/cpp:test_support", "//services/device/public/cpp:test_support", + "//services/media_session/public/mojom", ] } } @@ -199,6 +211,8 @@ static_library("test_support") { sources = [ "test_support/mock_assistant.cc", "test_support/mock_assistant.h", + "test_support/mock_assistant_interaction_subscriber.cc", + "test_support/mock_assistant_interaction_subscriber.h", ] deps = [ "//base", @@ -208,3 +222,10 @@ static_library("test_support") { "//testing/gmock", ] } + +buildflag_header("buildflags") { + header = "buildflags.h" + + flags = + [ "ENABLE_FAKE_ASSISTANT_MICROPHONE=$enable_fake_assistant_microphone" ] +} diff --git a/chromium/chromeos/services/assistant/public/cpp/BUILD.gn b/chromium/chromeos/services/assistant/public/cpp/BUILD.gn index 55a3a38f401..55789f29380 100644 --- a/chromium/chromeos/services/assistant/public/cpp/BUILD.gn +++ b/chromium/chromeos/services/assistant/public/cpp/BUILD.gn @@ -18,17 +18,22 @@ component("cpp") { "assistant_service.h", "assistant_settings.cc", "assistant_settings.h", - "default_assistant_interaction_subscriber.cc", - "default_assistant_interaction_subscriber.h", "device_actions.cc", "device_actions.h", "features.cc", "features.h", ] - deps = [ + public_deps = [ + "//ash/public/mojom", + "//chromeos/constants:constants", "//chromeos/services/assistant/public/mojom", - "//components/prefs", - "//mojo/public/cpp/bindings", + "//chromeos/services/network_config/public/mojom", + "//services/audio/public/mojom", + "//services/device/public/mojom", + "//services/media_session/public/mojom", + "//ui/accessibility/mojom", ] + + deps = [ "//components/prefs" ] } diff --git a/chromium/chromeos/services/assistant/public/mojom/BUILD.gn b/chromium/chromeos/services/assistant/public/mojom/BUILD.gn index 4f516d4a3a1..026d8b048f4 100644 --- a/chromium/chromeos/services/assistant/public/mojom/BUILD.gn +++ b/chromium/chromeos/services/assistant/public/mojom/BUILD.gn @@ -5,26 +5,9 @@ import("//mojo/public/tools/bindings/mojom.gni") mojom("mojom") { - sources = [ - "assistant.mojom", - "assistant_audio_decoder.mojom", - ] - - public_deps = [ - ":notification", - "//ash/public/mojom", - "//chromeos/services/network_config/public/mojom", - "//mojo/public/mojom/base", - "//services/audio/public/mojom", - "//services/device/public/mojom", - "//services/media_session/public/mojom", - "//ui/accessibility/mojom", - "//ui/gfx/geometry/mojom", - "//url/mojom:url_mojom_gurl", - ] + sources = [ "assistant_audio_decoder.mojom" ] - # Ash mojom disables variants, so its dependents must do the same. - disable_variants = true + public_deps = [ ":notification" ] } # This dependency is separated to avoid circular dependencies between the above diff --git a/chromium/chromeos/services/assistant/public/mojom/assistant.mojom b/chromium/chromeos/services/assistant/public/mojom/assistant.mojom deleted file mode 100644 index 3bedfaadbde..00000000000 --- a/chromium/chromeos/services/assistant/public/mojom/assistant.mojom +++ /dev/null @@ -1,311 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -module chromeos.assistant.mojom; - -import "ash/public/mojom/assistant_controller.mojom"; -import "ash/public/mojom/assistant_volume_control.mojom"; -import "chromeos/services/assistant/public/mojom/assistant_audio_decoder.mojom"; -import "chromeos/services/assistant/public/mojom/assistant_notification.mojom"; -import "chromeos/services/network_config/public/mojom/cros_network_config.mojom"; -import "mojo/public/mojom/base/string16.mojom"; -import "mojo/public/mojom/base/time.mojom"; -import "mojo/public/mojom/base/unguessable_token.mojom"; -import "services/audio/public/mojom/stream_factory.mojom"; -import "services/device/public/mojom/battery_monitor.mojom"; -import "services/device/public/mojom/wake_lock_provider.mojom"; -import "services/media_session/public/mojom/audio_focus.mojom"; -import "services/media_session/public/mojom/media_controller.mojom"; -import "ui/accessibility/mojom/ax_assistant_structure.mojom"; -import "ui/gfx/geometry/mojom/geometry.mojom"; -import "url/mojom/url.mojom"; - -// Interface to communicate with assistant backend. -// The Assistant instance is held by chromeos::assistant::Service, which is -// hosted in the browser process. -// The callers are mostly in ash, and some are in autotest. -interface Assistant { - // Starts an interaction to edit the reminder uniquely identified by - // |client_id|. In response to the request, LibAssistant will initiate - // a user facing interaction with the context pre-populated with details - // to edit the specified reminder. - StartEditReminderInteraction(string client_id); - - // Starts a screen context interaction. Results related to the screen context - // will be returned through the |AssistantInteractionSubscriber| interface to - // registered subscribers. - // |assistant_screenshot| contains JPEG data. - StartScreenContextInteraction( - ax.mojom.AssistantStructure? assistant_structure, - array<uint8> assistant_screenshot); - - // Starts a new Assistant text interaction. If |allow_tts| is true, the - // result will contain TTS. Otherwise TTS will not be present in the - // generated server response. Results will be returned through registered - // |AssistantInteractionSubscriber|. - StartTextInteraction( - string query, AssistantQuerySource source, bool allow_tts); - - // Starts a new Assistant voice interaction. - StartVoiceInteraction(); - - // Starts a warmer welcome interaction for Assistant launch. - // |num_warmer_welcome_triggered| is the count of warmer welcomes - // already triggered. If |allow_tts| is true, the result may contain TTS. - // Otherwise TTS will not be present in the generated server response. - StartWarmerWelcomeInteraction(int32 num_warmer_welcome_triggered, - bool allow_tts); - - // Stops the active Assistant interaction and cancel the conversation if - // |cancel_conversation|. If there is no active interaction, this method - // is a no-op. - StopActiveInteraction(bool cancel_conversation); - - // Registers assistant interaction event subscriber. Subscribers' - // implementation is responsible for selecting events of interest. - AddAssistantInteractionSubscriber( - pending_remote<AssistantInteractionSubscriber> subscriber); - - // Retrieves a notification. A voiceless interaction will be sent to server to - // retrieve the notification of |action_index|, which can trigger other - // Assistant events such as OnTextResponse to show the result in the UI. The - // retrieved notification will be removed from the UI. - // |action_index| is the index of the tapped action. The main UI in the - // notification contains the top level action, which index is 0. The buttons - // have the additional actions, which are indexed starting from 1. - RetrieveNotification(AssistantNotification notification, int32 action_index); - - // Dismisses a notification. - DismissNotification(AssistantNotification notification); - - // Invoked when accessibility status is changed. Note that though - // accessibility status has changed, |spoken_feedback_enabled| may not have. - OnAccessibilityStatusChanged(bool spoken_feedback_enabled); - - // Send Assistant feedback to Assistant server. - SendAssistantFeedback(AssistantFeedback feedback); - - // Invoked on entry to Assistant UI. - NotifyEntryIntoAssistantUi(AssistantEntryPoint entry_point); - - // Alarm/Timer methods ------------------------------------------------------- - - // Adds the specified |duration| to the timer identified by |id|. Note that - // this method is a no-op if there is no existing timer identified by |id|. - AddTimeToTimer(string id, mojo_base.mojom.TimeDelta duration); - - // Pauses the timer specified by |id|. Note that this method is a no-op if - // there is no existing timer identified by |id| or if a timer does exist but - // is already paused. - PauseTimer(string id); - - // Remove the alarm/timer specified by |id|. - // NOTE: this is a no-op if no such alarm/timer exists. - RemoveAlarmOrTimer(string id); - - // Resumes the timer specified by |id|. Note that this method is a no-op if - // there is no existing timer identified by |id| or if a timer does exist but - // isn't currently paused. - ResumeTimer(string id); -}; - -// Enumeration of Assistant entry points. These values are persisted to logs. -// Entries should not be renumbered and numeric values should never be reused. -// Only append to this enum is allowed if the possible entry source grows. -enum AssistantEntryPoint { - kUnspecified = 0, - kDeepLink = 1, - kHotkey = 2, - kHotword = 3, - // kLauncherSearchBoxDeprecated = 4, - kLongPressLauncher = 5, - kSetup = 6, - kStylus = 7, - kLauncherSearchResult = 8, - kLauncherSearchBoxIcon = 9, - kProactiveSuggestions = 10, - kLauncherChip = 11, -}; - -// Enumeration of Assistant exit points. These values are persisted to logs. -// Entries should not be renumbered and numeric values should never be reused. -// Only append to this enum is allowed if the possible exit source grows. -enum AssistantExitPoint { - // Includes keyboard interruptions (e.g. launching Chrome OS feedback - // using keyboard shortcuts, pressing search button). - kUnspecified = 0, - kCloseButton = 1, - kHotkey = 2, - kNewBrowserTabFromServer = 3, - kNewBrowserTabFromUser = 4, - kOutsidePress = 5, - kSetup = 6, - kStylus = 7, - kBackInLauncher = 8, - kLauncherClose = 9, - kLauncherOpen = 10, - kScreenshot = 11, - kOverviewMode = 12, -}; - -// Describes an Assistant interaction. -struct AssistantInteractionMetadata { - AssistantInteractionType type; - AssistantQuerySource source; - string query; -}; - -// Enumeration of possible Assistant interaction types. -enum AssistantInteractionType { - kText, - kVoice, -}; - -// Enumeration of possible Assistant query sources. These values are persisted -// to logs. Entries should not be renumbered and numeric values should never -// be reused. Append new values to the end. -enum AssistantQuerySource { - kUnspecified = 0, - kDeepLink = 1, - kDialogPlateTextField = 2, - kStylus = 3, - kSuggestionChip = 4, - kVoiceInput = 5, - kProactiveSuggestions = 6, - kLibAssistantInitiated = 7, - kWarmerWelcome = 8, - kConversationStarter = 9, - kWhatsOnMyScreen = 10, - kQuickAnswers = 11, - kLauncherChip = 12, -}; - -// Subscribes to Assistant's interaction event. These events are server driven -// in response to the user's direct interaction with the assistant. Responses -// from the assistant may contain untrusted third-party content. Subscriber -// implementations must be sure to handle the response data appropriately. -interface AssistantInteractionSubscriber { - // Assistant interaction has started. - OnInteractionStarted(AssistantInteractionMetadata metadata); - - // Assistant interaction has ended with the specified |resolution|. - OnInteractionFinished(AssistantInteractionResolution resolution); - - // Assistant got Html response with fallback text from server. - OnHtmlResponse(string response, string fallback); - - // Assistant got suggestions response from server. - OnSuggestionsResponse(array<AssistantSuggestion> response); - - // Assistant got text response from server. - OnTextResponse(string response); - - // Assistant got open URL response from server. |in_background| refers to - // being in background of Assistant UI, not in background of browser. - OnOpenUrlResponse(url.mojom.Url url, bool in_background); - - // Assistant got open Android app response from server. - OnOpenAppResponse(AndroidAppInfo app_info) => (bool app_opened); - - // Assistant speech recognition has started. - OnSpeechRecognitionStarted(); - - // Assistant speech recognition intermediate result is available. - OnSpeechRecognitionIntermediateResult(string high_confidence_text, - string low_confidence_text); - - // Assistant speech recognition detected end of utterance. - OnSpeechRecognitionEndOfUtterance(); - - // Assistant speech recognition final result is available. - OnSpeechRecognitionFinalResult(string final_result); - - // Assistant got an instantaneous speech level update in dB. - OnSpeechLevelUpdated(float speech_level); - - // Assistant has started speaking. When TTS is started due to an error that - // occurred during the interaction, |due_to_error| is true. - OnTtsStarted(bool due_to_error); - - // Assistant has started waiting. This occur during execution of a routine to - // give the user time to digest a response before continuing execution. - OnWaitStarted(); -}; - -// Enumeration of possible completions for an Assistant interaction. -enum AssistantInteractionResolution { - // Assistant interaction completed normally. - kNormal, - // Assistant interaction completed due to barge in or cancellation. - kInterruption, - // Assistant interaction completed due to error. - kError, - // Assistant interaction completed due to mic timeout. - kMicTimeout, - // Assistant interaction completed due to multi-device hotword loss. - kMultiDeviceHotwordLoss, -}; - -// Enumeration of possible Assistant suggestion types. -enum AssistantSuggestionType { - kUnspecified, - kConversationStarter, -}; - -// Models an Assistant suggestion. -struct AssistantSuggestion { - // Uniquely identifies a given suggestion. - mojo_base.mojom.UnguessableToken id; - - // Allows us to differentiate between a typical Assistant suggestion and an - // Assistant suggestion of another type (e.g. a conversation starter). - AssistantSuggestionType type = kUnspecified; - - // Display text. e.g. "Cancel". - string text; - - // Optional URL for icon. e.g. "https://www.gstatic.com/icon.png". - url.mojom.Url icon_url; - - // Optional URL for action. e.g. - // "https://www.google.com/search?query=action". - url.mojom.Url action_url; -}; - -// Models status of an app. -enum AppStatus { UNKNOWN, AVAILABLE, UNAVAILABLE, VERSION_MISMATCH, DISABLED }; - -// Models an Android app. -struct AndroidAppInfo { - // Unique name to identify a specific app. - string package_name; - - // Version number of the app. - int32 version; - - // Localized app name. - string localized_app_name; - - // Intent data to operate on. - string intent; - - // Status of the app. - AppStatus status; - - // The general action to be performed, such as ACTION_VIEW, ACTION_MAIN, etc. - string action; -}; - -// Details for Assistant feedback. -struct AssistantFeedback { - // User input to be sent with the feedback report. - string description; - - // Whether user consent to send debug info. - bool assistant_debug_info_allowed; - - // Screenshot if allowed by user. - // Raw data (non-encoded binary octets) - string screenshot_png; -}; diff --git a/chromium/chromeos/services/assistant/public/mojom/assistant_notification.mojom b/chromium/chromeos/services/assistant/public/mojom/assistant_notification.mojom index 0ddfe4b1c08..e10f8f61360 100644 --- a/chromium/chromeos/services/assistant/public/mojom/assistant_notification.mojom +++ b/chromium/chromeos/services/assistant/public/mojom/assistant_notification.mojom @@ -7,16 +7,6 @@ module chromeos.assistant.mojom; import "mojo/public/mojom/base/time.mojom"; import "url/mojom/url.mojom"; -// TODO(b:153495778): Remove notification types. -// Enumeration of notification types. -enum AssistantNotificationType { - // A notification of type |kSystem| will only be displayed within the Message - // Center. It is immediately added to the Message Center regardless of - // Assistant UI visibility state, although it may not be added if the user has - // opted out of seeing system notifications. - kSystem, -}; - // Models a notification button. struct AssistantNotificationButton { // Display text of the button. @@ -24,13 +14,21 @@ struct AssistantNotificationButton { // Optional URL to open when the tap action is invoked on the button. url.mojom.Url action_url; + + // If |true|, the associated notification will be removed on button click. + bool remove_notification_on_click = true; +}; + +// Enumeration of possible notification priorities. +// NOTE: This enum maps to a subset of message_center::NotificationPriority. +enum AssistantNotificationPriority { + kLow, // See message_center::NotificationPriority::LOW_PRIORITY. + kDefault, // See message_center::NotificationPriority::DEFAULT_PRIORITY. + kHigh, // See message_center::NotificationPriority::HIGH_PRIORITY. }; // Models an Assistant notification. struct AssistantNotification { - // Type of the notification. - AssistantNotificationType type = AssistantNotificationType.kSystem; - // Title of the notification. string title; @@ -60,10 +58,23 @@ struct AssistantNotification { // Obfuscated Gaia id of the intended recipient of the user. string obfuscated_gaia_id; - // Whether this notification can turn on the display if it was off. - bool is_high_priority = false; + // Priority for the notification. This affects whether or not the notification + // will pop up and/or have the capability to wake the display if it was off. + AssistantNotificationPriority priority = kDefault; // When the notification should expire. // Expressed as milliseconds since Unix Epoch. mojo_base.mojom.Time? expiry_time; + + // If |true|, the notification will be removed on click. + bool remove_on_click = true; + + // If |true|, the user can't remove the notification. + bool is_pinned = false; + + // If |true|, this notification was sent from the server. Clicking a + // notification that was sent from the server may result in a request to the + // server to retrieve the notification payload. One example of this would be + // notifications triggered by an Assistant reminder. + bool from_server = false; }; |