diff options
Diffstat (limited to 'chromium/chromeos/services')
18 files changed, 610 insertions, 362 deletions
diff --git a/chromium/chromeos/services/assistant/BUILD.gn b/chromium/chromeos/services/assistant/BUILD.gn index dc1eb5af16b..52e4387f54b 100644 --- a/chromium/chromeos/services/assistant/BUILD.gn +++ b/chromium/chromeos/services/assistant/BUILD.gn @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/buildflag_header.gni") import("//chromeos/assistant/assistant.gni") assert(is_chromeos) @@ -17,6 +18,8 @@ component("lib") { defines = [ "IS_ASSISTANT_SERVICE_IMPL" ] sources = [ + "assistant_interaction_logger.cc", + "assistant_interaction_logger.h", "assistant_manager_service.h", "fake_assistant_manager_service_impl.cc", "fake_assistant_manager_service_impl.h", @@ -41,6 +44,7 @@ component("lib") { "//components/prefs", "//components/signin/public/identity_manager", "//components/user_manager", + "//media", "//services/device/public/mojom", "//services/network/public/cpp:cpp", "//ui/accessibility:ax_assistant", @@ -101,7 +105,15 @@ component("lib") { "utils.h", ] + if (enable_fake_assistant_microphone) { + sources += [ + "platform/fake_input_device.cc", + "platform/fake_input_device.h", + ] + } + deps += [ + ":buildflags", "//chromeos/assistant/internal", "//chromeos/assistant/internal:libassistant", "//chromeos/assistant/internal/proto/google3", @@ -128,6 +140,7 @@ source_set("tests") { testonly = true deps = [ ":lib", + ":test_support", "//ash/public/cpp/assistant/test_support", "//ash/public/mojom", "//base", @@ -175,8 +188,6 @@ source_set("tests") { "test_support/fake_platform_api.h", "test_support/fake_service_context.cc", "test_support/fake_service_context.h", - "test_support/mock_assistant_interaction_subscriber.cc", - "test_support/mock_assistant_interaction_subscriber.h", "test_support/mock_media_manager.cc", "test_support/mock_media_manager.h", ] @@ -190,6 +201,7 @@ source_set("tests") { "//chromeos/services/network_config/public/mojom", "//services/audio/public/cpp:test_support", "//services/device/public/cpp:test_support", + "//services/media_session/public/mojom", ] } } @@ -199,6 +211,8 @@ static_library("test_support") { sources = [ "test_support/mock_assistant.cc", "test_support/mock_assistant.h", + "test_support/mock_assistant_interaction_subscriber.cc", + "test_support/mock_assistant_interaction_subscriber.h", ] deps = [ "//base", @@ -208,3 +222,10 @@ static_library("test_support") { "//testing/gmock", ] } + +buildflag_header("buildflags") { + header = "buildflags.h" + + flags = + [ "ENABLE_FAKE_ASSISTANT_MICROPHONE=$enable_fake_assistant_microphone" ] +} diff --git a/chromium/chromeos/services/assistant/public/cpp/BUILD.gn b/chromium/chromeos/services/assistant/public/cpp/BUILD.gn index 55a3a38f401..55789f29380 100644 --- a/chromium/chromeos/services/assistant/public/cpp/BUILD.gn +++ b/chromium/chromeos/services/assistant/public/cpp/BUILD.gn @@ -18,17 +18,22 @@ component("cpp") { "assistant_service.h", "assistant_settings.cc", "assistant_settings.h", - "default_assistant_interaction_subscriber.cc", - "default_assistant_interaction_subscriber.h", "device_actions.cc", "device_actions.h", "features.cc", "features.h", ] - deps = [ + public_deps = [ + "//ash/public/mojom", + "//chromeos/constants:constants", "//chromeos/services/assistant/public/mojom", - "//components/prefs", - "//mojo/public/cpp/bindings", + "//chromeos/services/network_config/public/mojom", + "//services/audio/public/mojom", + "//services/device/public/mojom", + "//services/media_session/public/mojom", + "//ui/accessibility/mojom", ] + + deps = [ "//components/prefs" ] } diff --git a/chromium/chromeos/services/assistant/public/mojom/BUILD.gn b/chromium/chromeos/services/assistant/public/mojom/BUILD.gn index 4f516d4a3a1..026d8b048f4 100644 --- a/chromium/chromeos/services/assistant/public/mojom/BUILD.gn +++ b/chromium/chromeos/services/assistant/public/mojom/BUILD.gn @@ -5,26 +5,9 @@ import("//mojo/public/tools/bindings/mojom.gni") mojom("mojom") { - sources = [ - "assistant.mojom", - "assistant_audio_decoder.mojom", - ] - - public_deps = [ - ":notification", - "//ash/public/mojom", - "//chromeos/services/network_config/public/mojom", - "//mojo/public/mojom/base", - "//services/audio/public/mojom", - "//services/device/public/mojom", - "//services/media_session/public/mojom", - "//ui/accessibility/mojom", - "//ui/gfx/geometry/mojom", - "//url/mojom:url_mojom_gurl", - ] + sources = [ "assistant_audio_decoder.mojom" ] - # Ash mojom disables variants, so its dependents must do the same. - disable_variants = true + public_deps = [ ":notification" ] } # This dependency is separated to avoid circular dependencies between the above diff --git a/chromium/chromeos/services/assistant/public/mojom/assistant.mojom b/chromium/chromeos/services/assistant/public/mojom/assistant.mojom deleted file mode 100644 index 3bedfaadbde..00000000000 --- a/chromium/chromeos/services/assistant/public/mojom/assistant.mojom +++ /dev/null @@ -1,311 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -module chromeos.assistant.mojom; - -import "ash/public/mojom/assistant_controller.mojom"; -import "ash/public/mojom/assistant_volume_control.mojom"; -import "chromeos/services/assistant/public/mojom/assistant_audio_decoder.mojom"; -import "chromeos/services/assistant/public/mojom/assistant_notification.mojom"; -import "chromeos/services/network_config/public/mojom/cros_network_config.mojom"; -import "mojo/public/mojom/base/string16.mojom"; -import "mojo/public/mojom/base/time.mojom"; -import "mojo/public/mojom/base/unguessable_token.mojom"; -import "services/audio/public/mojom/stream_factory.mojom"; -import "services/device/public/mojom/battery_monitor.mojom"; -import "services/device/public/mojom/wake_lock_provider.mojom"; -import "services/media_session/public/mojom/audio_focus.mojom"; -import "services/media_session/public/mojom/media_controller.mojom"; -import "ui/accessibility/mojom/ax_assistant_structure.mojom"; -import "ui/gfx/geometry/mojom/geometry.mojom"; -import "url/mojom/url.mojom"; - -// Interface to communicate with assistant backend. -// The Assistant instance is held by chromeos::assistant::Service, which is -// hosted in the browser process. -// The callers are mostly in ash, and some are in autotest. -interface Assistant { - // Starts an interaction to edit the reminder uniquely identified by - // |client_id|. In response to the request, LibAssistant will initiate - // a user facing interaction with the context pre-populated with details - // to edit the specified reminder. - StartEditReminderInteraction(string client_id); - - // Starts a screen context interaction. Results related to the screen context - // will be returned through the |AssistantInteractionSubscriber| interface to - // registered subscribers. - // |assistant_screenshot| contains JPEG data. - StartScreenContextInteraction( - ax.mojom.AssistantStructure? assistant_structure, - array<uint8> assistant_screenshot); - - // Starts a new Assistant text interaction. If |allow_tts| is true, the - // result will contain TTS. Otherwise TTS will not be present in the - // generated server response. Results will be returned through registered - // |AssistantInteractionSubscriber|. - StartTextInteraction( - string query, AssistantQuerySource source, bool allow_tts); - - // Starts a new Assistant voice interaction. - StartVoiceInteraction(); - - // Starts a warmer welcome interaction for Assistant launch. - // |num_warmer_welcome_triggered| is the count of warmer welcomes - // already triggered. If |allow_tts| is true, the result may contain TTS. - // Otherwise TTS will not be present in the generated server response. - StartWarmerWelcomeInteraction(int32 num_warmer_welcome_triggered, - bool allow_tts); - - // Stops the active Assistant interaction and cancel the conversation if - // |cancel_conversation|. If there is no active interaction, this method - // is a no-op. - StopActiveInteraction(bool cancel_conversation); - - // Registers assistant interaction event subscriber. Subscribers' - // implementation is responsible for selecting events of interest. - AddAssistantInteractionSubscriber( - pending_remote<AssistantInteractionSubscriber> subscriber); - - // Retrieves a notification. A voiceless interaction will be sent to server to - // retrieve the notification of |action_index|, which can trigger other - // Assistant events such as OnTextResponse to show the result in the UI. The - // retrieved notification will be removed from the UI. - // |action_index| is the index of the tapped action. The main UI in the - // notification contains the top level action, which index is 0. The buttons - // have the additional actions, which are indexed starting from 1. - RetrieveNotification(AssistantNotification notification, int32 action_index); - - // Dismisses a notification. - DismissNotification(AssistantNotification notification); - - // Invoked when accessibility status is changed. Note that though - // accessibility status has changed, |spoken_feedback_enabled| may not have. - OnAccessibilityStatusChanged(bool spoken_feedback_enabled); - - // Send Assistant feedback to Assistant server. - SendAssistantFeedback(AssistantFeedback feedback); - - // Invoked on entry to Assistant UI. - NotifyEntryIntoAssistantUi(AssistantEntryPoint entry_point); - - // Alarm/Timer methods ------------------------------------------------------- - - // Adds the specified |duration| to the timer identified by |id|. Note that - // this method is a no-op if there is no existing timer identified by |id|. - AddTimeToTimer(string id, mojo_base.mojom.TimeDelta duration); - - // Pauses the timer specified by |id|. Note that this method is a no-op if - // there is no existing timer identified by |id| or if a timer does exist but - // is already paused. - PauseTimer(string id); - - // Remove the alarm/timer specified by |id|. - // NOTE: this is a no-op if no such alarm/timer exists. - RemoveAlarmOrTimer(string id); - - // Resumes the timer specified by |id|. Note that this method is a no-op if - // there is no existing timer identified by |id| or if a timer does exist but - // isn't currently paused. - ResumeTimer(string id); -}; - -// Enumeration of Assistant entry points. These values are persisted to logs. -// Entries should not be renumbered and numeric values should never be reused. -// Only append to this enum is allowed if the possible entry source grows. -enum AssistantEntryPoint { - kUnspecified = 0, - kDeepLink = 1, - kHotkey = 2, - kHotword = 3, - // kLauncherSearchBoxDeprecated = 4, - kLongPressLauncher = 5, - kSetup = 6, - kStylus = 7, - kLauncherSearchResult = 8, - kLauncherSearchBoxIcon = 9, - kProactiveSuggestions = 10, - kLauncherChip = 11, -}; - -// Enumeration of Assistant exit points. These values are persisted to logs. -// Entries should not be renumbered and numeric values should never be reused. -// Only append to this enum is allowed if the possible exit source grows. -enum AssistantExitPoint { - // Includes keyboard interruptions (e.g. launching Chrome OS feedback - // using keyboard shortcuts, pressing search button). - kUnspecified = 0, - kCloseButton = 1, - kHotkey = 2, - kNewBrowserTabFromServer = 3, - kNewBrowserTabFromUser = 4, - kOutsidePress = 5, - kSetup = 6, - kStylus = 7, - kBackInLauncher = 8, - kLauncherClose = 9, - kLauncherOpen = 10, - kScreenshot = 11, - kOverviewMode = 12, -}; - -// Describes an Assistant interaction. -struct AssistantInteractionMetadata { - AssistantInteractionType type; - AssistantQuerySource source; - string query; -}; - -// Enumeration of possible Assistant interaction types. -enum AssistantInteractionType { - kText, - kVoice, -}; - -// Enumeration of possible Assistant query sources. These values are persisted -// to logs. Entries should not be renumbered and numeric values should never -// be reused. Append new values to the end. -enum AssistantQuerySource { - kUnspecified = 0, - kDeepLink = 1, - kDialogPlateTextField = 2, - kStylus = 3, - kSuggestionChip = 4, - kVoiceInput = 5, - kProactiveSuggestions = 6, - kLibAssistantInitiated = 7, - kWarmerWelcome = 8, - kConversationStarter = 9, - kWhatsOnMyScreen = 10, - kQuickAnswers = 11, - kLauncherChip = 12, -}; - -// Subscribes to Assistant's interaction event. These events are server driven -// in response to the user's direct interaction with the assistant. Responses -// from the assistant may contain untrusted third-party content. Subscriber -// implementations must be sure to handle the response data appropriately. -interface AssistantInteractionSubscriber { - // Assistant interaction has started. - OnInteractionStarted(AssistantInteractionMetadata metadata); - - // Assistant interaction has ended with the specified |resolution|. - OnInteractionFinished(AssistantInteractionResolution resolution); - - // Assistant got Html response with fallback text from server. - OnHtmlResponse(string response, string fallback); - - // Assistant got suggestions response from server. - OnSuggestionsResponse(array<AssistantSuggestion> response); - - // Assistant got text response from server. - OnTextResponse(string response); - - // Assistant got open URL response from server. |in_background| refers to - // being in background of Assistant UI, not in background of browser. - OnOpenUrlResponse(url.mojom.Url url, bool in_background); - - // Assistant got open Android app response from server. - OnOpenAppResponse(AndroidAppInfo app_info) => (bool app_opened); - - // Assistant speech recognition has started. - OnSpeechRecognitionStarted(); - - // Assistant speech recognition intermediate result is available. - OnSpeechRecognitionIntermediateResult(string high_confidence_text, - string low_confidence_text); - - // Assistant speech recognition detected end of utterance. - OnSpeechRecognitionEndOfUtterance(); - - // Assistant speech recognition final result is available. - OnSpeechRecognitionFinalResult(string final_result); - - // Assistant got an instantaneous speech level update in dB. - OnSpeechLevelUpdated(float speech_level); - - // Assistant has started speaking. When TTS is started due to an error that - // occurred during the interaction, |due_to_error| is true. - OnTtsStarted(bool due_to_error); - - // Assistant has started waiting. This occur during execution of a routine to - // give the user time to digest a response before continuing execution. - OnWaitStarted(); -}; - -// Enumeration of possible completions for an Assistant interaction. -enum AssistantInteractionResolution { - // Assistant interaction completed normally. - kNormal, - // Assistant interaction completed due to barge in or cancellation. - kInterruption, - // Assistant interaction completed due to error. - kError, - // Assistant interaction completed due to mic timeout. - kMicTimeout, - // Assistant interaction completed due to multi-device hotword loss. - kMultiDeviceHotwordLoss, -}; - -// Enumeration of possible Assistant suggestion types. -enum AssistantSuggestionType { - kUnspecified, - kConversationStarter, -}; - -// Models an Assistant suggestion. -struct AssistantSuggestion { - // Uniquely identifies a given suggestion. - mojo_base.mojom.UnguessableToken id; - - // Allows us to differentiate between a typical Assistant suggestion and an - // Assistant suggestion of another type (e.g. a conversation starter). - AssistantSuggestionType type = kUnspecified; - - // Display text. e.g. "Cancel". - string text; - - // Optional URL for icon. e.g. "https://www.gstatic.com/icon.png". - url.mojom.Url icon_url; - - // Optional URL for action. e.g. - // "https://www.google.com/search?query=action". - url.mojom.Url action_url; -}; - -// Models status of an app. -enum AppStatus { UNKNOWN, AVAILABLE, UNAVAILABLE, VERSION_MISMATCH, DISABLED }; - -// Models an Android app. -struct AndroidAppInfo { - // Unique name to identify a specific app. - string package_name; - - // Version number of the app. - int32 version; - - // Localized app name. - string localized_app_name; - - // Intent data to operate on. - string intent; - - // Status of the app. - AppStatus status; - - // The general action to be performed, such as ACTION_VIEW, ACTION_MAIN, etc. - string action; -}; - -// Details for Assistant feedback. -struct AssistantFeedback { - // User input to be sent with the feedback report. - string description; - - // Whether user consent to send debug info. - bool assistant_debug_info_allowed; - - // Screenshot if allowed by user. - // Raw data (non-encoded binary octets) - string screenshot_png; -}; diff --git a/chromium/chromeos/services/assistant/public/mojom/assistant_notification.mojom b/chromium/chromeos/services/assistant/public/mojom/assistant_notification.mojom index 0ddfe4b1c08..e10f8f61360 100644 --- a/chromium/chromeos/services/assistant/public/mojom/assistant_notification.mojom +++ b/chromium/chromeos/services/assistant/public/mojom/assistant_notification.mojom @@ -7,16 +7,6 @@ module chromeos.assistant.mojom; import "mojo/public/mojom/base/time.mojom"; import "url/mojom/url.mojom"; -// TODO(b:153495778): Remove notification types. -// Enumeration of notification types. -enum AssistantNotificationType { - // A notification of type |kSystem| will only be displayed within the Message - // Center. It is immediately added to the Message Center regardless of - // Assistant UI visibility state, although it may not be added if the user has - // opted out of seeing system notifications. - kSystem, -}; - // Models a notification button. struct AssistantNotificationButton { // Display text of the button. @@ -24,13 +14,21 @@ struct AssistantNotificationButton { // Optional URL to open when the tap action is invoked on the button. url.mojom.Url action_url; + + // If |true|, the associated notification will be removed on button click. + bool remove_notification_on_click = true; +}; + +// Enumeration of possible notification priorities. +// NOTE: This enum maps to a subset of message_center::NotificationPriority. +enum AssistantNotificationPriority { + kLow, // See message_center::NotificationPriority::LOW_PRIORITY. + kDefault, // See message_center::NotificationPriority::DEFAULT_PRIORITY. + kHigh, // See message_center::NotificationPriority::HIGH_PRIORITY. }; // Models an Assistant notification. struct AssistantNotification { - // Type of the notification. - AssistantNotificationType type = AssistantNotificationType.kSystem; - // Title of the notification. string title; @@ -60,10 +58,23 @@ struct AssistantNotification { // Obfuscated Gaia id of the intended recipient of the user. string obfuscated_gaia_id; - // Whether this notification can turn on the display if it was off. - bool is_high_priority = false; + // Priority for the notification. This affects whether or not the notification + // will pop up and/or have the capability to wake the display if it was off. + AssistantNotificationPriority priority = kDefault; // When the notification should expire. // Expressed as milliseconds since Unix Epoch. mojo_base.mojom.Time? expiry_time; + + // If |true|, the notification will be removed on click. + bool remove_on_click = true; + + // If |true|, the user can't remove the notification. + bool is_pinned = false; + + // If |true|, this notification was sent from the server. Clicking a + // notification that was sent from the server may result in a request to the + // server to retrieve the notification payload. One example of this would be + // notifications triggered by an Assistant reminder. + bool from_server = false; }; diff --git a/chromium/chromeos/services/cros_healthd/public/mojom/cros_healthd_probe.mojom b/chromium/chromeos/services/cros_healthd/public/mojom/cros_healthd_probe.mojom index 03759429c79..ab217747726 100644 --- a/chromium/chromeos/services/cros_healthd/public/mojom/cros_healthd_probe.mojom +++ b/chromium/chromeos/services/cros_healthd/public/mojom/cros_healthd_probe.mojom @@ -15,6 +15,8 @@ module chromeos.cros_healthd.mojom; enum CpuArchitectureEnum { kUnknown, kX86_64, + kAArch64, + kArmv7l, }; // An enumeration of each category of information that cros_healthd can report. @@ -29,6 +31,7 @@ enum ProbeCategoryEnum { kBacklight, kFan, kStatefulPartition, + kBluetooth, }; // An enumeration of the different categories of errors that can occur when @@ -129,6 +132,21 @@ struct NonRemovableBlockDeviceInfo { string name; // PSN: Product serial number, 32 bits uint32 serial; + // Bytes read since last boot. + uint64 bytes_read_since_last_boot; + // Bytes written since last boot. + uint64 bytes_written_since_last_boot; + // Time spent reading since last boot. + uint64 read_time_seconds_since_last_boot; + // Time spent writing since last boot. + uint64 write_time_seconds_since_last_boot; + // Time spent doing I/O since last boot. Counts the time the disk and queue + // were busy, so unlike the fields above, parallel requests are not counted + // multiple times. + uint64 io_time_seconds_since_last_boot; + // Time spent discarding since last boot. Discarding is writing to clear + // blocks which are no longer in use. Supported on kernels 4.18+. + UInt64Value? discard_time_seconds_since_last_boot; }; // Cached VPD probe result. Can either be populated with the CachedVpdInfo or an @@ -287,6 +305,27 @@ struct StatefulPartitionInfo { uint64 total_space; }; +// Bluetooth probe result. Can either be populated with the BluetoothAdapterInfo +// or an error retrieving the information. +union BluetoothResult { + // Valid BluetoothAdapterInfo. + array<BluetoothAdapterInfo> bluetooth_adapter_info; + // The error that occurred attempting to retrieve the BluetoothAdapterInfo. + ProbeError error; +}; + +// Information related to one of a device's Bluetooth adapters. +struct BluetoothAdapterInfo { + // The name of the adapter. + string name; + // The MAC address of the adapter. + string address; + // Indicates whether the adapter is on or off. + bool powered; + // The number of devices connected to this adapter. + uint32 num_connected_devices; +}; + // A collection of all the device's telemetry information that cros_healthd is // capable of reporting. Note that every field in TelemetryInfo is nullable, and // the response for a particular ProbeTelemetryInfo request will only contain @@ -324,4 +363,7 @@ struct TelemetryInfo { // kStatefulPartition was included in the categories input to // ProbeTelemetryInfo. StatefulPartitionResult? stateful_partition_result; + // Information about the device's Bluetooth adapters and devices. Only present + // when kBluetooth was included in the categories input to ProbeTelemetryInfo. + BluetoothResult? bluetooth_result; }; diff --git a/chromium/chromeos/services/ime/public/cpp/shared_lib/BUILD.gn b/chromium/chromeos/services/ime/public/cpp/shared_lib/BUILD.gn index 84993c890a5..51ca7deb6fe 100644 --- a/chromium/chromeos/services/ime/public/cpp/shared_lib/BUILD.gn +++ b/chromium/chromeos/services/ime/public/cpp/shared_lib/BUILD.gn @@ -5,12 +5,3 @@ source_set("interfaces") { sources = [ "interfaces.h" ] } - -shared_library("fake_shared_lib") { - testonly = true - sources = [ "fake_shared_lib.cc" ] - deps = [ "//chromeos/services/ime/public/cpp/shared_lib:interfaces" ] - - # This is the name expected by DecoderEngine. - output_name = "imedecoder" -} diff --git a/chromium/chromeos/services/machine_learning/public/mojom/BUILD.gn b/chromium/chromeos/services/machine_learning/public/mojom/BUILD.gn index 183467cef9e..c0926a2d478 100644 --- a/chromium/chromeos/services/machine_learning/public/mojom/BUILD.gn +++ b/chromium/chromeos/services/machine_learning/public/mojom/BUILD.gn @@ -7,6 +7,7 @@ import("//mojo/public/tools/bindings/mojom.gni") mojom("mojom") { sources = [ "graph_executor.mojom", + "handwriting_recognizer.mojom", "machine_learning_service.mojom", "model.mojom", "tensor.mojom", diff --git a/chromium/chromeos/services/machine_learning/public/mojom/handwriting_recognizer.mojom b/chromium/chromeos/services/machine_learning/public/mojom/handwriting_recognizer.mojom new file mode 100644 index 00000000000..49af2a37f62 --- /dev/null +++ b/chromium/chromeos/services/machine_learning/public/mojom/handwriting_recognizer.mojom @@ -0,0 +1,134 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Datatypes and interfaces of handwriting recognition API. + +// NOTE: This mojom exists in two places and must be kept in sync: +// Chromium: //chromeos/services/machine_learning/public/mojom/ +// Chrome OS: src/platform2/ml/mojom/ +// Note: Other repos downstream of Chromium might also use this mojom. +// Example: A backwards-compatible mojom change (and corresponding +// implementation change) can be made in Chrome OS first, then replicated to the +// clients (Chromium, other downstream repos) later. +// Use //chromeos/services/machine_learning/public/mojom/roll_mojom.sh to help +// replicate Chrome OS-side changes over to Chromium. + +module chromeos.machine_learning.mojom; + +import "mojo/public/mojom/base/time.mojom"; + +// A single point in an ink stroke. +struct InkPoint { + // (x, y) coordinates of the point. The upper-left corner of the writing area + // is (0, 0). The positive x-axis points to the right. The positive y-axis + // points down. + float x; + float y; + // Optional TimeDelta from the point was captured relative to the beginning of + // the ink. So first point of a request should have t=0. + mojo_base.mojom.TimeDelta? t; +}; + +// A single ink stroke. +struct InkStroke { + // The set of points representing the stroke. + array<InkPoint> points; +}; + +// The writing guide shown to the user during input. +struct WritingGuide { + // Size of the writing area. The writing area must have (0, 0) at the top + // left. + // width is the max value of x-axis (to the right) and height is the max + // value of y-axis (to the bottom). + float width; + float height; +}; + +// Options that will influence the recognition output. +struct RecognitionContext { + // The writing guide shown to the user when the ink was captured. + WritingGuide? writing_guide; + // The context before the written text. + string? pre_context; +}; + +// Defines a handwriting recognition query. +struct HandwritingRecognitionQuery { + // The set of ink strokes to be recognized. + array<InkStroke> ink; + // The optional context in which the ink was generated. + RecognitionContext? context; + // Maximum number of results to be returned. + uint32 max_num_results; + // If true, returns the segmentation results. This will make the response + // struct much larger. + bool return_segmentation; +}; + +// Represents a contiguous range of ink. Start and end strokes / points are +// inclusive. +struct HandwritingRecognizerInkRange { + // Zero-based start- and end-stroke indices in the ink strokes. + uint32 start_stroke; + uint32 end_stroke; + // Zero-based start- and end-point indices with the start / end strokes. + uint32 start_point; + uint32 end_point; +}; + +// A segment defines the substrokes that are associated with a substring of the +// recognized text. +struct HandwritingRecognizerSegment { + // The substring of the recognized text represented by this segment. + string sublabel; + // The ink ranges represented by this segment. + array<HandwritingRecognizerInkRange> ink_ranges; +}; + +// The ink segmentation information. +struct HandwritingRecognizerSegmentation { + // The grouping of the cut strokes into characters. + array<HandwritingRecognizerSegment> segments; +}; + +// One possible candidate from the handwriting recognition. +struct HandwritingRecognizerCandidate { + // The recognized text. + string text; + // Scores will most often correspond to -log(p(c)) or a scaling thereof, so + // lower scores represent higher confidence. + float score; + // The stroke segmentation that was used to generate the result. + HandwritingRecognizerSegmentation? segmentation; +}; + +// The handwriting recognition response. +struct HandwritingRecognizerResult { + // Status of the recognition response. + enum Status { + OK = 0, + ERROR = 1, + }; + Status status; + + // The recognition candidates with additional alternatives, sorted by lower + // score first (lower score present higher confidence). + array<HandwritingRecognizerCandidate> candidates; +}; + +// The specification of a handwriting recognizer. +struct HandwritingRecognizerSpec { + // The language the recognizer will handle. Only "en" (for english) and + // "gesture_in_context" (for gesture) are supported. + string language; +}; + +// The mojom interface for performing the recognition of handwritten text. +interface HandwritingRecognizer { + // Performs handwriting recognition on a set of ink strokes, and returns a set + // of alternative recognition results. + Recognize(HandwritingRecognitionQuery query) => + (HandwritingRecognizerResult result); +}; diff --git a/chromium/chromeos/services/machine_learning/public/mojom/machine_learning_service.mojom b/chromium/chromeos/services/machine_learning/public/mojom/machine_learning_service.mojom index 6819140f7a2..c22ba02f82c 100644 --- a/chromium/chromeos/services/machine_learning/public/mojom/machine_learning_service.mojom +++ b/chromium/chromeos/services/machine_learning/public/mojom/machine_learning_service.mojom @@ -7,14 +7,18 @@ // NOTE: This mojom exists in two places and must be kept in sync: // Chromium: //chromeos/services/machine_learning/public/mojom/ // Chrome OS: src/platform2/ml/mojom/ +// Note: Other repos downstream of Chromium might also use this mojom. // Example: A backwards-compatible mojom change (and corresponding // implementation change) can be made in Chrome OS first, then replicated to the -// client (Chromium) later. +// clients (Chromium, other downstream repos) later. +// Use //chromeos/services/machine_learning/public/mojom/roll_mojom.sh to help +// replicate Chrome OS-side changes over to Chromium. module chromeos.machine_learning.mojom; // NOTE: The base directory for 'import' statements is expected to differ // between Chromium and Chrome OS versions of this file. +import "chromeos/services/machine_learning/public/mojom/handwriting_recognizer.mojom"; import "chromeos/services/machine_learning/public/mojom/model.mojom"; import "chromeos/services/machine_learning/public/mojom/text_classifier.mojom"; @@ -24,6 +28,8 @@ enum LoadModelResult { OK = 0, MODEL_SPEC_ERROR = 1, LOAD_MODEL_ERROR = 2, + FEATURE_NOT_SUPPORTED_ERROR = 3, + LANGUAGE_NOT_SUPPORTED_ERROR = 4, }; // Top-level interface between Chromium and the ML Service daemon. @@ -40,4 +46,11 @@ interface MachineLearningService { // Create a new TextClassifier. LoadTextClassifier(pending_receiver<TextClassifier> receiver) => (LoadModelResult result); + // Create and initialize a handwriting recognizer. + LoadHandwritingModel(pending_receiver<HandwritingRecognizer> receiver) + => (LoadModelResult result); + // Create and initialize a handwriting recognizer with given |spec|. + LoadHandwritingModelWithSpec(HandwritingRecognizerSpec spec, + pending_receiver<HandwritingRecognizer> receiver) + => (LoadModelResult result); }; diff --git a/chromium/chromeos/services/network_config/public/mojom/cros_network_config.mojom b/chromium/chromeos/services/network_config/public/mojom/cros_network_config.mojom index 02d6cab680d..ba5dc03d3e6 100644 --- a/chromium/chromeos/services/network_config/public/mojom/cros_network_config.mojom +++ b/chromium/chromeos/services/network_config/public/mojom/cros_network_config.mojom @@ -547,6 +547,9 @@ struct ManagedWiFiProperties { string? tethering_state; // Indicates whether the network is eligible to be included in Chrome Sync. bool is_syncable; + // True for unshared networks and for shared networks configured by the + // active user. Updated whenever the passphrase is modified. + bool is_configured_by_active_user; }; union NetworkTypeManagedProperties { @@ -571,6 +574,7 @@ struct ManagedProperties { string guid; ManagedString? ip_address_config_type; array<IPConfigProperties>? ip_configs; + ManagedBoolean? metered; ManagedString? name; ManagedString? name_servers_config_type; ManagedInt32? priority; @@ -596,6 +600,11 @@ struct AutoConnectConfig { bool value; }; +// Wrapper to allow optional metered configuration. +struct MeteredConfig { + bool value; +}; + // Wrapper to allow optional priority configuration. struct PriorityConfig { int32 value; @@ -722,6 +731,7 @@ struct ConfigProperties { // preserve the existing guid. string? guid; string? ip_address_config_type; + MeteredConfig? metered; // Name is only required for new configurations. string? name; string? name_servers_config_type; @@ -883,6 +893,9 @@ interface CrosNetworkConfig { // Requests a scan for new networks. If the list updates, // CrosNetworkConfigObserver::OnNetworkStateListChanged() will be signaled. + // Note: If |type| is Cellular, a mobile network scan will be requested + // if supported. This is disruptive and should only be triggered by an + // explicit user action. RequestNetworkScan(NetworkType type); // Returns the global policy properties. These properties are not expected to diff --git a/chromium/chromeos/services/network_config/public/mojom/network_types.mojom b/chromium/chromeos/services/network_config/public/mojom/network_types.mojom index 6274e0ee5c6..f193fb2d011 100644 --- a/chromium/chromeos/services/network_config/public/mojom/network_types.mojom +++ b/chromium/chromeos/services/network_config/public/mojom/network_types.mojom @@ -19,6 +19,7 @@ enum ConnectionStateType { enum DeviceStateType { kUninitialized, kDisabled, + kDisabling, kEnabling, kEnabled, kProhibited, diff --git a/chromium/chromeos/services/network_health/public/mojom/BUILD.gn b/chromium/chromeos/services/network_health/public/mojom/BUILD.gn new file mode 100644 index 00000000000..0b8abc04bdc --- /dev/null +++ b/chromium/chromeos/services/network_health/public/mojom/BUILD.gn @@ -0,0 +1,17 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//mojo/public/tools/bindings/mojom.gni") + +mojom("mojom") { + sources = [ + "network_diagnostics.mojom", + "network_health.mojom", + ] + + public_deps = [ + "//chromeos/services/network_config/public/mojom:network_types", + "//mojo/public/mojom/base", + ] +} diff --git a/chromium/chromeos/services/network_health/public/mojom/network_diagnostics.mojom b/chromium/chromeos/services/network_health/public/mojom/network_diagnostics.mojom new file mode 100644 index 00000000000..bbb103eb175 --- /dev/null +++ b/chromium/chromeos/services/network_health/public/mojom/network_diagnostics.mojom @@ -0,0 +1,111 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +module chromeos.network_diagnostics.mojom; + +// Each routine can result in one of the possible verdicts. +[Extensible] +enum RoutineVerdict { + // Routine ran successfully and no connectivity problem found. + kNoProblem, + // Routine ran successfully and connectivity problem found. + kProblem, + // Routine has not been run. + kNotRun, +}; + +// Problems related to the SignalStrength routine. +[Extensible] +enum SignalStrengthProblem { + kSignalNotFound, + kWeakSignal, +}; + +// Problems related to the GatewayCanBePinged routine. +[Extensible] +enum GatewayCanBePingedProblem { + // All gateways are unreachable, hence cannot be pinged. + kUnreachableGateway, + // The default network cannot be pinged. + kFailedToPingDefaultNetwork, + // The default network has a latency above the threshold. + kDefaultNetworkAboveLatencyThreshold, + // One or more of the non-default networks has failed pings. + kUnsuccessfulNonDefaultNetworksPings, + // One of more of the non-default networks has a latency above the threshold. + kNonDefaultNetworksAboveLatencyThreshold, +}; + +// Messages related to the HasSecureWiFiConnection routine. +[Extensible] +enum HasSecureWiFiConnectionProblem { + kSecurityTypeNone, + kSecurityTypeWep8021x, + kSecurityTypeWepPsk, + kUnknownSecurityType, +}; + +// Messages related to the DnsResolverPresent routine. +[Extensible] +enum DnsResolverPresentProblem { + kNoNameServersFound, + kMalformedNameServers, + kEmptyNameServers, +}; + +// Messages related to the DnsLatencyProblem routine. +[Extensible] +enum DnsLatencyProblem { + // The routine was unable to successfully resolve all the hosts. Note that + // this will be true if one or more hosts could not be successfully resolved. + kFailedToResolveAllHosts, + // Average DNS latency across hosts is slightly above expected threshold + kSlightlyAboveThreshold, + // Average DNS latency across hosts is significantly above expected threshold + kSignificantlyAboveThreshold, +}; + +// Messages related to the DnsResolution routine. +[Extensible] +enum DnsResolutionProblem { + // The routine was unable to successfully resolve the test host + kFailedToResolveHost, +}; + +// This interface is to be used by any clients that need to run specific +// network-related diagnostics. Expected clients of this interface are +// NetworkHealth, cros_healthd, and a connectivity diagnostics Web UI (to name +// a few). The bound implementation is intended to live in the browser process. +interface NetworkDiagnosticsRoutines { + // Tests whether the device is connected to a LAN. It is possible that the + // device may be trapped in a captive portal yet pass this test successfully. + // Captive portal checks are done separately and are outside of the scope of + // this routine. + LanConnectivity() => (RoutineVerdict verdict); + + // Tests whether there is an acceptable signal strength on wireless networks. + SignalStrength() => (RoutineVerdict verdict, + array<SignalStrengthProblem> problems); + + // Tests whether the gateway of connected networks is pingable. + GatewayCanBePinged() => (RoutineVerdict verdict, + array<GatewayCanBePingedProblem> problems); + + // Tests whether the WiFi connection is secure. Note that if WiFi is not + // connected, the routine will result in a |kNotRun| verdict. + HasSecureWiFiConnection() => (RoutineVerdict verdict, + array<HasSecureWiFiConnectionProblem> problems); + + // Tests whether a DNS resolver is available to the browser. + DnsResolverPresent() => (RoutineVerdict verdict, + array<DnsResolverPresentProblem> problems); + + // Tests whether the DNS latency is below an acceptable threshold. + DnsLatency() => (RoutineVerdict verdict, + array<DnsLatencyProblem> problem); + + // Tests whether a DNS resolution can be completed successfully. + DnsResolution() => (RoutineVerdict verdict, + array<DnsResolutionProblem> problems); +}; diff --git a/chromium/chromeos/services/network_health/public/mojom/network_health.mojom b/chromium/chromeos/services/network_health/public/mojom/network_health.mojom new file mode 100644 index 00000000000..6c8d474e635 --- /dev/null +++ b/chromium/chromeos/services/network_health/public/mojom/network_health.mojom @@ -0,0 +1,64 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// API intended for retrieving a snapshot of the network health state. Returns +// properties of any available network technologies and any connected or +// connecting network when available. + +module chromeos.network_health.mojom; + +import "chromeos/services/network_config/public/mojom/network_types.mojom"; + +enum NetworkState { + // The network type is available but not yet initialized. + kUninitialized, + // The network type is available but disabled or disabling. + kDisabled, + // The network type is prohibited by policy. + kProhibited, + // The network type is available and enabled or enabling, but no network + // connection has been established. + kNotConnected, + // The network type is available and enabled, and a network connection is + // in progress. + kConnecting, + // The network is in a portal state. + kPortal, + // The network is in a connected state, but connectivity is limited. + kConnected, + // The network is connected and online. + kOnline, +}; + +// Contains information for a single network connection and underlying +// network technology e.g WiFi. +struct Network { + chromeos.network_config.mojom.NetworkType type; + NetworkState state; + // The unique identifier for the network when a network service exists. + string? guid; + // The user facing name of the network if available. + string? name; + // Optional string for the network's mac_address. + string? mac_address; +}; + +// Aggregate structure for all of the network health state. +struct NetworkHealthState { + // Networks will be sorted with active connections listed first. + array<Network> networks; +}; + +// Interface for retrieving aggregated information about the current network +// state and health. +interface NetworkHealthService { + // Returns a list of all the networks. Networks will be sorted with active + // connections listed first. + GetNetworkList() => (array<Network> networks); + + // Returns the current network health state. + // This is currently the same information provided by GetNetworkList. More + // information will be added over time. + GetHealthSnapshot() => (NetworkHealthState state); +}; diff --git a/chromium/chromeos/services/tts/BUILD.gn b/chromium/chromeos/services/tts/BUILD.gn new file mode 100644 index 00000000000..e2feb37c081 --- /dev/null +++ b/chromium/chromeos/services/tts/BUILD.gn @@ -0,0 +1,58 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//tools/generate_library_loader/generate_library_loader.gni") + +source_set("tts") { + sources = [ + "constants.cc", + "constants.h", + "tts_service.cc", + "tts_service.h", + ] + + deps = [ + ":libchrometts", + "//base", + "//chromeos/services/tts/public/mojom", + ] +} + +source_set("sandbox_hook") { + sources = [ + "tts_sandbox_hook.cc", + "tts_sandbox_hook.h", + ] + + deps = [ + ":libchrometts", + ":tts", + "//base", + "//sandbox/linux:sandbox_services", + "//services/service_manager/sandbox:sandbox", + ] +} + +generate_library_loader("libchrometts") { + name = "LibChromeTtsLoader" + output_h = "libchrometts.h" + output_cc = "libchrometts_loader.cc" + header = "<chrome_tts.h>" + bundled_header = "\"chromeos/services/tts/chrome_tts.h\"" + + functions = [ + "GoogleTtsSetLogger", + "GoogleTtsInit", + "GoogleTtsShutdown", + "GoogleTtsInstallVoice", + "GoogleTtsInitBuffered", + "GoogleTtsReadBuffered", + "GoogleTtsFinalizeBuffered", + "GoogleTtsGetEventBufferPtr", + "GoogleTtsGetEventBufferLen", + "GoogleTtsGetTimepointsCount", + "GoogleTtsGetTimepointsTimeInSecsAtIndex", + "GoogleTtsGetTimepointsCharIndexAtIndex", + ] +} diff --git a/chromium/chromeos/services/tts/public/mojom/BUILD.gn b/chromium/chromeos/services/tts/public/mojom/BUILD.gn new file mode 100644 index 00000000000..b677781871c --- /dev/null +++ b/chromium/chromeos/services/tts/public/mojom/BUILD.gn @@ -0,0 +1,9 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//mojo/public/tools/bindings/mojom.gni") + +mojom("mojom") { + sources = [ "tts_service.mojom" ] +} diff --git a/chromium/chromeos/services/tts/public/mojom/tts_service.mojom b/chromium/chromeos/services/tts/public/mojom/tts_service.mojom new file mode 100644 index 00000000000..a8fd1cdb6cf --- /dev/null +++ b/chromium/chromeos/services/tts/public/mojom/tts_service.mojom @@ -0,0 +1,85 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +module chromeos.tts.mojom; + +// Structure describing a point in time during speech synthesis. +struct Timepoint { + // The time, in seconds. + float time_sec; + + // The index in the text being spoken. + int32 char_index; +}; + +// A TTS event and associated metadata within a TTS stream. +struct TtsStreamItem { + // An internal serialized proto.speech.tts.TtsControllerEvent proto. + array<uint8> event_buffer_bytes; + + // Whether streaming is complete. + bool done; + + // A list of timepoints associated with the event above. + array<Timepoint> timepoints; +}; + +// The main interface to the TTS engine on Chrome OS. Only used by and private +// to the Chrome OS Google TTS engine component extension. TtsService lives in a +// tts-sandboxed process. TtsEngineExtensionObserver, the other end of this +// interface, in the browser process, brokers a connection between TtsService +// and the Google TTS engine component extension through a TtsStream, but does +// not participate otherwise. +interface TtsService { + // Binds a TtsStream to this service. + BindTtsStream(pending_receiver<TtsStream> receiver); +}; + +// Interface for the Google component TTS engine to control and consume a stream +// of TtsStreamItems produced by TtsService. There is only ever one TtsStream +// owned by the TtsService. +// +// The component extension sets up the stream's voice by doing: +// InstallVoice(data, "voice") +// InstallVoice(other_data, "other_voice") +// SelectVoice("other_voice") +// +// After reading from the stream (see below), the component extension can do: +// SelectVoice("voice") +// to change voices. +// +// The component extension calls the following three methods repeatedly, in +// order to read from the stream given text. For example, +// +// Init(<a proto containing text "Hello there.">) +// Read() +// Read() +// ... +// Finalize() +// Init(<proto containing text "Testing 1, 2, 3.") +// Read() +// Read() +// ... +// Finalize() +// +// Note that the component extension may call Finalize() early, if the TTS api +// wants to, for example, stop speech. +interface TtsStream { + // Forward and install the |voice_name| encoded by |voice_bytes|. + InstallVoice(string voice_name, array<uint8> voice_bytes) + => (bool success); + + // Selects a voice for streaming given a |voice_name|. + SelectVoice(string voice_name) => (bool success); + + // Initialize a new TTS stream given a serialized proto.speech.tts.Text proto. + Init(array<uint8> text_jspb) + => (bool success); + + // Read the next stream item. + Read() => (TtsStreamItem item); + + // Clean up and finish the current TTS stream. + Finalize(); +}; |