docs: deprecation generation support [1/2] (#2013)

docs: deprecation version generation support

schema: Add the following metadata keys:
 - changed_version
 - deprecated_version
 - new_version
 - changed_description
 - new_description
 - deprecation_description
 - changed
 - new

Generate formatted docs from this new metadata.
Update current descriptions containing unstructured data.
Switch current deprecation from a date to deprecated version.
Ensure changed/new/deprecated keys have associated version keys.

2 files changed, 120 insertions, 59 deletions

diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index dcaf13af..b0b5fccf 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -1,6 +1,5 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 """schema.py: Set of module functions for processing cloud-config schema."""
-
 import argparse
 import json
 import logging
@@ -11,6 +10,7 @@ import textwrap
 from collections import defaultdict
 from collections.abc import Iterable
 from copy import deepcopy
+from functools import partial
 from itertools import chain
 from typing import TYPE_CHECKING, List, NamedTuple, Optional, Type, Union, cast
 
@@ -162,17 +162,47 @@ def is_schema_byte_string(checker, instance):
     ) or isinstance(instance, (bytes,))
 
 
-def _add_deprecation_msg(description: Optional[str] = None) -> str:
-    if description:
-        return f"{DEPRECATED_PREFIX}{description}"
-    return DEPRECATED_PREFIX.replace(":", ".").strip()
+def _add_deprecated_changed_or_new_msg(
+    config: dict, annotate=False, filter_key=None
+) -> str:
+    """combine description with new/changed/deprecated message
+
+    deprecated/changed/new keys require a _version key (this is verified
+    in a unittest), a _description key is optional
+    """
+
+    def format_message(key: str):
+        if not config.get(f"{key}"):
+            return ""
+        key_description = config.get(f"{key}_description", "")
+        v = config.get(
+            f"{key}_version",
+            f"<missing {key}_version key, please file a bug report>",
+        )
+        msg = f"{key.capitalize()} in version {v}. {key_description}"
+        if annotate:
+            return f" {msg}"
 
+        # italicised RST - no whitespace between astrisk and text
+        return f"\n\n*{msg.strip()}*"
 
-def _validator_deprecated(
+    # define print order
+    filter_keys = (
+        filter_key if filter_key else ["deprecated", "changed", "new"]
+    )
+
+    # build a deprecation/new/changed string
+    changed_new_deprecated = "".join(map(format_message, filter_keys))
+    description = config.get("description", "")
+    return f"{description}{changed_new_deprecated}".rstrip()
+
+
+def _validator(
     _validator,
     deprecated: bool,
     _instance,
     schema: dict,
+    filter_key: str,
     error_type: Type[Exception] = SchemaDeprecationError,
 ):
     """Jsonschema validator for `deprecated` items.
@@ -181,11 +211,16 @@ def _validator_deprecated(
     otherwise the instance is consider faulty.
     """
     if deprecated:
-        description = schema.get("description")
-        msg = _add_deprecation_msg(description)
+        msg = _add_deprecated_changed_or_new_msg(
+            schema, annotate=True, filter_key=[filter_key]
+        )
         yield error_type(msg)
 
 
+_validator_deprecated = partial(_validator, filter_key="deprecated")
+_validator_changed = partial(_validator, filter_key="changed")
+
+
 def _anyOf(
     validator,
     anyOf,
@@ -314,6 +349,7 @@ def get_jsonschema_validator():
     # Add deprecation handling
     validators = dict(Draft4Validator.VALIDATORS)
     validators[DEPRECATED_KEY] = _validator_deprecated
+    validators["changed"] = _validator_changed
     validators["oneOf"] = _oneOf
     validators["anyOf"] = _anyOf
 
@@ -843,32 +879,35 @@ def _get_property_description(prop_config: dict) -> str:
     Order and deprecated property description after active descriptions.
     Add a trailing stop "." to any description not ending with ":".
     """
-    prop_descr = prop_config.get("description", "")
+
+    def assign_descriptions(
+        config: dict, descriptions: list, deprecated_descriptions: list
+    ):
+        if any(
+            map(
+                config.get,
+                ("deprecated_version", "changed_version", "new_version"),
+            )
+        ):
+            deprecated_descriptions.append(
+                _add_deprecated_changed_or_new_msg(config)
+            )
+        elif config.get("description"):
+            descriptions.append(_add_deprecated_changed_or_new_msg(config))
+
     oneOf = prop_config.get("oneOf", {})
     anyOf = prop_config.get("anyOf", {})
-    descriptions = []
-    deprecated_descriptions = []
-    if prop_descr:
-        prop_descr = prop_descr.rstrip(".")
-        if not prop_config.get(DEPRECATED_KEY):
-            descriptions.append(prop_descr)
-        else:
-            deprecated_descriptions.append(_add_deprecation_msg(prop_descr))
+    descriptions: list = []
+    deprecated_descriptions: list = []
+
+    assign_descriptions(prop_config, descriptions, deprecated_descriptions)
     for sub_item in chain(oneOf, anyOf):
-        if not sub_item.get("description"):
-            continue
-        if not sub_item.get(DEPRECATED_KEY):
-            descriptions.append(sub_item["description"].rstrip("."))
-        else:
-            deprecated_descriptions.append(
-                f"{DEPRECATED_PREFIX}{sub_item['description'].rstrip('.')}"
-            )
+        assign_descriptions(sub_item, descriptions, deprecated_descriptions)
+
     # order deprecated descrs last
     description = ". ".join(chain(descriptions, deprecated_descriptions))
     if description:
         description = f" {description}"
-        if description[-1] != ":":
-            description += "."
     return description
 
 
diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json
index d227e159..8bd7a80c 100644
--- a/cloudinit/config/schemas/schema-cloud-config-v1.json
+++ b/cloudinit/config/schemas/schema-cloud-config-v1.json
@@ -174,6 +174,7 @@
                   "label": "<group_name>",
                   "description": "When providing an object for users.groups the ``<group_name>`` keys are the groups to add this user to",
                   "deprecated": true,
+                  "deprecated_version": "23.1",
                   "type": [
                     "null"
                   ],
@@ -197,9 +198,11 @@
         },
         "lock-passwd": {
           "default": true,
-          "description": "Dropped after April 2027. Use ``lock_passwd``. Default: ``true``",
           "type": "boolean",
-          "deprecated": true
+          "description": "Default: ``true``",
+          "deprecated": true,
+          "deprecated_version": "22.3",
+          "deprecated_description": "Use ``lock_passwd`` instead."
         },
         "lock_passwd": {
           "default": true,
@@ -292,8 +295,9 @@
             },
             {
               "type": "boolean",
-              "deprecated": true,
-              "description": "The value ``false`` will be dropped after April 2027. Use ``null`` or no ``sudo`` key instead."
+              "changed": true,
+              "changed_version": "22.2",
+              "changed_description": "The value ``false`` is deprecated for this key, use ``null`` instead."
             }
           ]
         },
@@ -305,8 +309,9 @@
             },
             {
               "type": "string",
-              "description": "The use of ``string`` type will be dropped after April 2027. Use an ``integer`` instead.",
-              "deprecated": true
+              "changed": true,
+              "changed_description": "The use of ``string`` type is deprecated. Use an ``integer`` instead.",
+              "changed_version": "22.3"
             }
           ]
         }
@@ -361,10 +366,11 @@
       "additionalProperties": false,
       "properties": {
         "remove-defaults": {
-          "description": "Dropped after April 2027. Use ``remove_defaults``.",
           "type": "boolean",
           "default": false,
-          "deprecated": true
+          "deprecated": true,
+          "deprecated_version": "22.3",
+          "deprecated_description": "Use ``remove_defaults`` instead."
         },
         "remove_defaults": {
           "description": "Remove default CA certificates if true. Default: false",
@@ -920,7 +926,8 @@
             },
             {
               "deprecated": true,
-              "description": "Dropped after April 2027. Use ``ca_certs``."
+              "deprecated_version": "22.3",
+              "deprecated_description": "Use ``ca_certs`` instead."
             }
           ]
         }
@@ -1294,8 +1301,9 @@
                   "enum": [
                     false
                   ],
-                  "description": "Specifying a boolean ``false`` value for this key is deprecated. Use ``off`` instead.",
-                  "deprecated": true
+                  "changed": true,
+                  "changed_version": "22.3",
+                  "changed_description": "Specifying a boolean ``false`` value for ``mode`` is deprecated. Use ``off`` instead."
                 }
               ]
             },
@@ -1342,8 +1350,9 @@
                 },
                 {
                   "type": "string",
-                  "description": "Use a boolean value instead.",
-                  "deprecated": true
+                  "changed": true,
+                  "changed_version": "22.3",
+                  "changed_description": "Use a boolean value instead."
                 }
               ]
             }
@@ -1351,8 +1360,9 @@
         },
         "grub-dpkg": {
           "type": "object",
-          "description": "Use ``grub_dpkg`` instead",
-          "deprecated": true
+          "deprecated": true,
+          "deprecated_version": "22.2",
+          "deprecated_description": "Use ``grub_dpkg`` instead."
         }
       }
     },
@@ -1918,20 +1928,26 @@
         "apt_update": {
           "type": "boolean",
           "default": false,
-          "description": "Dropped after April 2027. Use ``package_update``. Default: ``false``",
-          "deprecated": true
+          "description": "Default: ``false``.",
+          "deprecated": true,
+          "deprecated_version": "22.2",
+          "deprecated_description": "Use ``package_update`` instead."
         },
         "apt_upgrade": {
           "type": "boolean",
           "default": false,
-          "description": "Dropped after April 2027. Use ``package_upgrade``. Default: ``false``",
-          "deprecated": true
+          "description": "Default: ``false``.",
+          "deprecated": true,
+          "deprecated_version": "22.2",
+          "deprecated_description": "Use ``package_upgrade`` instead."
         },
         "apt_reboot_if_required": {
           "type": "boolean",
           "default": false,
-          "description": "Dropped after April 2027. Use ``package_reboot_if_required``. Default: ``false``",
-          "deprecated": true
+          "description": "Default: ``false``.",
+          "deprecated": true,
+          "deprecated_version": "22.2",
+          "deprecated_description": "Use ``package_reboot_if_required`` instead."
         }
       }
     },
@@ -2005,8 +2021,9 @@
                 {
                   "type": "string",
                   "pattern": "^\\+?[0-9]+$",
-                  "deprecated": true,
-                  "description": "Use of string for this value will be dropped after April 2027. Use ``now`` or integer type."
+                  "changed": true,
+                  "changed_version": "22.3",
+                  "changed_description": "Use of type string for this value is deprecated. Use ``now`` or integer type."
                 },
                 {
                   "enum": [
@@ -2424,8 +2441,9 @@
                 },
                 {
                   "type": "string",
-                  "description": "Use of string for this value is DEPRECATED. Use a boolean value instead.",
-                  "deprecated": true
+                  "deprecated": true,
+                  "deprecated_version": "22.3",
+                  "deprecated_description": "Use of type string for this value is deprecated. Use a boolean instead."
                 }
               ]
             },
@@ -2522,8 +2540,9 @@
             },
             {
               "type": "string",
-              "description": "Use of non-boolean values for this field is DEPRECATED and will result in an error in a future version of cloud-init.",
-              "deprecated": true
+              "changed": true,
+              "changed_version": "22.3",
+              "changed_description": "Use of non-boolean values for this field is deprecated."
             }
           ],
           "description": "Sets whether or not to accept password authentication. ``true`` will enable password auth. ``false`` will disable. Default is to leave the value unchanged. In order for this config to be applied, SSH may need to be restarted. On systemd systems, this restart will only happen if the SSH service has already been started. On non-systemd systems, a restart will be attempted regardless of the service state."
@@ -2538,7 +2557,7 @@
               "description": "Whether to expire all user passwords such that a password will need to be reset on the user's next login. Default: ``true``"
             },
             "users": {
-              "description": "Replaces the deprecated ``list`` key. This key represents a list of existing users to set passwords for. Each item under users contains the following required keys: ``name`` and ``password`` or in the case of a randomly generated password, ``name`` and ``type``. The ``type`` key has a default value of ``hash``, and may alternatively be set to ``text`` or ``RANDOM``.",
+              "description": "This key represents a list of existing users to set passwords for. Each item under users contains the following required keys: ``name`` and ``password`` or in the case of a randomly generated password, ``name`` and ``type``. The ``type`` key has a default value of ``hash``, and may alternatively be set to ``text`` or ``RANDOM``.",
               "type": "array",
               "items": {
                 "minItems": 1,
@@ -2602,8 +2621,10 @@
                 }
               ],
               "minItems": 1,
-              "description": "List of ``username:password`` pairs. Each user will have the corresponding password set. A password can be randomly generated by specifying ``RANDOM`` or ``R`` as a user's password. A hashed password, created by a tool like ``mkpasswd``, can be specified. A regex (``r'\\$(1|2a|2y|5|6)(\\$.+){2}'``) is used to determine if a password value should be treated as a hash.\n\nUse of a multiline string for this field is DEPRECATED and will result in an error in a future version of cloud-init.",
-              "deprecated": true
+              "description": "List of ``username:password`` pairs. Each user will have the corresponding password set. A password can be randomly generated by specifying ``RANDOM`` or ``R`` as a user's password. A hashed password, created by a tool like ``mkpasswd``, can be specified. A regex (``r'\\$(1|2a|2y|5|6)(\\$.+){2}'``) is used to determine if a password value should be treated as a hash.",
+              "deprecated": true,
+              "deprecated_version": "22.2",
+              "deprecated_description": "Use ``users`` instead."
             }
           }
         },
@@ -2966,8 +2987,9 @@
               "enum": [
                 "template"
               ],
-              "description": "Value ``template`` will be dropped after April 2027. Use ``true`` instead.",
-              "deprecated": true
+              "changed_description": "Use of ``template`` is deprecated, use ``true`` instead.",
+              "changed": true,
+              "changed_version": "22.3"
             }
           ]
         },