Autorelease/manifest (#469)

new release flow

4 files changed, 192 insertions, 142 deletions

diff --git a/.github/scripts/packager.py b/.github/scripts/packager.py
index 3d9379189..ed5057a59 100755
--- a/.github/scripts/packager.py
+++ b/.github/scripts/packager.py
@@ -37,30 +37,6 @@ LABS_RELATIVE_EXCLUDE_FILES = [
     os.path.join('.git')
 ]
 
-'''
-- Take inputs 
-    - version will be specified
-        - This will be used to name directory
-    - baseline zip. From last release
-        - Used to compare contents of last release
-- setup stage
-    - Create 'zipper-output' directory
-        - This will house the zip
-        - and the directory used to zip. 'out/unzipped-package'
-    - Unzip the input zip into 'out/baseline'
-    - Git clone recursive into latest FreeRTOS master from Git into 'out/head-master
-
-- process stage
-    - remove all RELATIVE_FILE_EXCLUDES from 'out/unzipped-package'
-    - perform a filetree diff between 'out/unzipped-package' and 'out/head-master'
-        - Save to a file, to be confirmed by user
-        - Present and query user to authorize the diff
-    - zip contents of 'out/unzipped-package' --> 'out/FreeRTOSVXX.YY.ZZ.zip
-        - Use 7z compression, with compression set to max
-    - calculate zip file size diff, present to user
-    - Done
-'''
-
 # -------------------------------------------------------------------------------------------------
 # Helpers
 # -------------------------------------------------------------------------------------------------
@@ -105,36 +81,37 @@ def download_git_tree(git_link, root_dir, dir_name, ref='master', commit_id='HEA
     Download HEAD from Git Master. Place into working files dir
     '''
     args = ['git', '-C', root_dir, 'clone', '-b', ref, git_link, dir_name]
-    rc = subprocess.run(args).returncode
-    rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'checkout', '-f', commit_id]).returncode
-    rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'clean', '-fd']).returncode
+    subprocess.run(args, check=True)
+    subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'checkout', '-f', commit_id], check=True)
+    subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'clean', '-fd'], check=True)
     if recurse:
-        rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'submodule', 'update', '--init', '--recursive']).returncode
-
-    return os.path.join(root_dir, dir_name) if rc == 0 else None
+        subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'submodule', 'update', '--init', '--recursive'], check=True)
 
+    return os.path.join(root_dir, dir_name)
 
 def commit_git_tree_changes(repo_dir, commit_message=''):
-    rc = subprocess.run(['git', '-C', repo_dir, 'add', '-u']).returncode
-    rc += subprocess.run(['git', '-C', repo_dir, 'commit', '-m', commit_message]).returncode
+    subprocess.run(['git', '-C', repo_dir, 'add', '-u'], check=True)
+    subprocess.run(['git', '-C', repo_dir, 'commit', '-m', commit_message], check=True)
 
-    return rc
+    return 0
 
 def push_git_tree_changes(repo_dir, tag=None, force_tag=False):
-    rc = subprocess.run(['git', '-C', repo_dir, 'push']).returncode
+    subprocess.run(['git', '-C', repo_dir, 'push'], check=True)
 
     if tag != None:
         force_tag_arg = '-f' if force_tag else ''
-        rc += subprocess.run(['git', '-C', repo_dir, 'tag', force_tag_arg, tag]).returncode
-        rc += subprocess.run(['git', '-C', repo_dir, 'push', force_tag_arg, '--tags']).returncode
+        subprocess.run(['git', '-C', repo_dir, 'tag', force_tag_arg, tag], check=True)
+        subprocess.run(['git', '-C', repo_dir, 'push', force_tag_arg, '--tags'], check=True)
 
-    return rc
+    return 0
 
 def update_submodule_pointer(repo_dir, rel_submodule_path, new_submodule_ref):
-    rc = subprocess.run(['git', '-C', repo_dir, 'submodule', 'update', '--init']).returncode
-    rc += subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'fetch']).returncode
-    rc += subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'checkout', new_submodule_ref]).returncode
-    rc += subprocess.run(['git', '-C', repo_dir, 'add', rel_submodule_path]).returncode
+    subprocess.run(['git', '-C', repo_dir, 'submodule', 'update', '--init'], check=True)
+    subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'fetch'], check=True)
+    subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'checkout', new_submodule_ref], check=True)
+    subprocess.run(['git', '-C', repo_dir, 'add', rel_submodule_path], check=True)
+
+    return 0
 
 def setup_intermediate_files(scratch_dir, intree_dir, outtree_dir):
     cleanup_intermediate_files(scratch_dir)
@@ -206,6 +183,7 @@ def create_package(path_ziproot, path_outtree, package_name, exclude_files=[]):
     print('Done.')
 
     return path_outzip
+
 # -------------------------------------------------------------------------------------------------
 # CLI
 # -------------------------------------------------------------------------------------------------
diff --git a/.github/scripts/release.py b/.github/scripts/release.py
index 0706590ee..488ca0b53 100755
--- a/.github/scripts/release.py
+++ b/.github/scripts/release.py
@@ -1,5 +1,10 @@
 #!/usr/bin/env python3
 import os, shutil
+from yaml import load, dump
+try:
+    from yaml import CLoader as Loader, CDumper as Dumper
+except ImportError:
+    from yaml import Loader, Dumper
 from argparse import ArgumentParser
 
 import re
@@ -22,24 +27,6 @@ from packager import RELATIVE_FILE_EXCLUDES as FREERTOS_RELATIVE_FILE_EXCLUDES
 # PyGithub Docs - https://pygithub.readthedocs.io/en/latest/github_objects
 # REST API used by PyGithub - https://developer.github.com/v3/
 
-'''
-FUTURE ENHANCEMENTS
-    - Add mechanism that restores state of all affected to repos to BEFORE this script was run
-    - Input sanitizing
-        - Include regex patterns that MUST be honored for version strings, etc.
-    - Create a companion dependencies file that can be piped to pip3
-    - Ease of HTTPS authentication
-        - This should make it very easy to port to Github action. Currently, Github action mostly operates with 
-          via https endpoints, rather than SSH
-    - Break down some functions and any repeated work --> more granular (reasonably), less duplicated code
-    - Unit tests
-        - Theres already an option and some desired tests laid out via comments. See bottom
-    - All of the scratch-work directories/files should be placed under a single directory the name of which makes obvious
-      that it's a scratch-work dir (Ex. tmp-*, scratch-*, etc.)
-    - Intermediate checks
-        - 
-'''
-
 def info(msg, indent_level=0):
     print('%s[INFO]: %s' % (' ' * indent_level, str(msg)))
 
@@ -59,7 +46,9 @@ class BaseRelease:
         self.commit = commit
         self.git_ssh = git_ssh
         self.git_org = git_org
+        self.repo_path = None
         self.commit_msg_prefix = '[AUTO][RELEASE]: '
+        self.description = ''
 
         self.mGit = mGit # Save a handle to the authed git session
 
@@ -97,6 +86,10 @@ class BaseRelease:
         for r in releases:
             print(r)
 
+    def pushAutoCommits(self):
+        rc = push_git_tree_changes(self.repo_path, tag=self.tag, force_tag=True)
+        assert rc == 0, 'Failed to upload git tree changes'
+
 class KernelRelease(BaseRelease):
     def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'):
         super().__init__(mGit, version, commit, git_ssh=git_ssh, git_org=git_org)
@@ -105,7 +98,19 @@ class KernelRelease(BaseRelease):
         self.repo = mGit.get_repo(self.repo_name)
         self.tag = 'V%s' % version
 
-    def updateFileHeaderVersions(self, old_version_prefix):
+        # Download a local git repo for pushing commits
+        remote_name = self.getRemoteEndpoint(self.repo_name)
+        self.repo_path = 'tmp-release-freertos-kernel'
+
+        # Clean up any old work from previous runs
+        if os.path.exists(self.repo_path):
+            shutil.rmtree(self.repo_path)
+
+        # Download master:HEAD. Update its file header versions and kernel macros
+        self.repo_path = download_git_tree(remote_name, '.', self.repo_path, 'master', 'HEAD')
+        assert self.repo_path != None, 'Failed to download git tree'
+
+    def updateFileHeaderVersions(self):
         '''
         Adds changes for two commits
             1.) Updates to file headers
@@ -113,28 +118,35 @@ class KernelRelease(BaseRelease):
         Then tags commit #2 with the new tag version. Notes this will overwrite a tag it already exists
         Finally pushes all these changes
         '''
-        remote_name = self.getRemoteEndpoint(self.repo_name)
-        rel_repo_path = 'tmp-versioning-freertos-kernel'
+        target_version_prefixes = ['FreeRTOS Kernel V']
+        update_version_number_in_freertos_component(self.repo_path, '.', target_version_prefixes, 'FreeRTOS Kernel V%s' % self.version)
+        commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version)
 
-        # Clean up any old work from previous runs
-        if os.path.exists(rel_repo_path):
-            shutil.rmtree(rel_repo_path)
-
-        # Download master:HEAD. Update its file header versions and kernel macros
-        repo_path = download_git_tree(remote_name, '.', rel_repo_path, 'master', 'HEAD')
-        assert repo_path != None, 'Failed to download git tree'
+    def updateVersionMacros(self):
+        (major, minor, build) = self.version.split('.')
+        update_freertos_version_macros(os.path.join(self.repo_path, 'include', 'task.h'), major, minor, build)
+        commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix + 'Bump task.h version macros to "%s"' % self.version)
 
-        update_version_number_in_freertos_component(repo_path, '.', old_version_prefix, 'FreeRTOS Kernel V%s' % self.version)
-        commit_git_tree_changes(rel_repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version)
+    def createGitRelease(self):
+        '''
+        Creates/Overwrites release identified by target tag
+        '''
 
-        (major, minor, build) = self.version.split('.')
-        update_freertos_version_macros(os.path.join(repo_path, 'include', 'task.h'), major, minor, build)
-        commit_git_tree_changes(rel_repo_path, commit_message=self.commit_msg_prefix + 'Bump task.h version macros to "%s"' % self.version)
+        # If this release already exists, delete it
+        try:
+            release_queried = self.repo.get_release(self.tag)
 
-        # Commit the versioning, tag it, and upload all to remote
-        rc = push_git_tree_changes(repo_path, tag=self.tag, force_tag=True)
-        assert rc == 0, 'Failed to upload git tree changes'
+            info('Deleting existing release "%s"...' % self.tag)
+            release_queried.delete_release()
+        except UnknownObjectException:
+            info('Creating release/tag "%s"...' % self.tag)
 
+        # Create the new release endpoint at upload assets
+        release = self.repo.create_git_release(tag = self.tag,
+                                               name = 'V%s' % (self.version),
+                                               message = self.description,
+                                               draft = False,
+                                               prerelease = False)
 
 class FreertosRelease(BaseRelease):
     def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'):
@@ -146,29 +158,46 @@ class FreertosRelease(BaseRelease):
         self.description = 'Contains source code and example projects for the FreeRTOS Kernel and FreeRTOS+ libraries.'
         self.zip = None
 
-    def updateFileHeaderVersions(self, old_version_prefix, new_kernel_ref):
         remote_name = self.getRemoteEndpoint(self.repo_name)
-        rel_repo_path = 'tmp-versioning-freertos'
+        self.repo_path = 'tmp-release-freertos'
 
         # Clean up any old work from previous runs
-        if os.path.exists(rel_repo_path):
-            shutil.rmtree(rel_repo_path)
+        if os.path.exists(self.repo_path):
+            shutil.rmtree(self.repo_path)
 
         # Download master:HEAD. Update its file header versions and kernel submodule pointer
-        repo_path = download_git_tree(remote_name, '.', rel_repo_path, 'master', 'HEAD')
-        assert repo_path != None, 'Failed to download git tree'
+        self.repo_path = download_git_tree(remote_name, '.', self.repo_path, 'master', 'HEAD')
+        assert self.repo_path != None, 'Failed to download git tree'
 
-        update_version_number_in_freertos_component(repo_path, '.', old_version_prefix, 'FreeRTOS V%s' % self.version)
-        commit_git_tree_changes(repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version)
-
-        update_submodule_pointer(repo_path, os.path.join('FreeRTOS', 'Source'), new_kernel_ref)
-        commit_git_tree_changes(repo_path, commit_message=self.commit_msg_prefix + 'Bump kernel pointer "%s"' % new_kernel_ref)
-
-        # Commit the versioning, tag it, and upload all to remote
-        rc = push_git_tree_changes(repo_path, tag=self.tag, force_tag=True)
-        assert rc == 0, 'Failed to upload git tree changes'
+    def updateFileHeaderVersions(self):
+        target_version_substrings = ['FreeRTOS Kernel V', 'FreeRTOS V']
+        update_version_number_in_freertos_component(self.repo_path, '.', target_version_substrings, 'FreeRTOS V%s' % self.version)
+        commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version)
 
-    def CreateReleaseZip(self):
+    def updateSubmodulePointers(self):
+        '''
+        Reads the 'manifest.yml' file from the local FreeRTOS clone that is being used to stage the commits
+        '''
+        path_manifest = os.path.join(self.repo_path, 'manifest.yml')
+        assert os.path.exists(path_manifest), 'Missing manifest.yml'
+
+        with open(path_manifest, 'r') as fp:
+            manifest_data = fp.read()
+        yml = load(manifest_data, Loader=Loader)
+        assert 'dependencies' in yml, 'Manifest YML parsing error'
+        for dep in yml['dependencies']:
+            assert 'version' in dep, 'Failed to parse submodule tag from manifest'
+            assert 'repository' in dep and 'path' in dep['repository'], 'Failed to parse submodule path from manifest'
+            submodule_path = dep['repository']['path']
+            submodule_tag  = dep['version']
+
+            # Update the submodule to point to version noted in manifest file
+            update_submodule_pointer(self.repo_path, submodule_path, submodule_tag)
+
+        commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix
+                                                               + 'Bump submodules per manifest.yml for V%s' % self.version)
+
+    def createReleaseZip(self):
         '''
         At the moment, the only asset we upload is the
         '''
@@ -196,7 +225,7 @@ class FreertosRelease(BaseRelease):
                                   'FreeRTOSv%s' % self.version,
                                   exclude_files=FREERTOS_RELATIVE_FILE_EXCLUDES)
 
-    def Upload(self):
+    def createGitRelease(self):
         '''
         Creates/Overwrites release identified by target tag
         '''
@@ -222,25 +251,14 @@ class FreertosRelease(BaseRelease):
 
 def configure_argparser():
     parser = ArgumentParser(description='FreeRTOS Release tool')
-
-    parser.add_argument('--old-core-version',
-                        default=None,
-                        required=True,
-                        help='FreeRTOS Version to match and replace. (Ex. FreeRTOS V202011.00)')
-
     parser.add_argument('--new-core-version',
                         default=None,
-                        required=True,
-                        help='FreeRTOS Version to replace old version. (Ex. FreeRTOS V202011.00)')
-
-    parser.add_argument('--old-kernel-version',
-                        default=None,
-                        required=True,
-                        help='FreeRTOS-Kernel Version to match and replace. (Ex. "FreeRTOS Kernel V10.4.1")')
+                        required=False,
+                        help='FreeRTOS-Kernel Version to replace old version. (Ex. "FreeRTOS Kernel V10.4.1")')
 
     parser.add_argument('--new-kernel-version',
                         default=None,
-                        required=True,
+                        required=False,
                         help='FreeRTOS-Kernel Version to replace old version. (Ex. "FreeRTOS Kernel V10.4.1")')
 
     parser.add_argument('--git-org',
@@ -253,7 +271,6 @@ def configure_argparser():
                         action='store_true',
                         help='Use SSH endpoints to interface git remotes, instead of HTTPS')
 
-
     parser.add_argument('--unit-test',
                         action='store_true',
                         default=False,
@@ -261,50 +278,35 @@ def configure_argparser():
 
     return parser
 
-def sanitize_cmd_args(args):
-    info('TODO: Add cmdline input sanitizing')
-
 def main():
-    # CLI
     cmd = configure_argparser()
-
-    # Setup
     args = cmd.parse_args()
-    sanitize_cmd_args(args)
 
     # Auth
-    assert 'GITHUB_TOKEN' in os.environ, 'You must set env variable GITHUB_TOKEN to an authorized git PAT'
+    assert 'GITHUB_TOKEN' in os.environ, 'Set env{GITHUB_TOKEN} to an authorized git PAT'
     mGit = Github(os.environ.get('GITHUB_TOKEN'))
 
+    # Create release or test
     if args.unit_test:
-        pass
-    else:
-        # Update versions
+        return
+
+    if args.new_kernel_version:
         rel_kernel = KernelRelease(mGit, args.new_kernel_version, None, git_ssh=args.use_git_ssh, git_org=args.git_org)
-        rel_kernel.updateFileHeaderVersions(args.old_kernel_version)
+        rel_kernel.updateFileHeaderVersions()
+        rel_kernel.updateVersionMacros()
+        rel_kernel.pushAutoCommits()
+        rel_kernel.createGitRelease()
 
+    if args.new_core_version:
         rel_freertos = FreertosRelease(mGit, args.new_core_version, None, git_ssh=args.use_git_ssh, git_org=args.git_org)
-        rel_freertos.updateFileHeaderVersions(args.old_core_version, 'V%s' % args.new_kernel_version)
-
-        # Package contents of FreeRTOS/FreeRTOS and upload release assets to Git
-        rel_freertos.CreateReleaseZip()
-        rel_freertos.Upload()
+        rel_freertos.updateFileHeaderVersions()
+        rel_freertos.updateSubmodulePointers()
+        rel_freertos.pushAutoCommits()
+        rel_freertos.createReleaseZip()
+        rel_freertos.createGitRelease()
 
     info('Review script output for any unexpected behaviour.')
     info('Release done.')
 
 if __name__ == '__main__':
     main()
-
-#--------------------------------------------------------------------
-#                              TESTING
-#--------------------------------------------------------------------
-# Create new tag, verify creation
-
-# Create release endpoint, delete it, verify deletion
-
-# Overwrite an existing tag
-
-# Perform full operation, restore to state before operation, verify restored state
-
-# Run zipping operation, check versions, pathing, etc
diff --git a/.github/scripts/versioning.py b/.github/scripts/versioning.py
index 7beec75d7..69c17a94f 100755
--- a/.github/scripts/versioning.py
+++ b/.github/scripts/versioning.py
@@ -224,7 +224,8 @@ def update_freertos_version_macros(path_macrofile, major, minor, build):
 
     print('Done. Replaced "%s" --> "V%s.%s.%s".' % (old_version_number, major, minor, build))
 
-def update_version_number_in_freertos_component(component, root_dir, old_version_prefix, new_version, verbose=False):
+def update_version_number_in_freertos_component(component, root_dir, old_version_prefix_list, new_version, verbose=False):
+    assert isinstance(old_version_prefix_list, list), 'Expected a list for arg(old_version_prefix_list)'
     print('Updating "%s"...' % component)
     component_files = list_files_in_a_component(component, root_dir, ext_filter=None)
     version_numbers = defaultdict(list)
@@ -242,8 +243,9 @@ def update_version_number_in_freertos_component(component, root_dir, old_version
         old_version_string = vkey[0]
         new_version_string = new_version
 
-        if old_version_prefix in old_version_string:
-            if old_version_string != new_version_string:
+        # Check if any of the associated versioning strings are present. Update if so
+        for old_prefix in old_version_prefix_list:
+            if old_prefix in old_version_string and old_version_string != new_version_string:
                 files_using_old_version = version_numbers[vkey]
 
                 if verbose:
diff --git a/manifest.yml b/manifest.yml
new file mode 100644
index 000000000..ad508d79a
--- /dev/null
+++ b/manifest.yml
@@ -0,0 +1,68 @@
+name : "FreeRTOS"
+version: "202012.00"
+description: "This is the standard distribution of FreeRTOS."
+
+dependencies:
+  - name: "FreeRTOS-Kernel"
+    version: "V10.4.3"
+    repository:
+      type: "git"
+      url: "https://github.com/FreeRTOS/FreeRTOS-Kernel.git"
+      path: "FreeRTOS/Source"
+
+  - name: "FreeRTOS-Plus-TCP"
+    version: "V2.3.2"
+    repository:
+      type: "git"
+      url: "https://github.com/FreeRTOS/FreeRTOS-Plus-TCP.git"
+      path: "FreeRTOS-Plus/Source/FreeRTOS-Plus-TCP"
+
+  - name: "coreJSON"
+    version: "v3.0.0"
+    repository:
+      type: "git"
+      url: "https://github.com/FreeRTOS/coreJSON.git"
+      path: "FreeRTOS-Plus/Source/coreJSON"
+
+  - name: "coreHTTP"
+    version: "v1.0.1"
+    repository:
+      type: "git"
+      url: "https://github.com/FreeRTOS/coreHTTP.git"
+      path: "FreeRTOS-Plus/Source/Application-Protocols/coreHTTP"
+
+  - name: "coreMQTT"
+    version: "v1.0.2"
+    repository:
+      type: "git"
+      url: "https://github.com/FreeRTOS/coreMQTT.git"
+      path: "FreeRTOS-Plus/Source/Application-Protocols/coreMQTT"
+
+  - name: "corePKCS11"
+    version: "v3.0.0"
+    repository:
+      type: "git"
+      url: "https://github.com/FreeRTOS/corePKCS11.git"
+      path: "FreeRTOS-Plus/Source/corePKCS11"
+
+  - name: "device-defender"
+    version: "v1.0.1"
+    repository:
+      type: "git"
+      url: "https://github.com/aws/Device-Defender-for-AWS-IoT-embedded-sdk.git"
+      path: "FreeRTOS-Plus/Source/AWS/device-defender"
+
+  - name: "device-shadow"
+    version: "v1.0.2"
+    repository:
+      type: "git"
+      url: "https://github.com/aws/Device-Shadow-for-AWS-IoT-embedded-sdk.git"
+      path: "FreeRTOS-Plus/Source/AWS/device-shadow"
+
+  - name: "jobs"
+    version: "v1.0.1"
+    repository:
+      type: "git"
+      url: "https://github.com/aws/Jobs-for-AWS-IoT-embedded-sdk.git"
+      path: "FreeRTOS-Plus/Source/AWS/jobs"
+