Merge pull request #10357 from xclaesse/update-wrapdb

Make `meson wrap update` command update all wraps in parallel

5 files changed, 109 insertions, 54 deletions

diff --git a/docs/markdown/Commands.md b/docs/markdown/Commands.md
index 11ca6e07a..c3284611a 100644
--- a/docs/markdown/Commands.md
+++ b/docs/markdown/Commands.md
@@ -266,6 +266,9 @@ Manages subprojects of the Meson project. *Since 0.59.0* commands are run on
 multiple subprojects in parallel by default, use `--num-processes=1` if it is
 not desired.
 
+Since *0.64.0* the `update` subcommand will not download new wrap files
+from WrapDB any more. Use `meson wrap update` command for that instead.
+
 {{ subprojects_arguments.inc }}
 
 ### test
diff --git a/docs/markdown/snippets/update_wrapdb.md b/docs/markdown/snippets/update_wrapdb.md
new file mode 100644
index 000000000..a4e4f278e
--- /dev/null
+++ b/docs/markdown/snippets/update_wrapdb.md
@@ -0,0 +1,8 @@
+## Update all wraps from WrapDB with `meson wrap update` command
+
+The command `meson wrap update`, with no extra argument, will now update all wraps
+that comes from WrapDB to the latest version. The extra `--force` argument will
+also replace wraps that do not come from WrapDB if one is available.
+
+The command `meson subprojects update` will not download new wrap files from
+WrapDB any more.
diff --git a/mesonbuild/msubprojects.py b/mesonbuild/msubprojects.py
index ab527bb42..247690a64 100755
--- a/mesonbuild/msubprojects.py
+++ b/mesonbuild/msubprojects.py
@@ -15,13 +15,13 @@ import zipfile
 
 from . import mlog
 from .mesonlib import quiet_git, GitException, Popen_safe, MesonException, windows_proof_rmtree
-from .wrap.wrap import Resolver, WrapException, ALL_TYPES
-from .wrap import wraptool
+from .wrap.wrap import (Resolver, WrapException, ALL_TYPES, PackageDefinition,
+                        parse_patch_url, update_wrap_file, get_releases)
 
 if T.TYPE_CHECKING:
     from typing_extensions import Protocol
 
-    from .wrap.wrap import PackageDefinition
+    SubParsers = argparse._SubParsersAction[argparse.ArgumentParser]
 
     class Arguments(Protocol):
         sourcedir: str
@@ -35,6 +35,10 @@ if T.TYPE_CHECKING:
         rebase: bool
         reset: bool
 
+    class UpdateWrapDBArguments(Arguments):
+        force: bool
+        releases: T.Dict[str, T.Any]
+
     class CheckoutArguments(Arguments):
         b: bool
         branch_name: str
@@ -130,21 +134,53 @@ class Runner:
         self.logger.done(self.wrap.name, self.log_queue)
         return result
 
-    def update_wrapdb_file(self) -> None:
+    @staticmethod
+    def pre_update_wrapdb(options: 'UpdateWrapDBArguments') -> None:
+        options.releases = get_releases(options.allow_insecure)
+
+    def update_wrapdb(self) -> bool:
+        self.log(f'Checking latest WrapDB version for {self.wrap.name}...')
+        options = T.cast('UpdateWrapDBArguments', self.options)
+
+        # Check if this wrap is in WrapDB
+        info = options.releases.get(self.wrap.name)
+        if not info:
+            self.log('  -> Wrap not found in wrapdb')
+            return True
+
+        # Determine current version
         try:
-            patch_url = self.wrap.get('patch_url')
-            branch, revision = wraptool.parse_patch_url(patch_url)
+            wrapdb_version = self.wrap.get('wrapdb_version')
+            branch, revision = wrapdb_version.split('-', 1)
         except WrapException:
-            return
-        new_branch, new_revision = wraptool.get_latest_version(self.wrap.name, self.options.allow_insecure)
+            # Fallback to parsing the patch URL to determine current version.
+            # This won't work for projects that have upstream Meson support.
+            try:
+                patch_url = self.wrap.get('patch_url')
+                branch, revision = parse_patch_url(patch_url)
+            except WrapException:
+                if not options.force:
+                    self.log('  ->', mlog.red('Could not determine current version, use --force to update any way'))
+                    return False
+                branch = revision = None
+
+        # Download latest wrap if version differs
+        latest_version = info['versions'][0]
+        new_branch, new_revision = latest_version.rsplit('-', 1)
         if new_branch != branch or new_revision != revision:
-            wraptool.update_wrap_file(self.wrap.filename, self.wrap.name, new_branch, new_revision, self.options.allow_insecure)
-            self.log('  -> New wrap file downloaded.')
+            filename = self.wrap.filename if self.wrap.has_wrap else f'{self.wrap.filename}.wrap'
+            update_wrap_file(filename, self.wrap.name,
+                             new_branch, new_revision,
+                             options.allow_insecure)
+            self.log('  -> New version downloaded:', mlog.blue(latest_version))
+        else:
+            self.log('  -> Already at latest version:', mlog.blue(latest_version))
+
+        return True
 
     def update_file(self) -> bool:
         options = T.cast('UpdateArguments', self.options)
 
-        self.update_wrapdb_file()
         if not os.path.isdir(self.repo_dir):
             # The subproject is not needed, or it is a tarball extracted in
             # 'libfoo-1.0' directory and the version has been bumped and the new
@@ -591,6 +627,16 @@ def add_subprojects_argument(p: argparse.ArgumentParser) -> None:
     p.add_argument('subprojects', nargs='*',
                    help='List of subprojects (default: all)')
 
+def add_wrap_update_parser(subparsers: 'SubParsers') -> argparse.ArgumentParser:
+    p = subparsers.add_parser('update', help='Update wrap files from WrapDB (Since 0.63.0)')
+    p.add_argument('--force', default=False, action='store_true',
+                   help='Update wraps that does not seems to come from WrapDB')
+    add_common_arguments(p)
+    add_subprojects_argument(p)
+    p.set_defaults(subprojects_func=Runner.update_wrapdb)
+    p.set_defaults(pre_func=Runner.pre_update_wrapdb)
+    return p
+
 def add_arguments(parser: argparse.ArgumentParser) -> None:
     subparsers = parser.add_subparsers(title='Commands', dest='command')
     subparsers.required = True
@@ -669,6 +715,9 @@ def run(options: 'Arguments') -> int:
     executor = ThreadPoolExecutor(options.num_processes)
     if types:
         wraps = [wrap for wrap in wraps if wrap.type in types]
+    pre_func = getattr(options, 'pre_func', None)
+    if pre_func:
+        pre_func(options)
     logger = Logger(len(wraps))
     for wrap in wraps:
         dirname = Path(subprojects_dir, wrap.directory).as_posix()
diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py
index 1cc55ee7f..b1c22053b 100644
--- a/mesonbuild/wrap/wrap.py
+++ b/mesonbuild/wrap/wrap.py
@@ -107,6 +107,36 @@ def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool
     except urllib.error.URLError as excp:
         raise WrapException(f'WrapDB connection failed to {urlstring} with error {excp}')
 
+def get_releases_data(allow_insecure: bool) -> bytes:
+    url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True)
+    return url.read()
+
+def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
+    data = get_releases_data(allow_insecure)
+    return T.cast('T.Dict[str, T.Any]', json.loads(data.decode()))
+
+def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None:
+    url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap',
+                         allow_insecure, True)
+    with open(wrapfile, 'wb') as f:
+        f.write(url.read())
+
+def parse_patch_url(patch_url: str) -> T.Tuple[str, str]:
+    u = urllib.parse.urlparse(patch_url)
+    if u.netloc != 'wrapdb.mesonbuild.com':
+        raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch')
+    arr = u.path.strip('/').split('/')
+    if arr[0] == 'v1':
+        # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip
+        return arr[-3], arr[-2]
+    elif arr[0] == 'v2':
+        # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch
+        tag = arr[-2]
+        _, version = tag.rsplit('_', 1)
+        version, revision = version.rsplit('-', 1)
+        return version, revision
+    else:
+        raise WrapException(f'Invalid wrapdb URL {patch_url}')
 
 class WrapException(MesonException):
     pass
@@ -283,17 +313,17 @@ class Resolver:
         if not os.path.isdir(self.subdir_root):
             return
         root, dirs, files = next(os.walk(self.subdir_root))
+        ignore_dirs = {'packagecache', 'packagefiles'}
         for i in files:
             if not i.endswith('.wrap'):
                 continue
             fname = os.path.join(self.subdir_root, i)
             wrap = PackageDefinition(fname, self.subproject)
             self.wraps[wrap.name] = wrap
-            if wrap.directory in dirs:
-                dirs.remove(wrap.directory)
+            ignore_dirs |= {wrap.directory, wrap.name}
         # Add dummy package definition for directories not associated with a wrap file.
         for i in dirs:
-            if i in ['packagecache', 'packagefiles']:
+            if i in ignore_dirs:
                 continue
             fname = os.path.join(self.subdir_root, i)
             wrap = PackageDefinition(fname, self.subproject)
diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py
index 80a58ab1e..bcf0e67c5 100644
--- a/mesonbuild/wrap/wraptool.py
+++ b/mesonbuild/wrap/wraptool.py
@@ -12,18 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import sys, os
 import configparser
 import shutil
 import typing as T
 
 from glob import glob
-from urllib.parse import urlparse
-from .wrap import open_wrapdburl, WrapException
+from .wrap import (open_wrapdburl, WrapException, get_releases, get_releases_data,
+                   update_wrap_file, parse_patch_url)
 from pathlib import Path
 
-from .. import mesonlib
+from .. import mesonlib, msubprojects
 
 if T.TYPE_CHECKING:
     import argparse
@@ -49,11 +48,8 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
     p.add_argument('name')
     p.set_defaults(wrap_func=install)
 
-    p = subparsers.add_parser('update', help='update the project to its newest available release')
-    p.add_argument('--allow-insecure', default=False, action='store_true',
-                   help='Allow insecure server connections.')
-    p.add_argument('name')
-    p.set_defaults(wrap_func=update)
+    p = msubprojects.add_wrap_update_parser(subparsers)
+    p.set_defaults(wrap_func=msubprojects.run)
 
     p = subparsers.add_parser('info', help='show available versions of a project')
     p.add_argument('--allow-insecure', default=False, action='store_true',
@@ -75,14 +71,6 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
                    help='Allow insecure server connections.')
     p.set_defaults(wrap_func=update_db)
 
-def get_releases_data(allow_insecure: bool) -> bytes:
-    url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True)
-    return url.read()
-
-def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
-    data = get_releases_data(allow_insecure)
-    return T.cast('T.Dict[str, T.Any]', json.loads(data.decode()))
-
 def list_projects(options: 'argparse.Namespace') -> None:
     releases = get_releases(options.allow_insecure)
     for p in releases.keys():
@@ -123,23 +111,6 @@ def install(options: 'argparse.Namespace') -> None:
         f.write(url.read())
     print(f'Installed {name} version {version} revision {revision}')
 
-def parse_patch_url(patch_url: str) -> T.Tuple[str, str]:
-    u = urlparse(patch_url)
-    if u.netloc != 'wrapdb.mesonbuild.com':
-        raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch')
-    arr = u.path.strip('/').split('/')
-    if arr[0] == 'v1':
-        # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip
-        return arr[-3], arr[-2]
-    elif arr[0] == 'v2':
-        # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch
-        tag = arr[-2]
-        _, version = tag.rsplit('_', 1)
-        version, revision = version.rsplit('-', 1)
-        return version, revision
-    else:
-        raise WrapException(f'Invalid wrapdb URL {patch_url}')
-
 def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional[str]]:
     cp = configparser.ConfigParser(interpolation=None)
     cp.read(wrapfile)
@@ -160,12 +131,6 @@ def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional
         patch_filename = wrap_data['patch_filename']
     return branch, revision, wrap_data['directory'], wrap_data['source_filename'], patch_filename
 
-def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str, allow_insecure: bool) -> None:
-    url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap',
-                         allow_insecure, True)
-    with open(wrapfile, 'wb') as f:
-        f.write(url.read())
-
 def update(options: 'argparse.Namespace') -> None:
     name = options.name
     if not os.path.isdir('subprojects'):