diff options
-rw-r--r-- | tools/scripts/cipd_package.py | 103 | ||||
-rw-r--r-- | tools/scripts/git_submodule.py | 32 | ||||
-rwxr-xr-x | tools/scripts/init-repository.py | 23 | ||||
-rwxr-xr-x | tools/scripts/take_snapshot.py | 43 | ||||
-rw-r--r-- | tools/scripts/version_resolver.py | 41 |
5 files changed, 207 insertions, 35 deletions
diff --git a/tools/scripts/cipd_package.py b/tools/scripts/cipd_package.py new file mode 100644 index 000000000..a9dfce91e --- /dev/null +++ b/tools/scripts/cipd_package.py @@ -0,0 +1,103 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +import glob +import os +import re +import subprocess +import sys +import version_resolver as resolver + +androidx_package_name = 'chromium/third_party/androidx' + +def subprocessCall(args): + print(args) + return subprocess.call(args) + +def subprocessCheckOutput(args): + print(args) + return subprocess.check_output(args).decode() + +class PackageDEPSParser(resolver.DEPSParser): + def __init__(self): + super().__init__() + + def createEntitiesFromScope(self, scope): + entities = [] + for dep in scope: + if (type(scope[dep]) == dict and 'packages' in scope[dep] \ + and 'dep_type' in scope[dep] and scope[dep]['dep_type'] == 'cipd'): + subdir = self.subdir(dep) + if subdir is None: + continue + entity = CIPDEntity(subdir, sp=self.topmost_supermodule_path_prefix) + path = entity.pathRelativeToTopMostSupermodule() + for pkg in scope[dep]['packages']: + p = Package(pkg['package'], pkg['version'], path) + entity.packages.append(p) + entities.append(entity) + return entities + + def parse(self, deps_content): + exec(deps_content, self.global_scope, self.local_scope) + entities = [] + entities.extend(self.createEntitiesFromScope(self.local_scope['deps'])) + return entities + +class Package: + def __init__(self, name, version, path): + self.name = name + self.version = version + self.path = path + self.fileName = name.replace('/','.') + '.pkg' + + def fetchAndDeploy(self): + if os.path.isdir(self.path): + currentDir = os.getcwd() + os.chdir(self.path) + subprocessCall(['cipd', 'pkg-fetch', self.name ,'-out' , self.fileName, '-version', self.version ]) + subprocessCall(['cipd', 'pkg-deploy', self.fileName , '-root', '.' ]) + os.chdir(currentDir) + else: + print('-- missing directory' + self.path + ' skipping') + + def listFiles(self): + if os.path.isdir(self.path): + currentDir = os.getcwd() + os.chdir(self.path) + files = [] + if os.path.isfile(self.fileName): + files = subprocessCheckOutput(['cipd', 'pkg-inspect', self.fileName]).splitlines() + files = map( lambda x: x.replace( ' F ', self.path + '/'), files) + else: + print('-- missing package file ' + self.path + ' skipping') + os.chdir(currentDir) + return files + else: + print('-- missing directory' + self.path + ' skipping') + return [] + +class CIPDEntity: + def __init__(self, path='', packages=[], os=[], sp=''): + self.path = path + self.packages = [] + self.topmost_supermodule_path_prefix = sp + + def pathRelativeToTopMostSupermodule(self): + return os.path.normpath(os.path.join(self.topmost_supermodule_path_prefix, self.path)) + + def findPackage(self, package): + pkg = [p for p in self.packages if p.name == package] + if len(pkg) > 1: + raise Exception(package + " is ambiguous package name for" + self.path) + return pkg[0] if pkg else None + + def readEntities(self): + cipd_packages = [] + cipd_packages = resolver.read(PackageDEPSParser) + print('DEPS file provides the following packages:') + for cipd_package in cipd_packages: + print(cipd_package.pathRelativeToTopMostSupermodule() + ':') + for package in cipd_package.packages: + print(' * {:<80}'.format(package.name) + package.version) + return cipd_packages diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py index 07b5eb756..b2c21de27 100644 --- a/tools/scripts/git_submodule.py +++ b/tools/scripts/git_submodule.py @@ -18,15 +18,9 @@ def subprocessCheckOutput(args): print(args) return subprocess.check_output(args).decode() -class DEPSParser: +class SubmoduleDEPSParser(resolver.DEPSParser): def __init__(self): - self.global_scope = { - 'Var': lambda var_name: '{%s}' % var_name, - 'Str': str, - 'deps_os': {}, - } - self.local_scope = {} - self.topmost_supermodule_path_prefix = '' + super().__init__() def get_vars(self): """Returns a dictionary of effective variable values @@ -37,9 +31,6 @@ class DEPSParser: #result.update(self.custom_vars or {}) return result - def get_recursedeps(self): - return self.local_scope["recursedeps"] - def createSubmodulesFromScope(self, scope, os): submodules = [] for dep in scope: @@ -48,25 +39,18 @@ class DEPSParser: url = scope[dep] elif (type(scope[dep]) == dict and 'url' in scope[dep]): url = scope[dep]['url'] - - if ('condition' in scope[dep]) and (not 'checkout_linux' in scope[dep]['condition']): + if ('condition' in scope[dep]) and \ + (not 'checkout_linux' in scope[dep]['condition']) and \ + (not 'checkout_android_native_support' in scope[dep]['condition']): url = '' - if url: url = url.format(**self.get_vars()) repo_rev = url.split('@') repo = repo_rev[0] rev = repo_rev[1] - subdir = dep - if subdir.startswith('src/'): - subdir = subdir[4:] - # Don't skip submodules that have a supermodule path prefix set (at the moment these - # are 2nd level deep submodules). - elif not self.topmost_supermodule_path_prefix: - # Ignore the information about chromium itself since we get that from git, - # also ignore anything outside src/ (e.g. depot_tools) + subdir = self.subdir(dep) + if subdir is None: continue - submodule = Submodule(subdir, repo, sp=self.topmost_supermodule_path_prefix) submodule.os = os @@ -294,7 +278,7 @@ class Submodule: def readSubmodules(self, use_deps=False): submodules = [] if use_deps: - submodules = resolver.readSubmodules() + submodules = resolver.read(SubmoduleDEPSParser) print('DEPS file provides the following submodules:') for submodule in submodules: print('{:<80}'.format(submodule.pathRelativeToTopMostSupermodule()) + '{:<120}'.format(submodule.url) + submodule.ref) diff --git a/tools/scripts/init-repository.py b/tools/scripts/init-repository.py index 4dc5ca98a..e6ec0c440 100755 --- a/tools/scripts/init-repository.py +++ b/tools/scripts/init-repository.py @@ -12,6 +12,7 @@ import argparse qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) import git_submodule as GitSubmodule +import cipd_package as CIPDPackage import version_resolver as resolver chromium_src = os.environ.get('CHROMIUM_SRC_DIR') @@ -21,13 +22,14 @@ use_external_chromium = False parser = argparse.ArgumentParser(description='Initialize QtWebEngine repository.') parser.add_argument('--baseline-upstream', action='store_true', help='initialize using upstream Chromium submodule w/o applying patches (for maintenance purposes only)') +parser.add_argument('--use_cipd', action='store_true', help='fetch and deploy packages with cipd') group = parser.add_mutually_exclusive_group() group.add_argument('-u', '--upstream', action='store_true', help='initialize using upstream Chromium submodule') group.add_argument('-s', '--snapshot', action='store_true', help='initialize using flat Chromium snapshot submodule (default)') args = parser.parse_args() - if args.baseline_upstream: args.upstream = True + args.use_cipd = True if chromium_src: chromium_src = os.path.abspath(chromium_src) @@ -110,6 +112,20 @@ def initSnapshot(): snapshot.os = 'all' snapshot.initialize() +def initPackages(): + # 'androidx' it is the only so far cipd package we need + third_party_upstream_chromium = os.path.join(qtwebengine_root, 'src/3rdparty_upstream/chromium') + currentDir = os.getcwd() + os.chdir(third_party_upstream_chromium) + cipd = CIPDPackage.CIPDEntity(third_party_upstream_chromium) + cipd_entites = cipd.readEntities() + for e in cipd_entites: + pkg = e.findPackage(CIPDPackage.androidx_package_name) + if pkg: + print('-- fetching and deploying ' + CIPDPackage.androidx_package_name) + pkg.fetchAndDeploy() + os.chdir(currentDir) + os.chdir(qtwebengine_root) if args.upstream: @@ -119,3 +135,8 @@ if args.upstream: subprocess.call(['python3', os.path.join(qtwebengine_root, 'tools', 'scripts', 'patch_upstream.py')]) if args.snapshot: initSnapshot() +if args.use_cipd: + cipdNotFound = subprocess.call(['which', 'cipd']) + if cipdNotFound: + raise Exception("You need cipd from depo tools.Try setting ./src/3rdparty_upstream/chromium/third_party/depot_tools/cipd in your PATH.") + initPackages() diff --git a/tools/scripts/take_snapshot.py b/tools/scripts/take_snapshot.py index e2e454060..41cd46b3c 100755 --- a/tools/scripts/take_snapshot.py +++ b/tools/scripts/take_snapshot.py @@ -12,6 +12,7 @@ import shutil from distutils.version import StrictVersion import git_submodule as GitSubmodule +import cipd_package as CIPDPackage qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) os.chdir(qtwebengine_root) @@ -32,6 +33,11 @@ def isInChromiumBlacklist(file_path): if file_path.endswith('.gn') or file_path.endswith('.gni') or file_path.endswith('.typemap') or \ file_path.endswith('.mojom'): return False + # Add android dependencies info so gn build tree can be parsed + if file_path.endswith('.info') or file_path.endswith('.pydeps'): + return False + if file_path.endswith('.mailmap') or file_path.endswith('.tat.gz.sha1'): + return True if (file_path.startswith('android_webview') or file_path.startswith('apps/') or file_path.startswith('ash/') @@ -121,6 +127,7 @@ def isInChromiumBlacklist(file_path): or file_path.startswith('third_party/accessibility') or file_path.startswith('third_party/afl') or file_path.startswith('third_party/android_') + or file_path.startswith('third_party/androidx') or file_path.startswith('third_party/angle/third_party/deqp') or file_path.startswith('third_party/angle/third_party/glmark2') or file_path.startswith('third_party/angle/third_party/VK-GL-CTS') @@ -201,7 +208,6 @@ def isInChromiumBlacklist(file_path): or file_path.startswith('third_party/google_') or file_path.startswith('third_party/grpc/') or file_path.startswith('third_party/hunspell_dictionaries') - or file_path.startswith('third_party/icu/android') or file_path.startswith('third_party/icu/cast') or file_path.startswith('third_party/icu/chromeos') or file_path.startswith('third_party/instrumented_libraries') @@ -213,6 +219,7 @@ def isInChromiumBlacklist(file_path): or file_path.startswith('third_party/libFuzzer') or file_path.startswith('third_party/liblouis') or file_path.startswith('third_party/libphonenumber') + or file_path.startswith('third_party/libunwindstack') or file_path.startswith('third_party/logilab') or file_path.startswith('third_party/markdown') or file_path.startswith('third_party/openh264/src/res') @@ -292,7 +299,7 @@ def printProgress(current, total): sys.stdout.write("\r{} of {}".format(current, total)) sys.stdout.flush() -def copyFile(src, dst): +def copyFile(src, dst, use_link = True, force_remove = False): src = os.path.abspath(src) dst = os.path.abspath(dst) dst_dir = os.path.dirname(dst) @@ -300,11 +307,14 @@ def copyFile(src, dst): if not os.path.isdir(dst_dir): os.makedirs(dst_dir) - if os.path.exists(dst): + if force_remove or os.path.exists(dst): os.remove(dst) try: - os.link(src, dst) + if use_link: + os.link(src, dst) + else: + shutil.copy(src,dst) # Qt uses LF-only but Chromium isn't. subprocess.call(['dos2unix', '--keep-bom', '--quiet', dst]) except OSError as exception: @@ -336,6 +346,16 @@ def listFilesInCurrentRepository(use_deps=False): files.append(os.path.join(submodule.pathRelativeToTopMostSupermodule(), submodule_file)) return files +def listPackageFilesInCurrentRepositoryForPackage(packageName): + cipd = CIPDPackage.CIPDEntity(os.getcwd()) + cipd_entities = cipd.readEntities() + files = [] + for e in cipd_entities: + pkg = e.findPackage(CIPDPackage.androidx_package_name) + if pkg: + files.extend(pkg.listFiles()) + return files + def exportGn(): third_party_upstream_gn = os.path.join(third_party_upstream, 'gn') third_party_gn = os.path.join(third_party, 'gn') @@ -378,11 +398,15 @@ def exportChromium(): files.append(b'build/util/LASTCHANGE.committime') files.append(b'skia/ext/skia_commit_hash.h') files.append(b'gpu/config/gpu_lists_version.h') + + files.extend(listPackageFilesInCurrentRepositoryForPackage(CIPDPackage.androidx_package_name)) + for root, directories, local_files in os.walk(third_party_upstream_chromium + '/third_party/node/node_modules'): for name in local_files: f = os.path.relpath(os.path.join(root, name)) files.append(f) + symlinks = [] print('copying files to ' + third_party_chromium) for i in range(len(files)): printProgress(i+1, len(files)) @@ -391,7 +415,16 @@ def exportChromium(): else: f = files[i] if not isInChromiumBlacklist(f) and not isInGitBlacklist(f): - copyFile(f, os.path.join(third_party_chromium, f)) + d = os.path.join(third_party_chromium, f) + copyFile(f,d) + # make sure we did not make a hardlink of symlink which is broken afterwards + if os.path.islink(f): + symlinks.append((f,d)) + # this is mostly used for files coming from cipd packages + for s in symlinks: + if not os.path.exists(s[1]): + print('fixing ivalid link ' + s[1]) + copyFile(s[0],s[1], use_link = False, force_remove = True) # We need to gzip transport_security_state_static.json since it is otherwise too big for our git configuration: subprocess.call(['gzip', '-n', third_party_chromium + '/net/http/transport_security_state_static.json']) diff --git a/tools/scripts/version_resolver.py b/tools/scripts/version_resolver.py index c8e37a6de..99b4d2869 100644 --- a/tools/scripts/version_resolver.py +++ b/tools/scripts/version_resolver.py @@ -11,8 +11,39 @@ import sys import json import urllib3 import git_submodule as GitSubmodule - -chromium_version = '108.0.5359.109' +from abc import ABC, abstractmethod + +class DEPSParser(ABC): + def __init__(self): + self.global_scope = { + 'Var': lambda var_name: '{%s}' % var_name, + 'Str': str, + 'deps_os': {}, + } + self.local_scope = {} + self.topmost_supermodule_path_prefix = '' + + def subdir(self, dep): + if dep.startswith('src/'): + return dep[4:] + # Don't skip submodules that have a supermodule path prefix set (at the moment these + # are 2nd level deep submodules). + elif not self.topmost_supermodule_path_prefix: + # Ignore the information about chromium itself since we get that from git, + # also ignore anything outside src/ (e.g. depot_tools) + return None + else: + return dep + + @abstractmethod + def parse(self): + pass + + def get_recursedeps(self): + return self.local_scope["recursedeps"] + + +chromium_version = '108.0.5359.220' chromium_branch = '5359' ninja_version = 'v1.8.2' @@ -59,10 +90,10 @@ def readReleaseChannels(): channels[os].append({ 'channel': ver['channel'], 'version': ver['version'], 'branch': ver['true_branch'] }) return channels -def readSubmodules(): +def read(parserCls): git_deps = subprocess.check_output(['git', 'show', chromium_version +':DEPS']) - parser = GitSubmodule.DEPSParser() + parser = parserCls() git_submodules = parser.parse(git_deps) submodule_dict = {} @@ -80,7 +111,7 @@ def readSubmodules(): with open(extra_deps_file_path, 'r') as extra_deps_file: extra_deps = extra_deps_file.read() if extra_deps: - extradeps_parser = GitSubmodule.DEPSParser() + extradeps_parser = parserCls() extradeps_parser.topmost_supermodule_path_prefix = extradeps_dir extradeps_submodules = extradeps_parser.parse(extra_deps) for sub in extradeps_submodules: |