diff options
Diffstat (limited to 'tools/openblas_support.py')
| -rw-r--r-- | tools/openblas_support.py | 197 |
1 files changed, 65 insertions, 132 deletions
diff --git a/tools/openblas_support.py b/tools/openblas_support.py index 50837177b..d11ad173b 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -2,6 +2,7 @@ import glob import hashlib import os import platform +import sysconfig import sys import shutil import tarfile @@ -12,97 +13,36 @@ from tempfile import mkstemp, gettempdir from urllib.request import urlopen, Request from urllib.error import HTTPError -OPENBLAS_V = '0.3.12' -# Temporary build of OpenBLAS to test a fix for dynamic detection of CPU -OPENBLAS_LONG = 'v0.3.12-buffersize20' +OPENBLAS_V = '0.3.13' +OPENBLAS_LONG = 'v0.3.13-62-gaf2b0d02' BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs' BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download' -ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64', - 'i686', 'ppc64le', 's390x'] -sha256_vals = { - "openblas-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": - "7249d68c02e6b6339e06edfeab1fecddf29ee1e67a3afaa77917c320c43de840", - "openblas64_-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": - "6488e0961a5926e47242f63b63b41cfdd661e6f1d267e8e313e397cde4775c17", - "openblas-v0.3.7-527-g79fd006c-win32-gcc_7_1_0.zip": - "5fb0867ca70b1d0fdbf68dd387c0211f26903d74631420e4aabb49e94aa3930d", - "openblas-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": - "69434bd626bbc495da9ce8c36b005d140c75e3c47f94e88c764a199e820f9259", - "openblas64_-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": - "093f6d953e3fa76a86809be67bd1f0b27656671b5a55b233169cfaa43fd63e22", - "openblas-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": - "42676c69dc48cd6e412251b39da6b955a5a0e00323ddd77f9137f7c259d35319", - "openblas64_-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": - "5aec167af4052cf5e9e3e416c522d9794efabf03a2aea78b9bb3adc94f0b73d8", - "openblas-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": - "fa67c6cc29d4cc5c70a147c80526243239a6f95fc3feadcf83a78176cd9c526b", - "openblas64_-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": - "9ad34e89a5307dcf5823bf5c020580d0559a0c155fe85b44fc219752e61852b0", - "openblas-v0.3.7-527-g79fd006c-manylinux2010_i686.tar.gz": - "0b8595d316c8b7be84ab1f1d5a0c89c1b35f7c987cdaf61d441bcba7ab4c7439", - "openblas-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": - "3e1c7d6472c34e7210e3605be4bac9ddd32f613d44297dc50cf2d067e720c4a9", - "openblas64_-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": - "a0885873298e21297a04be6cb7355a585df4fa4873e436b4c16c0a18fc9073ea", - "openblas-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": - "79b454320817574e20499d58f05259ed35213bea0158953992b910607b17f240", - "openblas64_-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": - "9fddbebf5301518fc4a5d2022a61886544a0566868c8c014359a1ee6b17f2814", - "openblas-v0.3.7-527-g79fd006c-manylinux1_i686.tar.gz": - "24fb92684ec4676185fff5c9340f50c3db6075948bcef760e9c715a8974e4680", - "openblas-v0.3.7-527-g79fd006c-manylinux1_x86_64.tar.gz": - "ebb8236b57a1b4075fd5cdc3e9246d2900c133a42482e5e714d1e67af5d00e62", - "openblas-v0.3.10-win_amd64-gcc_7_1_0.zip": - "e5356a2aa4aa7ed9233b2ca199fdd445f55ba227f004ebc63071dfa2426e9b09", - "openblas64_-v0.3.10-win_amd64-gcc_7_1_0.zip": - "aea3f9c8bdfe0b837f0d2739a6c755b12b6838f6c983e4ede71b4e1b576e6e77", - "openblas-v0.3.10-win32-gcc_7_1_0.zip": - "af1ad3172b23f7c6ef2234151a71d3be4d92010dad4dfb25d07cf5a20f009202", - "openblas64_-v0.3.10-macosx_10_9_x86_64-gf_1becaaa.tar.gz": - "38b61c58d63048731d6884fea7b63f8cbd610e85b138c6bac0e39fd77cd4699b", - "openblas-v0.3.10-manylinux2014_aarch64.tar.gz": - "c4444b9836ec26f7772fae02851961bf73177ff2aa436470e56fab8a1ef8d405", - "openblas-v0.3.10-manylinux2010_x86_64.tar.gz": - "cb7988c4a015aece9c49b1169f51c4ac2287fb9aab8114c8ab67792138ffc85e", - "openblas-v0.3.10-manylinux2010_i686.tar.gz": - "dc637801dd80ebd6394ea8b4a97f8858e4224870ea9214de08bebbdddd8e206e", - "openblas-v0.3.10-manylinux1_x86_64.tar.gz": - "ec1f9e9b2a62d5cb9e2634b88ee2da7cb6b07702d5a0e8b190d680a31adfa23a", - "openblas-v0.3.10-manylinux1_i686.tar.gz": - "b13d9d14e6bd452c0fbadb5cd5fda05b98b1e14043edb13ead90694d4cc07f0e", - "openblas-v0.3.10-manylinux2014_ppc64le.tar.gz": - "1cbc8176986099cf0cbb8f64968d5a14880d602d4b3c59a91d75b69b8760cde3", - "openblas-v0.3.10-manylinux2014_s390x.tar.gz": - "fa6722f0b12507ab0a65f38501ed8435b573df0adc0b979f47cdc4c9e9599475", - "openblas-v0.3.10-macosx_10_9_x86_64-gf_1becaaa.tar.gz": - "c6940b5133e687ae7a4f9c7c794f6a6d92b619cf41e591e5db07aab5da118199", - "openblas64_-v0.3.10-manylinux2014_s390x.tar.gz": - "e0347dd6f3f3a27d2f5e76d382e8a4a68e2e92f5f6a10e54ef65c7b14b44d0e8", - "openblas64_-v0.3.10-manylinux2014_ppc64le.tar.gz": - "4b96a51ac767ec0aabb821c61bcd3420e82e987fc93f7e1f85aebb2a845694eb", - "openblas64_-v0.3.10-manylinux2010_x86_64.tar.gz": - "f68fea21fbc73d06b7566057cad2ed8c7c0eb71fabf9ed8a609f86e5bc60ce5e", - "openblas64_-v0.3.10-manylinux2014_aarch64.tar.gz": - "15e6eed8cb0df8b88e52baa136ffe1769c517e9de7bcdfd81ec56420ae1069e9", -} - - +SUPPORTED_PLATFORMS = [ + 'linux-aarch64', + 'linux-x86_64', + 'linux-i686', + 'linux-ppc64le', + 'linux-s390x', + 'win-amd64', + 'win-32', + 'macosx-x86_64', + 'macosx-arm64', +] IS_32BIT = sys.maxsize < 2**32 -def get_arch(): - if platform.system() == 'Windows': - ret = 'windows' - elif platform.system() == 'Darwin': - ret = 'darwin' - else: - ret = platform.uname().machine - # What do 32 bit machines report? - # If they are a docker, they can report x86_64 - if 'x86' in ret and IS_32BIT: - ret = 'i686' - assert ret in ARCHITECTURES, f'invalid architecture {ret}' - return ret +def get_plat(): + plat = sysconfig.get_platform() + plat_split = plat.split("-") + arch = plat_split[-1] + if arch == "win32": + plat = "win-32" + elif arch in ["universal2", "intel"]: + plat = f"macosx-{platform.uname().machine}" + elif len(plat_split) > 2: + plat = f"{plat_split[0]}-{arch}" + assert plat in SUPPORTED_PLATFORMS, f'invalid platform {plat}' + return plat def get_ilp64(): @@ -120,34 +60,38 @@ def get_manylinux(arch): default = '2014' ret = os.environ.get("MB_ML_VER", default) # XXX For PEP 600 this can be a glibc version - assert ret in ('1', '2010', '2014'), f'invalid MB_ML_VER {ret}' + assert ret in ('1', '2010', '2014', '_2_24'), f'invalid MB_ML_VER {ret}' return ret -def download_openblas(target, arch, ilp64, is_32bit): - ml_ver = get_manylinux(arch) +def download_openblas(target, plat, ilp64): + osname, arch = plat.split("-") fnsuffix = {None: "", "64_": "64_"}[ilp64] filename = '' headers = {'User-Agent': ('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ; ' '(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3')} - if arch in ('aarch64', 'ppc64le', 's390x', 'x86_64', 'i686'): + suffix = None + if osname == "linux": + ml_ver = get_manylinux(arch) suffix = f'manylinux{ml_ver}_{arch}.tar.gz' - filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'tar.gz' - elif arch == 'darwin': + elif plat == 'macosx-x86_64': suffix = 'macosx_10_9_x86_64-gf_1becaaa.tar.gz' - filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'tar.gz' - elif arch == 'windows': - if is_32bit: + elif plat == 'macosx-arm64': + suffix = 'macosx_11_0_arm64-gf_f26990f.tar.gz' + typ = 'tar.gz' + elif osname == 'win': + if plat == "win-32": suffix = 'win32-gcc_8_1_0.zip' else: suffix = 'win_amd64-gcc_8_1_0.zip' - filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'zip' - if not filename: + + if not suffix: return None + filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' req = Request(url=filename, headers=headers) try: response = urlopen(req) @@ -162,22 +106,13 @@ def download_openblas(target, arch, ilp64, is_32bit): data = response.read() # Verify hash key = os.path.basename(filename) - sha256_returned = hashlib.sha256(data).hexdigest() - if 0: - if key not in sha256_vals: - raise ValueError( - f'\nkey "{key}" with hash "{sha256_returned}" not in sha256_vals\n') - sha256_expected = sha256_vals[key] - if sha256_returned != sha256_expected: - # print(f'\nkey "{key}" with hash "{sha256_returned}" mismatch\n') - raise ValueError(f'sha256 hash mismatch for filename {filename}') print("Saving to file", file=sys.stderr) with open(target, 'wb') as fid: fid.write(data) return typ -def setup_openblas(arch=get_arch(), ilp64=get_ilp64(), is_32bit=IS_32BIT): +def setup_openblas(plat=get_plat(), ilp64=get_ilp64()): ''' Download and setup an openblas library for building. If successful, the configuration script will find it automatically. @@ -189,12 +124,13 @@ def setup_openblas(arch=get_arch(), ilp64=get_ilp64(), is_32bit=IS_32BIT): To determine success, do ``os.path.exists(msg)`` ''' _, tmp = mkstemp() - if not arch: - raise ValueError('unknown architecture') - typ = download_openblas(tmp, arch, ilp64, is_32bit) + if not plat: + raise ValueError('unknown platform') + typ = download_openblas(tmp, plat, ilp64) if not typ: return '' - if arch == 'windows': + osname, arch = plat.split("-") + if osname == 'win': if not typ == 'zip': return f'expecting to download zipfile on windows, not {typ}' return unpack_windows_zip(tmp) @@ -294,23 +230,22 @@ def make_init(dirname): """)) -def test_setup(arches): +def test_setup(plats): ''' Make sure all the downloadable files exist and can be opened ''' def items(): - """ yields all combinations of arch, ilp64, is_32bit + """ yields all combinations of arch, ilp64 """ - for arch in arches: - yield arch, None, False - if arch not in ('i686',): - yield arch, '64_', False - if arch in ('windows',): - yield arch, None, True - if arch in ('i686', 'x86_64'): + for plat in plats: + yield plat, None + osname, arch = plat.split("-") + if arch not in ('i686', 'arm64', '32'): + yield plat, '64_' + if osname == "linux" and arch in ('i686', 'x86_64'): oldval = os.environ.get('MB_ML_VER', None) os.environ['MB_ML_VER'] = '1' - yield arch, None, False + yield plat, None # Once we create x86_64 and i686 manylinux2014 wheels... # os.environ['MB_ML_VER'] = '2014' # yield arch, None, False @@ -320,25 +255,23 @@ def test_setup(arches): os.environ.pop('MB_ML_VER') errs = [] - for arch, ilp64, is_32bit in items(): - if arch == '': - continue - if arch not in arches: + for plat, ilp64 in items(): + osname, _ = plat.split("-") + if plat not in plats: continue target = None try: try: - target = setup_openblas(arch, ilp64, is_32bit) + target = setup_openblas(plat, ilp64) except Exception as e: - print(f'Could not setup {arch} with ilp64 {ilp64}, ' - f'32bit {is_32bit}:') + print(f'Could not setup {plat} with ilp64 {ilp64}, ') print(e) errs.append(e) continue if not target: - raise RuntimeError(f'Could not setup {arch}') + raise RuntimeError(f'Could not setup {plat}') print(target) - if arch == 'windows': + if osname == 'win': if not target.endswith('.a'): raise RuntimeError("Not .a extracted!") else: @@ -389,7 +322,7 @@ if __name__ == '__main__': 'architecture') parser.add_argument('--test', nargs='*', default=None, help='Test different architectures. "all", or any of ' - f'{ARCHITECTURES}') + f'{SUPPORTED_PLATFORMS}') parser.add_argument('--check_version', nargs='?', default='', help='Check provided OpenBLAS version string ' 'against available OpenBLAS') @@ -400,6 +333,6 @@ if __name__ == '__main__': print(setup_openblas()) else: if len(args.test) == 0 or 'all' in args.test: - test_setup(ARCHITECTURES) + test_setup(SUPPORTED_PLATFORMS) else: test_setup(args.test) |