Wheels6 (#1767)

4 files changed, 148 insertions, 179 deletions

diff --git a/scripts/internal/download_wheels.py b/scripts/internal/download_wheels.py
deleted file mode 100644
index 1834f1b3..00000000
--- a/scripts/internal/download_wheels.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""
-Script which downloads wheel files hosted on GitHub:
-https://github.com/giampaolo/psutil/actions
-It needs an access token string generated from personal GitHub profile:
-https://github.com/settings/tokens
-The token must be created with at least "public_repo" scope/rights.
-If you lose it, just generate a new token.
-REST API doc:
-https://developer.github.com/v3/actions/artifacts/
-"""
-
-import argparse
-import collections
-import json
-import os
-import requests
-import shutil
-import zipfile
-
-from psutil import __version__ as PSUTIL_VERSION
-from psutil._common import bytes2human
-from psutil._common import print_color
-
-
-USER = ""
-PROJECT = ""
-TOKEN = ""
-OUTFILE = "wheels.zip"
-
-
-# --- GitHub API
-
-
-def get_artifacts():
-    base_url = "https://api.github.com/repos/%s/%s" % (USER, PROJECT)
-    url = base_url + "/actions/artifacts"
-    res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN})
-    res.raise_for_status()
-    data = json.loads(res.content)
-    return data
-
-
-def download_zip(url):
-    print("downloading: " + url)
-    res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN})
-    res.raise_for_status()
-    totbytes = 0
-    with open(OUTFILE, 'wb') as f:
-        for chunk in res.iter_content(chunk_size=16384):
-            f.write(chunk)
-            totbytes += len(chunk)
-    print("got %s, size %s)" % (OUTFILE, bytes2human(totbytes)))
-
-
-# --- extract
-
-
-def rename_27_wheels():
-    # See: https://github.com/giampaolo/psutil/issues/810
-    src = 'dist/psutil-%s-cp27-cp27m-win32.whl' % PSUTIL_VERSION
-    dst = 'dist/psutil-%s-cp27-none-win32.whl' % PSUTIL_VERSION
-    print("rename: %s\n        %s" % (src, dst))
-    os.rename(src, dst)
-    src = 'dist/psutil-%s-cp27-cp27m-win_amd64.whl' % PSUTIL_VERSION
-    dst = 'dist/psutil-%s-cp27-none-win_amd64.whl' % PSUTIL_VERSION
-    print("rename: %s\n        %s" % (src, dst))
-    os.rename(src, dst)
-
-
-def extract():
-    with zipfile.ZipFile(OUTFILE, 'r') as zf:
-        zf.extractall('dist')
-
-
-def print_wheels():
-    def is64bit(name):
-        return name.endswith(('x86_64.whl', 'amd64.whl'))
-
-    groups = collections.defaultdict(list)
-    for name in os.listdir('dist'):
-        plat = name.split('-')[-1]
-        pyimpl = name.split('-')[3]
-        ispypy = 'pypy' in pyimpl
-        if 'linux' in plat:
-            if ispypy:
-                groups['pypy_on_linux'].append(name)
-            else:
-                groups['linux'].append(name)
-        elif 'win' in plat:
-            if ispypy:
-                groups['pypy_on_windows'].append(name)
-            else:
-                groups['windows'].append(name)
-        elif 'macosx' in plat:
-            if ispypy:
-                groups['pypy_on_macos'].append(name)
-            else:
-                groups['macos'].append(name)
-        else:
-            assert 0, name
-
-    totsize = 0
-    templ = "%-54s %7s %7s %7s"
-    for platf, names in groups.items():
-        ppn = "%s (total = %s)" % (platf.replace('_', ' '), len(names))
-        s = templ % (ppn, "size", "arch", "pyver")
-        print_color('\n' + s, color=None, bold=True)
-        for name in sorted(names):
-            path = os.path.join('dist', name)
-            size = os.path.getsize(path)
-            totsize += size
-            arch = '64' if is64bit(name) else '32'
-            pyver = 'pypy' if name.split('-')[3].startswith('pypy') else 'py'
-            pyver += name.split('-')[2][2:]
-            s = templ % (name, bytes2human(size), arch, pyver)
-            if 'pypy' in pyver:
-                print_color(s, color='violet')
-            else:
-                print_color(s, color='brown')
-
-
-def run():
-    if os.path.isdir('dist'):
-        shutil.rmtree('dist')
-    data = get_artifacts()
-    download_zip(data['artifacts'][0]['archive_download_url'])
-    os.mkdir('dist')
-    extract()
-    # rename_27_wheels()
-    print_wheels()
-
-
-def main():
-    global USER, PROJECT, TOKEN
-    parser = argparse.ArgumentParser(description='GitHub wheels downloader')
-    parser.add_argument('--user', required=True)
-    parser.add_argument('--project', required=True)
-    parser.add_argument('--tokenfile', required=True)
-    args = parser.parse_args()
-    USER = args.user
-    PROJECT = args.project
-    with open(os.path.expanduser(args.tokenfile)) as f:
-        TOKEN = f.read().strip()
-    run()
-
-
-if __name__ == '__main__':
-    main()
diff --git a/scripts/internal/win_download_wheels.py b/scripts/internal/download_wheels_appveyor.py
index 8dae0573..b7c0aeae 100755
--- a/scripts/internal/win_download_wheels.py
+++ b/scripts/internal/download_wheels_appveyor.py
@@ -15,10 +15,8 @@ http://code.saghul.net/index.php/2015/09/09/
 from __future__ import print_function
 import argparse
 import concurrent.futures
-import errno
 import os
 import requests
-import shutil
 import sys
 
 from psutil import __version__ as PSUTIL_VERSION
@@ -29,33 +27,12 @@ from psutil._common import print_color
 BASE_URL = 'https://ci.appveyor.com/api'
 PY_VERSIONS = ['2.7', '3.5', '3.6', '3.7', '3.8']
 TIMEOUT = 30
-COLORS = True
-
-
-def safe_makedirs(path):
-    try:
-        os.makedirs(path)
-    except OSError as err:
-        if err.errno == errno.EEXIST:
-            if not os.path.isdir(path):
-                raise
-        else:
-            raise
-
-
-def safe_rmtree(path):
-    def onerror(fun, path, excinfo):
-        exc = excinfo[1]
-        if exc.errno != errno.ENOENT:
-            raise
-
-    shutil.rmtree(path, onerror=onerror)
 
 
 def download_file(url):
     local_fname = url.split('/')[-1]
     local_fname = os.path.join('dist', local_fname)
-    safe_makedirs('dist')
+    os.makedirs('dist', exist_ok=True)
     r = requests.get(url, stream=True, timeout=TIMEOUT)
     tot_bytes = 0
     with open(local_fname, 'wb') as f:
@@ -102,7 +79,6 @@ def rename_27_wheels():
 
 
 def run(options):
-    safe_rmtree('dist')
     urls = get_file_urls(options)
     completed = 0
     exc = None
diff --git a/scripts/internal/download_wheels_github.py b/scripts/internal/download_wheels_github.py
new file mode 100644
index 00000000..4aa50d5d
--- /dev/null
+++ b/scripts/internal/download_wheels_github.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Script which downloads wheel files hosted on GitHub:
+https://github.com/giampaolo/psutil/actions
+It needs an access token string generated from personal GitHub profile:
+https://github.com/settings/tokens
+The token must be created with at least "public_repo" scope/rights.
+If you lose it, just generate a new token.
+REST API doc:
+https://developer.github.com/v3/actions/artifacts/
+"""
+
+import argparse
+import json
+import os
+import requests
+import zipfile
+
+from psutil._common import bytes2human
+from psutil.tests import safe_rmpath
+
+
+USER = ""
+PROJECT = ""
+TOKEN = ""
+OUTFILE = "wheels-github.zip"
+
+
+def get_artifacts():
+    base_url = "https://api.github.com/repos/%s/%s" % (USER, PROJECT)
+    url = base_url + "/actions/artifacts"
+    res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN})
+    res.raise_for_status()
+    data = json.loads(res.content)
+    return data
+
+
+def download_zip(url):
+    print("downloading: " + url)
+    res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN})
+    res.raise_for_status()
+    totbytes = 0
+    with open(OUTFILE, 'wb') as f:
+        for chunk in res.iter_content(chunk_size=16384):
+            f.write(chunk)
+            totbytes += len(chunk)
+    print("got %s, size %s)" % (OUTFILE, bytes2human(totbytes)))
+
+
+def run():
+    data = get_artifacts()
+    download_zip(data['artifacts'][0]['archive_download_url'])
+    os.makedirs('dist', exist_ok=True)
+    with zipfile.ZipFile(OUTFILE, 'r') as zf:
+        zf.extractall('dist')
+
+
+def main():
+    global USER, PROJECT, TOKEN
+    parser = argparse.ArgumentParser(description='GitHub wheels downloader')
+    parser.add_argument('--user', required=True)
+    parser.add_argument('--project', required=True)
+    parser.add_argument('--tokenfile', required=True)
+    args = parser.parse_args()
+    USER = args.user
+    PROJECT = args.project
+    with open(os.path.expanduser(args.tokenfile)) as f:
+        TOKEN = f.read().strip()
+    try:
+        run()
+    finally:
+        safe_rmpath(OUTFILE)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/internal/print_wheels.py b/scripts/internal/print_wheels.py
new file mode 100644
index 00000000..be8290e0
--- /dev/null
+++ b/scripts/internal/print_wheels.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Nicely print wheels print in dist/ directory."""
+
+import collections
+import glob
+import os
+
+from psutil._common import print_color
+from psutil._common import bytes2human
+
+
+def main():
+    def is64bit(name):
+        return name.endswith(('x86_64.whl', 'amd64.whl'))
+
+    groups = collections.defaultdict(list)
+    for path in glob.glob('dist/*.whl'):
+        name = os.path.basename(path)
+        plat = name.split('-')[-1]
+        pyimpl = name.split('-')[3]
+        ispypy = 'pypy' in pyimpl
+        if 'linux' in plat:
+            if ispypy:
+                groups['pypy_on_linux'].append(name)
+            else:
+                groups['linux'].append(name)
+        elif 'win' in plat:
+            if ispypy:
+                groups['pypy_on_windows'].append(name)
+            else:
+                groups['windows'].append(name)
+        elif 'macosx' in plat:
+            if ispypy:
+                groups['pypy_on_macos'].append(name)
+            else:
+                groups['macos'].append(name)
+        else:
+            assert 0, name
+
+    totsize = 0
+    templ = "%-54s %7s %7s %7s"
+    for platf, names in groups.items():
+        ppn = "%s (total = %s)" % (platf.replace('_', ' '), len(names))
+        s = templ % (ppn, "size", "arch", "pyver")
+        print_color('\n' + s, color=None, bold=True)
+        for name in sorted(names):
+            path = os.path.join('dist', name)
+            size = os.path.getsize(path)
+            totsize += size
+            arch = '64' if is64bit(name) else '32'
+            pyver = 'pypy' if name.split('-')[3].startswith('pypy') else 'py'
+            pyver += name.split('-')[2][2:]
+            s = templ % (name, bytes2human(size), arch, pyver)
+            if 'pypy' in pyver:
+                print_color(s, color='violet')
+            else:
+                print_color(s, color='brown')
+
+
+if __name__ == '__main__':
+    main()