pypi download stats script

6 files changed, 303 insertions, 180 deletions

diff --git a/scripts/internal/download_wheels.py b/scripts/internal/download_wheels.py
deleted file mode 100644
index 1834f1b3..00000000
--- a/scripts/internal/download_wheels.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""
-Script which downloads wheel files hosted on GitHub:
-https://github.com/giampaolo/psutil/actions
-It needs an access token string generated from personal GitHub profile:
-https://github.com/settings/tokens
-The token must be created with at least "public_repo" scope/rights.
-If you lose it, just generate a new token.
-REST API doc:
-https://developer.github.com/v3/actions/artifacts/
-"""
-
-import argparse
-import collections
-import json
-import os
-import requests
-import shutil
-import zipfile
-
-from psutil import __version__ as PSUTIL_VERSION
-from psutil._common import bytes2human
-from psutil._common import print_color
-
-
-USER = ""
-PROJECT = ""
-TOKEN = ""
-OUTFILE = "wheels.zip"
-
-
-# --- GitHub API
-
-
-def get_artifacts():
-    base_url = "https://api.github.com/repos/%s/%s" % (USER, PROJECT)
-    url = base_url + "/actions/artifacts"
-    res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN})
-    res.raise_for_status()
-    data = json.loads(res.content)
-    return data
-
-
-def download_zip(url):
-    print("downloading: " + url)
-    res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN})
-    res.raise_for_status()
-    totbytes = 0
-    with open(OUTFILE, 'wb') as f:
-        for chunk in res.iter_content(chunk_size=16384):
-            f.write(chunk)
-            totbytes += len(chunk)
-    print("got %s, size %s)" % (OUTFILE, bytes2human(totbytes)))
-
-
-# --- extract
-
-
-def rename_27_wheels():
-    # See: https://github.com/giampaolo/psutil/issues/810
-    src = 'dist/psutil-%s-cp27-cp27m-win32.whl' % PSUTIL_VERSION
-    dst = 'dist/psutil-%s-cp27-none-win32.whl' % PSUTIL_VERSION
-    print("rename: %s\n        %s" % (src, dst))
-    os.rename(src, dst)
-    src = 'dist/psutil-%s-cp27-cp27m-win_amd64.whl' % PSUTIL_VERSION
-    dst = 'dist/psutil-%s-cp27-none-win_amd64.whl' % PSUTIL_VERSION
-    print("rename: %s\n        %s" % (src, dst))
-    os.rename(src, dst)
-
-
-def extract():
-    with zipfile.ZipFile(OUTFILE, 'r') as zf:
-        zf.extractall('dist')
-
-
-def print_wheels():
-    def is64bit(name):
-        return name.endswith(('x86_64.whl', 'amd64.whl'))
-
-    groups = collections.defaultdict(list)
-    for name in os.listdir('dist'):
-        plat = name.split('-')[-1]
-        pyimpl = name.split('-')[3]
-        ispypy = 'pypy' in pyimpl
-        if 'linux' in plat:
-            if ispypy:
-                groups['pypy_on_linux'].append(name)
-            else:
-                groups['linux'].append(name)
-        elif 'win' in plat:
-            if ispypy:
-                groups['pypy_on_windows'].append(name)
-            else:
-                groups['windows'].append(name)
-        elif 'macosx' in plat:
-            if ispypy:
-                groups['pypy_on_macos'].append(name)
-            else:
-                groups['macos'].append(name)
-        else:
-            assert 0, name
-
-    totsize = 0
-    templ = "%-54s %7s %7s %7s"
-    for platf, names in groups.items():
-        ppn = "%s (total = %s)" % (platf.replace('_', ' '), len(names))
-        s = templ % (ppn, "size", "arch", "pyver")
-        print_color('\n' + s, color=None, bold=True)
-        for name in sorted(names):
-            path = os.path.join('dist', name)
-            size = os.path.getsize(path)
-            totsize += size
-            arch = '64' if is64bit(name) else '32'
-            pyver = 'pypy' if name.split('-')[3].startswith('pypy') else 'py'
-            pyver += name.split('-')[2][2:]
-            s = templ % (name, bytes2human(size), arch, pyver)
-            if 'pypy' in pyver:
-                print_color(s, color='violet')
-            else:
-                print_color(s, color='brown')
-
-
-def run():
-    if os.path.isdir('dist'):
-        shutil.rmtree('dist')
-    data = get_artifacts()
-    download_zip(data['artifacts'][0]['archive_download_url'])
-    os.mkdir('dist')
-    extract()
-    # rename_27_wheels()
-    print_wheels()
-
-
-def main():
-    global USER, PROJECT, TOKEN
-    parser = argparse.ArgumentParser(description='GitHub wheels downloader')
-    parser.add_argument('--user', required=True)
-    parser.add_argument('--project', required=True)
-    parser.add_argument('--tokenfile', required=True)
-    args = parser.parse_args()
-    USER = args.user
-    PROJECT = args.project
-    with open(os.path.expanduser(args.tokenfile)) as f:
-        TOKEN = f.read().strip()
-    run()
-
-
-if __name__ == '__main__':
-    main()
diff --git a/scripts/internal/win_download_wheels.py b/scripts/internal/download_wheels_appveyor.py
index 8dae0573..b7c0aeae 100755
--- a/scripts/internal/win_download_wheels.py
+++ b/scripts/internal/download_wheels_appveyor.py
@@ -15,10 +15,8 @@ http://code.saghul.net/index.php/2015/09/09/
 from __future__ import print_function
 import argparse
 import concurrent.futures
-import errno
 import os
 import requests
-import shutil
 import sys
 
 from psutil import __version__ as PSUTIL_VERSION
@@ -29,33 +27,12 @@ from psutil._common import print_color
 BASE_URL = 'https://ci.appveyor.com/api'
 PY_VERSIONS = ['2.7', '3.5', '3.6', '3.7', '3.8']
 TIMEOUT = 30
-COLORS = True
-
-
-def safe_makedirs(path):
-    try:
-        os.makedirs(path)
-    except OSError as err:
-        if err.errno == errno.EEXIST:
-            if not os.path.isdir(path):
-                raise
-        else:
-            raise
-
-
-def safe_rmtree(path):
-    def onerror(fun, path, excinfo):
-        exc = excinfo[1]
-        if exc.errno != errno.ENOENT:
-            raise
-
-    shutil.rmtree(path, onerror=onerror)
 
 
 def download_file(url):
     local_fname = url.split('/')[-1]
     local_fname = os.path.join('dist', local_fname)
-    safe_makedirs('dist')
+    os.makedirs('dist', exist_ok=True)
     r = requests.get(url, stream=True, timeout=TIMEOUT)
     tot_bytes = 0
     with open(local_fname, 'wb') as f:
@@ -102,7 +79,6 @@ def rename_27_wheels():
 
 
 def run(options):
-    safe_rmtree('dist')
     urls = get_file_urls(options)
     completed = 0
     exc = None
diff --git a/scripts/internal/download_wheels_github.py b/scripts/internal/download_wheels_github.py
new file mode 100755
index 00000000..4aa50d5d
--- /dev/null
+++ b/scripts/internal/download_wheels_github.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Script which downloads wheel files hosted on GitHub:
+https://github.com/giampaolo/psutil/actions
+It needs an access token string generated from personal GitHub profile:
+https://github.com/settings/tokens
+The token must be created with at least "public_repo" scope/rights.
+If you lose it, just generate a new token.
+REST API doc:
+https://developer.github.com/v3/actions/artifacts/
+"""
+
+import argparse
+import json
+import os
+import requests
+import zipfile
+
+from psutil._common import bytes2human
+from psutil.tests import safe_rmpath
+
+
+USER = ""
+PROJECT = ""
+TOKEN = ""
+OUTFILE = "wheels-github.zip"
+
+
+def get_artifacts():
+    base_url = "https://api.github.com/repos/%s/%s" % (USER, PROJECT)
+    url = base_url + "/actions/artifacts"
+    res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN})
+    res.raise_for_status()
+    data = json.loads(res.content)
+    return data
+
+
+def download_zip(url):
+    print("downloading: " + url)
+    res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN})
+    res.raise_for_status()
+    totbytes = 0
+    with open(OUTFILE, 'wb') as f:
+        for chunk in res.iter_content(chunk_size=16384):
+            f.write(chunk)
+            totbytes += len(chunk)
+    print("got %s, size %s)" % (OUTFILE, bytes2human(totbytes)))
+
+
+def run():
+    data = get_artifacts()
+    download_zip(data['artifacts'][0]['archive_download_url'])
+    os.makedirs('dist', exist_ok=True)
+    with zipfile.ZipFile(OUTFILE, 'r') as zf:
+        zf.extractall('dist')
+
+
+def main():
+    global USER, PROJECT, TOKEN
+    parser = argparse.ArgumentParser(description='GitHub wheels downloader')
+    parser.add_argument('--user', required=True)
+    parser.add_argument('--project', required=True)
+    parser.add_argument('--tokenfile', required=True)
+    args = parser.parse_args()
+    USER = args.user
+    PROJECT = args.project
+    with open(os.path.expanduser(args.tokenfile)) as f:
+        TOKEN = f.read().strip()
+    try:
+        run()
+    finally:
+        safe_rmpath(OUTFILE)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py
index 9569c367..180bf377 100755
--- a/scripts/internal/print_announce.py
+++ b/scripts/internal/print_announce.py
@@ -58,7 +58,7 @@ Links
 
 --
 
-Giampaolo - http://grodola.blogspot.com
+Giampaolo - https://gmpy.dev/about
 """
 
 
diff --git a/scripts/internal/print_downloads.py b/scripts/internal/print_downloads.py
new file mode 100755
index 00000000..81132db9
--- /dev/null
+++ b/scripts/internal/print_downloads.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Print PYPI statistics in MarkDown format.
+Useful sites:
+* https://pepy.tech/project/psutil
+* https://pypistats.org/packages/psutil
+* https://hugovk.github.io/top-pypi-packages/
+"""
+
+from __future__ import print_function
+import json
+import os
+import subprocess
+import sys
+
+import pypinfo  # NOQA
+
+from psutil._common import memoize
+
+
+AUTH_FILE = os.path.expanduser("~/.pypinfo.json")
+PKGNAME = 'psutil'
+DAYS = 30
+LIMIT = 100
+GITHUB_SCRIPT_URL = "https://github.com/giampaolo/psutil/blob/master/" \
+                    "scripts/internal/pypistats.py"
+bytes_billed = 0
+
+
+# --- get
+
+@memoize
+def sh(cmd):
+    assert os.path.exists(AUTH_FILE)
+    env = os.environ.copy()
+    env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTH_FILE
+    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE, universal_newlines=True)
+    stdout, stderr = p.communicate()
+    if p.returncode != 0:
+        raise RuntimeError(stderr)
+    assert not stderr, stderr
+    return stdout.strip()
+
+
+@memoize
+def query(cmd):
+    global bytes_billed
+    ret = json.loads(sh(cmd))
+    bytes_billed += ret['query']['bytes_billed']
+    return ret
+
+
+def top_packages():
+    return query(
+        f"pypinfo --all --json --days {DAYS} --limit {LIMIT} '' project")
+
+
+def ranking():
+    data = top_packages()
+    for i, line in enumerate(data['rows'], 1):
+        if line['project'] == PKGNAME:
+            return i
+    raise ValueError(f"can't find {PKGNAME}")
+
+
+def downloads():
+    data = top_packages()
+    for line in data['rows']:
+        if line['project'] == PKGNAME:
+            return line['download_count']
+    raise ValueError(f"can't find {PKGNAME}")
+
+
+def downloads_pyver():
+    return query(f"pypinfo --json --days {DAYS} {PKGNAME} pyversion")
+
+
+def downloads_by_country():
+    return query(f"pypinfo --json --days {DAYS} {PKGNAME} country")
+
+
+def downloads_by_system():
+    return query(f"pypinfo --json --days {DAYS} {PKGNAME} system")
+
+
+def downloads_by_distro():
+    return query(f"pypinfo --json --days {DAYS} {PKGNAME} distro")
+
+
+# --- print
+
+
+templ = "| %-30s | %15s |"
+
+
+def print_row(left, right):
+    if isinstance(right, int):
+        right = '{0:,}'.format(right)
+    print(templ % (left, right))
+
+
+def print_header(left, right="Downloads"):
+    print_row(left, right)
+    s = templ % ("-" * 30, "-" * 15)
+    print("|:" + s[2:-2] + ":|")
+
+
+def print_markdown_table(title, left, rows):
+    pleft = left.replace('_', ' ').capitalize()
+    print("### " + title)
+    print()
+    print_header(pleft)
+    for row in rows:
+        lval = row[left]
+        print_row(lval, row['download_count'])
+    print()
+
+
+def main():
+    last_update = top_packages()['last_update']
+    print("# Download stats")
+    print("")
+    s = f"psutil download statistics of the last {DAYS} days (last update "
+    s += f"*{last_update}*).\n"
+    s += f"Generated via [pypistats.py]({GITHUB_SCRIPT_URL}) script.\n"
+    print(s)
+
+    data = [
+        {'what': 'Per month', 'download_count': downloads()},
+        {'what': 'Per day', 'download_count': int(downloads() / 30)},
+        {'what': 'PYPI ranking', 'download_count': ranking()}
+    ]
+    print_markdown_table('Overview', 'what', data)
+    print_markdown_table('Operating systems', 'system_name',
+                         downloads_by_system()['rows'])
+    print_markdown_table('Distros', 'distro_name',
+                         downloads_by_distro()['rows'])
+    print_markdown_table('Python versions', 'python_version',
+                         downloads_pyver()['rows'])
+    print_markdown_table('Countries', 'country',
+                         downloads_by_country()['rows'])
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    finally:
+        print("bytes billed: %s" % bytes_billed, file=sys.stderr)
diff --git a/scripts/internal/print_wheels.py b/scripts/internal/print_wheels.py
new file mode 100755
index 00000000..be8290e0
--- /dev/null
+++ b/scripts/internal/print_wheels.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Nicely print wheels print in dist/ directory."""
+
+import collections
+import glob
+import os
+
+from psutil._common import print_color
+from psutil._common import bytes2human
+
+
+def main():
+    def is64bit(name):
+        return name.endswith(('x86_64.whl', 'amd64.whl'))
+
+    groups = collections.defaultdict(list)
+    for path in glob.glob('dist/*.whl'):
+        name = os.path.basename(path)
+        plat = name.split('-')[-1]
+        pyimpl = name.split('-')[3]
+        ispypy = 'pypy' in pyimpl
+        if 'linux' in plat:
+            if ispypy:
+                groups['pypy_on_linux'].append(name)
+            else:
+                groups['linux'].append(name)
+        elif 'win' in plat:
+            if ispypy:
+                groups['pypy_on_windows'].append(name)
+            else:
+                groups['windows'].append(name)
+        elif 'macosx' in plat:
+            if ispypy:
+                groups['pypy_on_macos'].append(name)
+            else:
+                groups['macos'].append(name)
+        else:
+            assert 0, name
+
+    totsize = 0
+    templ = "%-54s %7s %7s %7s"
+    for platf, names in groups.items():
+        ppn = "%s (total = %s)" % (platf.replace('_', ' '), len(names))
+        s = templ % (ppn, "size", "arch", "pyver")
+        print_color('\n' + s, color=None, bold=True)
+        for name in sorted(names):
+            path = os.path.join('dist', name)
+            size = os.path.getsize(path)
+            totsize += size
+            arch = '64' if is64bit(name) else '32'
+            pyver = 'pypy' if name.split('-')[3].startswith('pypy') else 'py'
+            pyver += name.split('-')[2][2:]
+            s = templ % (name, bytes2human(size), arch, pyver)
+            if 'pypy' in pyver:
+                print_color(s, color='violet')
+            else:
+                print_color(s, color='brown')
+
+
+if __name__ == '__main__':
+    main()