diff options
author | Jason R. Coombs <jaraco@jaraco.com> | 2015-11-17 16:53:27 -0500 |
---|---|---|
committer | Jason R. Coombs <jaraco@jaraco.com> | 2015-11-17 16:53:27 -0500 |
commit | 2f9f41cdefb786c6ba11e94845fcb93811eb26a7 (patch) | |
tree | fd286210ff0683bab730f1eb18b44dc2206b1fd4 | |
parent | 3e1da0da1436b2247f5a6576adb25c58ed560560 (diff) | |
download | python-setuptools-bitbucket-2f9f41cdefb786c6ba11e94845fcb93811eb26a7.tar.gz |
Adding script from idg serpro. Ref #432.
-rw-r--r-- | scripts/upload-old-releases-as-zip.py | 234 |
1 files changed, 234 insertions, 0 deletions
diff --git a/scripts/upload-old-releases-as-zip.py b/scripts/upload-old-releases-as-zip.py new file mode 100644 index 00000000..53492819 --- /dev/null +++ b/scripts/upload-old-releases-as-zip.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import errno +import glob +import hashlib +import json +import os +import shutil +import tarfile +import tempfile +import urllib +import urllib2 + +from distutils.version import LooseVersion + +OK = '\033[92m' +FAIL = '\033[91m' +END = '\033[0m' +DISTRIBUTION = "setuptools" + + +class SetuptoolsOldReleasesWithoutZip(object): + """docstring for SetuptoolsOldReleases""" + + def __init__(self): + super(SetuptoolsOldReleasesWithoutZip, self).__init__() + self.dirpath = tempfile.mkdtemp(prefix=DISTRIBUTION) + print "Downloading %s releases..." % DISTRIBUTION + print "All releases will be downloaded to %s" % self.dirpath + self.data_json_setuptools = self.get_json_data(DISTRIBUTION) + self.valid_releases_numbers = sorted([ + release + for release in self.data_json_setuptools['releases'] + # This condition is motivated by 13.0 release, which + # comes as "13.0": [], in the json + if self.data_json_setuptools['releases'][release] + ], key=LooseVersion) + self.total_downloaded_ok = 0 + + def get_json_data(self, package_name): + """ + "releases": { + "0.7.2": [ + { + "has_sig": false, + "upload_time": "2013-06-09T16:10:00", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.2.tar.gz", # NOQA + "md5_digest": "de44cd90f8a1c713d6c2bff67bbca65d", + "downloads": 159014, + "filename": "setuptools-0.7.2.tar.gz", + "packagetype": "sdist", + "size": 633077 + } + ], + "0.7.3": [ + { + "has_sig": false, + "upload_time": "2013-06-18T21:08:56", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-0.7.3.tar.gz", # NOQA + "md5_digest": "c854adacbf9067d330a847f06f7a8eba", + "downloads": 30594, + "filename": "setuptools-0.7.3.tar.gz", + "packagetype": "sdist", + "size": 751152 + } + ], + "12.3": [ + { + "has_sig": false, + "upload_time": "2015-02-26T19:15:51", + "comment_text": "", + "python_version": "3.4", + "url": "https://pypi.python.org/packages/3.4/s/setuptools/setuptools-12.3-py2.py3-none-any.whl", # NOQA + "md5_digest": "31f51a38497a70efadf5ce8d4c2211ab", + "downloads": 288451, + "filename": "setuptools-12.3-py2.py3-none-any.whl", + "packagetype": "bdist_wheel", + "size": 501904 + }, + { + "has_sig": false, + "upload_time": "2015-02-26T19:15:43", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.tar.gz", # NOQA + "md5_digest": "67614b6d560fa4f240e99cd553ec7f32", + "downloads": 110109, + "filename": "setuptools-12.3.tar.gz", + "packagetype": "sdist", + "size": 635025 + }, + { + "has_sig": false, + "upload_time": "2015-02-26T19:15:47", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/s/setuptools/setuptools-12.3.zip", # NOQA + "md5_digest": "abc799e7db6e7281535bf342bfc41a12", + "downloads": 67539, + "filename": "setuptools-12.3.zip", + "packagetype": "sdist", + "size": 678783 + } + ], + """ + url = "https://pypi.python.org/pypi/%s/json" % (package_name,) + data = json.load(urllib2.urlopen(urllib2.Request(url))) + + # Mainly for debug. + json_filename = "%s/%s.json" % (self.dirpath, DISTRIBUTION) + with open(json_filename, 'w') as outfile: + json.dump( + data, + outfile, + sort_keys=True, + indent=4, + separators=(',', ': '), + ) + + return data + + def get_setuptools_releases_without_zip_counterpart(self): + # Get set(all_valid_releases) - set(releases_with_zip), so now we have + # the releases without zip. + return set(self.valid_releases_numbers) - set([ + release + for release in self.valid_releases_numbers + for same_version_release_dict in self.data_json_setuptools['releases'][release] # NOQA + if 'zip' in same_version_release_dict['filename'] + ]) + + def download_setuptools_releases_without_zip_counterpart(self): + try: + releases_without_zip = self.get_setuptools_releases_without_zip_counterpart() # NOQA + failed_md5_releases = [] + # This is a "strange" loop, going through all releases and + # testing only the release I need to download, but I thought it + # would be mouch more readable than trying to iterate through + # releases I need and get into traverse hell values inside dicts + # inside dicts of the json to get the distribution's url to + # download. + for release in self.valid_releases_numbers: + if release in releases_without_zip: + for same_version_release_dict in self.data_json_setuptools['releases'][release]: # NOQA + if 'tar.gz' in same_version_release_dict['filename']: + print "Downloading %s..." % release + local_file = '%s/%s' % ( + self.dirpath, + same_version_release_dict["filename"] + ) + urllib.urlretrieve( + same_version_release_dict["url"], + local_file + ) + targz = open(local_file, 'rb').read() + hexdigest = hashlib.md5(targz).hexdigest() + if (hexdigest != same_version_release_dict['md5_digest']): # NOQA + print FAIL + "FAIL: md5 for %s didn't match!" % release + END # NOQA + failed_md5_releases.append(release) + else: + self.total_downloaded_ok += 1 + print 'Total releases without zip: %s' % len(releases_without_zip) + print 'Total downloaded: %s' % self.total_downloaded_ok + if failed_md5_releases: + raise(Exception( + FAIL + ( + "FAIL: these releases %s failed the md5 check!" % + ','.join(failed_md5_releases) + ) + END + )) + elif self.total_downloaded_ok != len(releases_without_zip): + raise(Exception( + FAIL + ( + "FAIL: Unknown error occured. Please check the logs." + ) + END + )) + else: + print OK + "All releases downloaded and md5 checked." + END + + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + def convert_targz_to_zip(self): + print "Converting the tar.gz to zip..." + files = glob.glob('%s/*.tar.gz' % self.dirpath) + total_converted = 0 + for targz in sorted(files, key=LooseVersion): + # Extract and remove tar. + tar = tarfile.open(targz) + tar.extractall(path=self.dirpath) + tar.close() + os.remove(targz) + + # Zip the extracted tar. + setuptools_folder_path = targz.replace('.tar.gz', '') + setuptools_folder_name = setuptools_folder_path.split("/")[-1] + print setuptools_folder_name + shutil.make_archive( + setuptools_folder_path, + 'zip', + self.dirpath, + setuptools_folder_name + ) + # Exclude extracted tar folder. + shutil.rmtree(setuptools_folder_path.replace('.zip', '')) + total_converted += 1 + print 'Total converted: %s' % total_converted + if self.total_downloaded_ok != total_converted: + raise(Exception( + FAIL + ( + "FAIL: Total number of downloaded releases is different" + " from converted ones. Please check the logs." + ) + END + )) + print "Done with the tar.gz->zip. Check folder %s." % main.dirpath + + def upload_zips_to_pypi(self): + print 'Uploading to pypi...' + zips = sorted(glob.glob('%s/*.zip' % self.dirpath), key=LooseVersion) + for zips in glob.glob('%s/*.zip' % self.dirpath): + # Put the twine upload code here + pass + + +main = SetuptoolsOldReleasesWithoutZip() +main.download_setuptools_releases_without_zip_counterpart() +main.convert_targz_to_zip() +main.upload_zips_to_pypi() |