1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Create a Baserock .lorry file for a given Python package
#
# Copyright © 2014, 2015 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
from __future__ import print_function
import subprocess
import requests
import json
import sys
import shutil
import tempfile
import xmlrpclib
import logging
import yaml
import pkg_resources
from importer_base import WebServiceClient
from importer_python_common import *
from utils import warn, error
import utils
def filter_urls(urls, extensions):
def match(url):
return ('.'.join(url['url'].split('.')[-2:]) in extensions
or url['url'].split('.')[-1] in extensions)
return filter(match, urls)
def get_releases(client, package_name):
try:
releases = client.package_releases(package_name, True)
except Exception as e:
error("Couldn't fetch release data:", e)
return releases
def generate_archive_lorry(lorry_prefix, client, package_name, version=None):
releases = get_releases(client, package_name)
if len(releases) == 0:
error("Couldn't find any releases for package %s" % package_name)
logging.debug('Found releases: %s', str(releases))
# Use latest release if no version specified
version = version or releases[0]
if version not in releases:
error("Couldn't find any releases of %s with version: %s"
% (package_name, version))
logging.debug('Fetching urls for package %s with version %s',
package_name, version)
try:
# Get a list of dicts, the dicts contain the urls.
urls = client.release_urls(package_name, version)
except Exception as e:
error("Couldn't fetch release urls:", e)
tar_extensions = ('tar.gz', 'tgz', 'tar.Z', 'tar.bz2', 'tbz2',
'tar.lzma', 'tar.xz', 'tlz', 'txz', 'tar')
tarball_urls = filter_urls(urls, tar_extensions)
zip_urls = filter_urls(urls, ('zip',))
archive_type = None
if len(tarball_urls) > 0:
urls = tarball_urls
archive_type = 'tarball'
elif len(zip_urls) > 0:
urls = zip_urls
archive_type = 'zip'
elif len(urls) > 0:
warn("None of these urls look like archives:")
for url in urls:
warn("\t%s" % url['url'])
error("Cannot proceed")
else:
error("Couldn't find any download urls for package %s" % package_name)
url = urls[0]['url']
# TODO: shouldn't be hardcoding ext name here
return utils.str_archive_lorry(archive_type,
'python', lorry_prefix, package_name, url)
class PythonLorryExtension(ImportExtension):
def __init__(self):
super(PythonLorryExtension, self).__init__()
self.apiclient = WebServiceClient('python_api_cache')
def fetch_package_metadata(self, package_name):
r = self.apiclient.request('%s/%s/json' % (PYPI_URL, package_name))
r.raise_for_status()
return r
def run(self):
if len(sys.argv) not in [2, 3]:
print('usage: %s NAME [VERSION]' % sys.argv[0], file=sys.stderr)
sys.exit(1)
client = xmlrpclib.ServerProxy(PYPI_URL)
package_name = sys.argv[1]
version = sys.argv[2] if len(sys.argv) == 3 else None
logging.debug('Looking for package: %s with version %s',
package_name, version)
with open(self.local_data_path('python.yaml')) as f:
lorry_prefix = yaml.load(f)['lorry-prefix']
new_proj_name = name_or_closest(client, package_name)
if new_proj_name == None:
error("Couldn't find any project with name '%s'" % package_name)
logging.debug('Treating %s as %s' % (package_name, new_proj_name))
package_name = new_proj_name
print(generate_archive_lorry(lorry_prefix, client,
package_name, version))
if __name__ == '__main__':
PythonLorryExtension().run()
|
