diff options
-rw-r--r-- | buildstream/plugins/sources/_downloadablefilesource.py | 80 | ||||
-rw-r--r-- | dev-requirements.txt | 1 | ||||
-rw-r--r-- | tests/sources/remote.py | 43 | ||||
-rw-r--r-- | tests/sources/tar.py | 86 | ||||
-rw-r--r-- | tests/sources/zip.py | 52 | ||||
-rw-r--r-- | tests/testutils/file_server.py | 19 | ||||
-rw-r--r-- | tests/testutils/ftp_server.py | 32 | ||||
-rw-r--r-- | tests/testutils/http_server.py | 117 |
8 files changed, 429 insertions, 1 deletions
diff --git a/buildstream/plugins/sources/_downloadablefilesource.py b/buildstream/plugins/sources/_downloadablefilesource.py index 7d1fc07bf..f5c5b3d08 100644 --- a/buildstream/plugins/sources/_downloadablefilesource.py +++ b/buildstream/plugins/sources/_downloadablefilesource.py @@ -5,16 +5,77 @@ import urllib.request import urllib.error import contextlib import shutil +import netrc from buildstream import Source, SourceError, Consistency from buildstream import utils +class _NetrcFTPOpener(urllib.request.FTPHandler): + + def __init__(self, netrc_config): + self.netrc = netrc_config + + def _split(self, netloc): + userpass, hostport = urllib.parse.splituser(netloc) + host, port = urllib.parse.splitport(hostport) + if userpass: + user, passwd = urllib.parse.splitpasswd(userpass) + else: + user = None + passwd = None + return host, port, user, passwd + + def _unsplit(self, host, port, user, passwd): + if port: + host = '{}:{}'.format(host, port) + if user: + if passwd: + user = '{}:{}'.format(user, passwd) + host = '{}@{}'.format(user, host) + + return host + + def ftp_open(self, req): + host, port, user, passwd = self._split(req.host) + + if user is None and self.netrc: + entry = self.netrc.authenticators(host) + if entry: + user, _, passwd = entry + + req.host = self._unsplit(host, port, user, passwd) + + return super().ftp_open(req) + + +class _NetrcPasswordManager: + + def __init__(self, netrc_config): + self.netrc = netrc_config + + def add_password(self, realm, uri, user, passwd): + pass + + def find_user_password(self, realm, authuri): + if not self.netrc: + return None, None + parts = urllib.parse.urlsplit(authuri) + entry = self.netrc.authenticators(parts.hostname) + if not entry: + return None, None + else: + login, _, password = entry + return login, password + + class DownloadableFileSource(Source): # pylint: disable=attribute-defined-outside-init COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag'] + __urlopener = None + def configure(self, node): self.original_url = self.node_get_member(node, str, 'url') self.ref = self.node_get_member(node, str, 'ref', None) @@ -118,7 +179,8 @@ class DownloadableFileSource(Source): if etag and self.get_consistency() == Consistency.CACHED: request.add_header('If-None-Match', etag) - with contextlib.closing(urllib.request.urlopen(request)) as response: + opener = self.__get_urlopener() + with contextlib.closing(opener.open(request)) as response: info = response.info() etag = info['ETag'] if 'ETag' in info else None @@ -164,3 +226,19 @@ class DownloadableFileSource(Source): def _get_mirror_file(self, sha=None): return os.path.join(self._get_mirror_dir(), sha or self.ref) + + def __get_urlopener(self): + if not DownloadableFileSource.__urlopener: + try: + netrc_config = netrc.netrc() + except FileNotFoundError: + DownloadableFileSource.__urlopener = urllib.request.build_opener() + except netrc.NetrcParseError as e: + self.warn('{}: While reading .netrc: {}'.format(self, e)) + return urllib.request.build_opener() + else: + netrc_pw_mgr = _NetrcPasswordManager(netrc_config) + http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr) + ftp_handler = _NetrcFTPOpener(netrc_config) + DownloadableFileSource.__urlopener = urllib.request.build_opener(http_auth, ftp_handler) + return DownloadableFileSource.__urlopener diff --git a/dev-requirements.txt b/dev-requirements.txt index 1b175c257..380e734c8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,3 +9,4 @@ pytest-pep8 pytest-pylint pytest-xdist pytest-timeout +pyftpdlib diff --git a/tests/sources/remote.py b/tests/sources/remote.py index d3968395f..a4a9d5965 100644 --- a/tests/sources/remote.py +++ b/tests/sources/remote.py @@ -5,6 +5,7 @@ import pytest from buildstream._exceptions import ErrorDomain from buildstream import _yaml from tests.testutils import cli +from tests.testutils.file_server import create_file_server DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), @@ -22,6 +23,16 @@ def generate_project(project_dir, tmpdir): }, project_file) +def generate_project_file_server(server, project_dir): + project_file = os.path.join(project_dir, "project.conf") + _yaml.dump({ + 'name': 'foo', + 'aliases': { + 'tmpdir': server.base_url() + } + }, project_file) + + # Test that without ref, consistency is set appropriately. @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) def test_no_ref(cli, tmpdir, datafiles): @@ -164,3 +175,35 @@ def test_executable(cli, tmpdir, datafiles): assert (mode & stat.S_IEXEC) # Assert executable by anyone assert(mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)) + + +@pytest.mark.parametrize('server_type', ('FTP', 'HTTP')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file')) +def test_use_netrc(cli, datafiles, server_type, tmpdir): + fake_home = os.path.join(str(tmpdir), 'fake_home') + os.makedirs(fake_home, exist_ok=True) + project = str(datafiles) + checkoutdir = os.path.join(str(tmpdir), 'checkout') + + os.environ['HOME'] = fake_home + with open(os.path.join(fake_home, '.netrc'), 'wb') as f: + os.fchmod(f.fileno(), 0o700) + f.write(b'machine 127.0.0.1\n') + f.write(b'login testuser\n') + f.write(b'password 12345\n') + + with create_file_server(server_type) as server: + server.add_user('testuser', '12345', project) + generate_project_file_server(server, project) + + server.start() + + result = cli.run(project=project, args=['fetch', 'target.bst']) + result.assert_success() + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) + result.assert_success() + + checkout_file = os.path.join(checkoutdir, 'file') + assert(os.path.exists(checkout_file)) diff --git a/tests/sources/tar.py b/tests/sources/tar.py index 1fd79f10b..1a1f54f87 100644 --- a/tests/sources/tar.py +++ b/tests/sources/tar.py @@ -3,11 +3,13 @@ import pytest import tarfile import tempfile import subprocess +import urllib.parse from shutil import copyfile, rmtree from buildstream._exceptions import ErrorDomain from buildstream import _yaml from tests.testutils import cli +from tests.testutils.file_server import create_file_server from tests.testutils.site import HAVE_LZIP from . import list_dir_contents @@ -49,6 +51,16 @@ def generate_project(project_dir, tmpdir): }, project_file) +def generate_project_file_server(base_url, project_dir): + project_file = os.path.join(project_dir, "project.conf") + _yaml.dump({ + 'name': 'foo', + 'aliases': { + 'tmpdir': base_url + } + }, project_file) + + # Test that without ref, consistency is set appropriately. @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) def test_no_ref(cli, tmpdir, datafiles): @@ -302,3 +314,77 @@ def test_read_only_dir(cli, tmpdir, datafiles): else: os.remove(path) rmtree(str(tmpdir), onerror=make_dir_writable) + + +@pytest.mark.parametrize('server_type', ('FTP', 'HTTP')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +def test_use_netrc(cli, datafiles, server_type, tmpdir): + file_server_files = os.path.join(str(tmpdir), 'file_server') + fake_home = os.path.join(str(tmpdir), 'fake_home') + os.makedirs(file_server_files, exist_ok=True) + os.makedirs(fake_home, exist_ok=True) + project = str(datafiles) + checkoutdir = os.path.join(str(tmpdir), 'checkout') + + os.environ['HOME'] = fake_home + with open(os.path.join(fake_home, '.netrc'), 'wb') as f: + os.fchmod(f.fileno(), 0o700) + f.write(b'machine 127.0.0.1\n') + f.write(b'login testuser\n') + f.write(b'password 12345\n') + + with create_file_server(server_type) as server: + server.add_user('testuser', '12345', file_server_files) + generate_project_file_server(server.base_url(), project) + + src_tar = os.path.join(file_server_files, 'a.tar.gz') + _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar) + + server.start() + + result = cli.run(project=project, args=['track', 'target.bst']) + result.assert_success() + result = cli.run(project=project, args=['fetch', 'target.bst']) + result.assert_success() + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) + result.assert_success() + + original_dir = os.path.join(str(datafiles), 'content', 'a') + original_contents = list_dir_contents(original_dir) + checkout_contents = list_dir_contents(checkoutdir) + assert(checkout_contents == original_contents) + + +@pytest.mark.parametrize('server_type', ('FTP', 'HTTP')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir): + file_server_files = os.path.join(str(tmpdir), 'file_server') + fake_home = os.path.join(str(tmpdir), 'fake_home') + os.makedirs(file_server_files, exist_ok=True) + os.makedirs(fake_home, exist_ok=True) + project = str(datafiles) + checkoutdir = os.path.join(str(tmpdir), 'checkout') + + os.environ['HOME'] = fake_home + with open(os.path.join(fake_home, '.netrc'), 'wb') as f: + os.fchmod(f.fileno(), 0o700) + f.write(b'machine 127.0.0.1\n') + f.write(b'login testuser\n') + f.write(b'password 12345\n') + + with create_file_server(server_type) as server: + server.add_user('otheruser', '12345', file_server_files) + parts = urllib.parse.urlsplit(server.base_url()) + base_url = urllib.parse.urlunsplit([parts[0]] + ['otheruser@{}'.format(parts[1])] + list(parts[2:])) + generate_project_file_server(base_url, project) + + src_tar = os.path.join(file_server_files, 'a.tar.gz') + _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar) + + server.start() + + result = cli.run(project=project, args=['track', 'target.bst']) + result.assert_main_error(ErrorDomain.STREAM, None) + result.assert_task_error(ErrorDomain.SOURCE, None) diff --git a/tests/sources/zip.py b/tests/sources/zip.py index 73767ee79..6ad6d4077 100644 --- a/tests/sources/zip.py +++ b/tests/sources/zip.py @@ -5,6 +5,7 @@ import zipfile from buildstream._exceptions import ErrorDomain from buildstream import _yaml from tests.testutils import cli +from tests.testutils.file_server import create_file_server from . import list_dir_contents DATA_DIR = os.path.join( @@ -35,6 +36,16 @@ def generate_project(project_dir, tmpdir): }, project_file) +def generate_project_file_server(server, project_dir): + project_file = os.path.join(project_dir, "project.conf") + _yaml.dump({ + 'name': 'foo', + 'aliases': { + 'tmpdir': server.base_url() + } + }, project_file) + + # Test that without ref, consistency is set appropriately. @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) def test_no_ref(cli, tmpdir, datafiles): @@ -176,3 +187,44 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles): original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert(checkout_contents == original_contents) + + +@pytest.mark.parametrize('server_type', ('FTP', 'HTTP')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +def test_use_netrc(cli, datafiles, server_type, tmpdir): + file_server_files = os.path.join(str(tmpdir), 'file_server') + fake_home = os.path.join(str(tmpdir), 'fake_home') + os.makedirs(file_server_files, exist_ok=True) + os.makedirs(fake_home, exist_ok=True) + project = str(datafiles) + checkoutdir = os.path.join(str(tmpdir), 'checkout') + + os.environ['HOME'] = fake_home + with open(os.path.join(fake_home, '.netrc'), 'wb') as f: + os.fchmod(f.fileno(), 0o700) + f.write(b'machine 127.0.0.1\n') + f.write(b'login testuser\n') + f.write(b'password 12345\n') + + with create_file_server(server_type) as server: + server.add_user('testuser', '12345', file_server_files) + generate_project_file_server(server, project) + + src_zip = os.path.join(file_server_files, 'a.zip') + _assemble_zip(os.path.join(str(datafiles), 'content'), src_zip) + + server.start() + + result = cli.run(project=project, args=['track', 'target.bst']) + result.assert_success() + result = cli.run(project=project, args=['fetch', 'target.bst']) + result.assert_success() + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) + result.assert_success() + + original_dir = os.path.join(str(datafiles), 'content', 'a') + original_contents = list_dir_contents(original_dir) + checkout_contents = list_dir_contents(checkoutdir) + assert(checkout_contents == original_contents) diff --git a/tests/testutils/file_server.py b/tests/testutils/file_server.py new file mode 100644 index 000000000..05f896013 --- /dev/null +++ b/tests/testutils/file_server.py @@ -0,0 +1,19 @@ +from contextlib import contextmanager + +from .ftp_server import SimpleFtpServer +from .http_server import SimpleHttpServer + + +@contextmanager +def create_file_server(file_server_type): + if file_server_type == 'FTP': + server = SimpleFtpServer() + elif file_server_type == 'HTTP': + server = SimpleHttpServer() + else: + assert False + + try: + yield server + finally: + server.stop() diff --git a/tests/testutils/ftp_server.py b/tests/testutils/ftp_server.py new file mode 100644 index 000000000..52c05f8ba --- /dev/null +++ b/tests/testutils/ftp_server.py @@ -0,0 +1,32 @@ +import multiprocessing + +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import FTPServer + + +class SimpleFtpServer(multiprocessing.Process): + def __init__(self): + super().__init__() + self.authorizer = DummyAuthorizer() + handler = FTPHandler + handler.authorizer = self.authorizer + self.server = FTPServer(('127.0.0.1', 0), handler) + + def run(self): + self.server.serve_forever() + + def stop(self): + self.server.close_all() + self.server.close() + self.terminate() + self.join() + + def allow_anonymous(self, cwd): + self.authorizer.add_anonymous(cwd) + + def add_user(self, user, password, cwd): + self.authorizer.add_user(user, password, cwd, perm='elradfmwMT') + + def base_url(self): + return 'ftp://127.0.0.1:{}'.format(self.server.address[1]) diff --git a/tests/testutils/http_server.py b/tests/testutils/http_server.py new file mode 100644 index 000000000..129003836 --- /dev/null +++ b/tests/testutils/http_server.py @@ -0,0 +1,117 @@ +import multiprocessing +import os +import posixpath +import html +import threading +import base64 +from http.server import SimpleHTTPRequestHandler, HTTPServer, HTTPStatus + + +class Unauthorized(Exception): + pass + + +class RequestHandler(SimpleHTTPRequestHandler): + + def get_root_dir(self): + authorization = self.headers.get('authorization') + if not authorization: + if not self.server.anonymous_dir: + raise Unauthorized('unauthorized') + return self.server.anonymous_dir + else: + authorization = authorization.split() + if len(authorization) != 2 or authorization[0].lower() != 'basic': + raise Unauthorized('unauthorized') + try: + decoded = base64.decodebytes(authorization[1].encode('ascii')) + user, password = decoded.decode('ascii').split(':') + expected_password, directory = self.server.users[user] + if password == expected_password: + return directory + except: + raise Unauthorized('unauthorized') + return None + + def unauthorized(self): + shortmsg, longmsg = self.responses[HTTPStatus.UNAUTHORIZED] + self.send_response(HTTPStatus.UNAUTHORIZED, shortmsg) + self.send_header('Connection', 'close') + + content = (self.error_message_format % { + 'code': HTTPStatus.UNAUTHORIZED, + 'message': html.escape(longmsg, quote=False), + 'explain': html.escape(longmsg, quote=False) + }) + body = content.encode('UTF-8', 'replace') + self.send_header('Content-Type', self.error_content_type) + self.send_header('Content-Length', str(len(body))) + self.send_header('WWW-Authenticate', 'Basic realm="{}"'.format(self.server.realm)) + self.end_headers() + self.end_headers() + + if self.command != 'HEAD' and body: + self.wfile.write(body) + + def do_GET(self): + try: + super().do_GET() + except Unauthorized: + self.unauthorized() + + def do_HEAD(self): + try: + super().do_HEAD() + except Unauthorized: + self.unauthorized() + + def translate_path(self, path): + path = path.split('?', 1)[0] + path = path.split('#', 1)[0] + path = posixpath.normpath(path) + assert(posixpath.isabs(path)) + path = posixpath.relpath(path, '/') + return os.path.join(self.get_root_dir(), path) + + +class AuthHTTPServer(HTTPServer): + def __init__(self, *args, **kwargs): + self.users = {} + self.anonymous_dir = None + self.realm = 'Realm' + super().__init__(*args, **kwargs) + + +class SimpleHttpServer(multiprocessing.Process): + def __init__(self): + self.__stop = multiprocessing.Queue() + super().__init__() + self.server = AuthHTTPServer(('127.0.0.1', 0), RequestHandler) + self.started = False + + def start(self): + self.started = True + super().start() + + def run(self): + t = threading.Thread(target=self.server.serve_forever) + t.start() + self.__stop.get() + self.server.shutdown() + t.join() + + def stop(self): + if not self.started: + return + self.__stop.put(None) + self.terminate() + self.join() + + def allow_anonymous(self, cwd): + self.server.anonymous_dir = cwd + + def add_user(self, user, password, cwd): + self.server.users[user] = (password, cwd) + + def base_url(self): + return 'http://127.0.0.1:{}'.format(self.server.server_port) |