From 6897b5811cb29fd17349ba3a5a5d41f16658ba79 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 27 May 2014 02:21:36 +0200 Subject: github migration including travis support, Makefile, README and HISTORY turned into RsT files, python 3.4 fix, pep8ify --- .travis.yml | 21 ++++++++++++++++ HISTORY | 41 ------------------------------- HISTORY.rst | 48 ++++++++++++++++++++++++++++++++++++ LICENSE | 2 +- MANIFEST.in | 9 +++++++ Makefile | 58 ++++++++++++++++++++++++++++++++++++++++++++ README | 48 ------------------------------------ README.rst | 48 ++++++++++++++++++++++++++++++++++++ sendfilemodule.c | 13 +--------- setup.py | 32 ++++++++++++------------ test/benchmark.py | 48 +++++++++++++++++++----------------- test/test_sendfile.py | 67 ++++++++++++++++++++++++++++++--------------------- 12 files changed, 266 insertions(+), 169 deletions(-) create mode 100644 .travis.yml delete mode 100644 HISTORY create mode 100644 HISTORY.rst create mode 100644 MANIFEST.in create mode 100644 Makefile delete mode 100644 README create mode 100644 README.rst diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ebc45e8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +language: python +python: + - 2.6 + - 2.7 + - 3.2 + - 3.3 + - 3.4 + - pypy +install: + - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then pip install unittest2; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install unittest2; fi +script: + - pip install flake8 + - python setup.py build + - python setup.py install + - python test/test_sendfile.py + - make flake8 +os: + - linux + - osx + diff --git a/HISTORY b/HISTORY deleted file mode 100644 index 56f0529..0000000 --- a/HISTORY +++ /dev/null @@ -1,41 +0,0 @@ -Bug tracker at http://code.google.com/p/py-sendfile/issues/list - -Version 2.0.0 - 2012-01-12 -========================== - -(Giampaolo Rodolà took over maintenance) - -##: complete rewriting except AIX code -##: non blocking sockets support -##: threads support (release GIL) -#1: unit tests -#2: python 3 support -#3: FreeBSD implementation is broken -#4: python large file support -#5: header/trailer are now keyword arguments -#6: exposed SF_NODISKIO, SF_MNOWAIT and SF_SYNC constants on FreeBSD -#8: benchmark script -#10: Mac OSX support -#13: Sun OS support - - -Version 1.2.4 - 2009-03-06 -========================== - -(Stephan Peijnik took over maintenance) - -## Add AIX support. - - -Version 1.2.3 - 2008-04-09 -========================== - -## Use setuptools instead of distutils. - - -Version 1.2.2 - 2008-03-29 -========================== - -(Ben Woolley) - -## First release including support for Linux, FreeBSD and DragonflyBSD platforms. diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..5cff994 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,48 @@ +Bug tracker at https://github.com/giampaolo/pysendfile/issues + +Version 2.0.1 (unreleased) +========================== + +- #20: host tarball on PYPI +- #21: project migrated from google code to github +- #21: project migrated from SVN to GIT +- #22: use of travis continuous integration +- #23: add a Makefile +- #24: pysendfile won't compile on python 3.4 + +Version 2.0.0 - 2012-01-12 +========================== + +(Giampaolo Rodolà took over maintenance) + +- ##: complete rewriting except AIX code +- ##: non blocking sockets support +- ##: threads support (release GIL) +- #1: unit tests +- #2: python 3 support +- #3: FreeBSD implementation is broken +- #4: python large file support +- #5: header/trailer are now keyword arguments +- #6: exposed SF_NODISKIO, SF_MNOWAIT and SF_SYNC constants on FreeBSD +- #8: benchmark script +- #10: Mac OSX support +- #13: Sun OS support + +Version 1.2.4 - 2009-03-06 +========================== + +(Stephan Peijnik took over maintenance) + +- ## Add AIX support. + +Version 1.2.3 - 2008-04-09 +========================== + +- ## Use setuptools instead of distutils. + +Version 1.2.2 - 2008-03-29 +========================== + +(Ben Woolley) + +- ## First release including support for Linux, FreeBSD and DragonflyBSD platforms. diff --git a/LICENSE b/LICENSE index 6e3ae9a..2df1b74 100644 --- a/LICENSE +++ b/LICENSE @@ -8,7 +8,7 @@ AIX support code by: Copyright (C) 2008-2009 Niklas Edmundsson Rewritten from scratch and maintained by: - Copyright (C) 2009-2012 Giampaolo Rodola' + Copyright (C) 2009-2014 Giampaolo Rodola' Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..b2ef742 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +include .travis.yml +include HISTORY.rst +include LICENSE +include Makefile +include MANIFEST.in +include README.rst +include sendfilemodule.c +include setup.py +recursive-include test *.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8981d6a --- /dev/null +++ b/Makefile @@ -0,0 +1,58 @@ +# Shortcuts for various tasks (UNIX only). +# To use a specific Python version run: +# $ make install PYTHON=python3.3 + +# You can set these variables from the command line. +PYTHON = python +TSCRIPT = test/test_sendfile.py + +all: test + +clean: + rm -f `find . -type f -name \*.py[co]` + rm -f `find . -type f -name \*.so` + rm -f `find . -type f -nam1e .\*~` + rm -f `find . -type f -name \*.orig` + rm -f `find . -type f -name \*.bak` + rm -f `find . -type f -name \*.rej` + rm -rf `find . -type d -name __pycache__` + rm -rf *.egg-info + rm -rf *\$testfile* + rm -rf .tox + rm -rf build + rm -rf dist + rm -rf docs/_build + +build: clean + $(PYTHON) setup.py build + +install: build + if test $(PYTHON) = python2.5; then \ + $(PYTHON) setup.py install; \ + else \ + $(PYTHON) setup.py install --user; \ + fi + +uninstall: + cd ..; $(PYTHON) -m pip uninstall -y -v pysendfile; \ + +test: install + $(PYTHON) $(TSCRIPT) + +# requires "pip install pep8" +pep8: + @git ls-files | grep \\.py$ | xargs pep8 + +# requires "pip install pyflakes" +pyflakes: + @export PYFLAKES_NODOCTEST=1 && \ + git ls-files | grep \\.py$ | xargs pyflakes + +# requires "pip install flake8" +flake8: + @git ls-files | grep \\.py$ | xargs flake8 + + +# Upload source tarball on https://pypi.python.org/pypi/pysendfile. +upload-src: clean + $(PYTHON) setup.py sdist upload diff --git a/README b/README deleted file mode 100644 index 7dbb128..0000000 --- a/README +++ /dev/null @@ -1,48 +0,0 @@ -Quick links -=========== - -* Home page: http://code.google.com/p/pysendfile -* Download: https://pypi.python.org/pypi/pysendfile - -About -===== - -A python interface to sendfile(2) system call. -Note: as of Python 3.3 you can simply use `os.sendfile() `_ instead. - -Install -======= - -$ pip install pysendfile - -Supported platforms -=================== - -* Linux -* OSX -* FreeBSD -* Dragon Fly BSD -* SunOS -* AIX (non properly tested) - -Python versions from 2.5 to 3.4 by using a single code base. - -Example usage -============= - -:: - - import socket - from sendfile import sendfile - - file = open("somefile", "rb") - blocksize = os.path.getsize("somefile") - sock = socket.socket() - sock.connect(("127.0.0.1", 8021)) - offset = 0 - - while True: - sent = sendfile(sock.fileno(), file.fileno(), offset, blocksize) - if sent == 0: - break # EOF - offset += sent diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..7dbb128 --- /dev/null +++ b/README.rst @@ -0,0 +1,48 @@ +Quick links +=========== + +* Home page: http://code.google.com/p/pysendfile +* Download: https://pypi.python.org/pypi/pysendfile + +About +===== + +A python interface to sendfile(2) system call. +Note: as of Python 3.3 you can simply use `os.sendfile() `_ instead. + +Install +======= + +$ pip install pysendfile + +Supported platforms +=================== + +* Linux +* OSX +* FreeBSD +* Dragon Fly BSD +* SunOS +* AIX (non properly tested) + +Python versions from 2.5 to 3.4 by using a single code base. + +Example usage +============= + +:: + + import socket + from sendfile import sendfile + + file = open("somefile", "rb") + blocksize = os.path.getsize("somefile") + sock = socket.socket() + sock.connect(("127.0.0.1", 8021)) + offset = 0 + + while True: + sent = sendfile(sock.fileno(), file.fileno(), offset, blocksize) + if sent == 0: + break # EOF + offset += sent diff --git a/sendfilemodule.c b/sendfilemodule.c index d46ef7a..fd69af6 100644 --- a/sendfilemodule.c +++ b/sendfilemodule.c @@ -1,7 +1,3 @@ -/* - * $Id$ - */ - /* * pysendfile * @@ -14,7 +10,7 @@ * Copyright (C) 2008,2009 Niklas Edmundsson * * Rewritten from scratch and maintained by Giampaolo Rodola' - * Copyright (C) 2009,2012 + * Copyright (C) 2009,2014 * * * The MIT License @@ -392,13 +388,6 @@ initsendfile(void) #endif if (module == NULL) INITERROR; - struct module_state *st = GETSTATE(module); - - st->error = PyErr_NewException("sendfile.Error", NULL, NULL); - if (st->error == NULL) { - Py_DECREF(module); - INITERROR; - } #if PY_MAJOR_VERSION >= 3 return module; diff --git a/setup.py b/setup.py index 8391123..6eb3e83 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,9 @@ #!/usr/bin/env python -# $Id$ # ====================================================================== # This software is distributed under the MIT license reproduced below: # -# Copyright (C) 2009-2012 Giampaolo Rodola' +# Copyright (C) 2009-2014 Giampaolo Rodola' # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby @@ -30,6 +29,10 @@ try: except ImportError: from distutils.core import setup, Extension +NAME = 'pysendfile' +VERSION = '2.0.1' + + if sys.version_info < (2, 5): sys.exit('python version not supported (< 2.5)') @@ -38,24 +41,19 @@ if 'sunos' in sys.platform: else: libraries = [] -name = 'pysendfile' -version = '2.0.0' -download_url = "http://pysendfile.googlecode.com/files/" + name + "-" + \ - version + ".tar.gz" def main(): - setup(name=name, - url='http://code.google.com/p/pysendfile/', - version=version, + setup(name=NAME, + url='https://github.com/giampaolo/pysendfile', + version=VERSION, description='A Python interface to sendfile(2)', - long_description=open('README', 'r').read(), + long_description=open('README.rst', 'r').read(), author='Giampaolo Rodola', author_email='g.rodola@gmail.com', - download_url=download_url, platforms='UNIX', license='MIT', keywords=['sendfile', 'python', 'performance', 'ftp'], - classifiers = [ + classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Operating System :: POSIX :: Linux', @@ -65,7 +63,6 @@ def main(): 'Operating System :: POSIX :: SunOS/Solaris', 'Operating System :: POSIX :: AIX', 'Programming Language :: C', - 'Programming Language :: Python :: 2.4', 'Programming Language :: Python :: 2.5', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', @@ -73,15 +70,16 @@ def main(): 'Programming Language :: Python :: 3.0', 'Programming Language :: Python :: 3.1', 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', 'Topic :: System :: Networking', 'Topic :: System :: Operating System', 'Topic :: Internet :: File Transfer Protocol (FTP)', 'Topic :: Internet :: WWW/HTTP', 'License :: OSI Approved :: MIT License', ], - ext_modules = [Extension('sendfile', - sources=['sendfilemodule.c'], - libraries=libraries)], - ) + ext_modules=[Extension('sendfile', + sources=['sendfilemodule.c'], + libraries=libraries)]) main() diff --git a/test/benchmark.py b/test/benchmark.py index 858bb8f..0d864c4 100644 --- a/test/benchmark.py +++ b/test/benchmark.py @@ -1,10 +1,9 @@ #!/usr/bin/env python -# $Id$ # ====================================================================== # This software is distributed under the MIT license reproduced below: # -# Copyright (C) 2009-2012 Giampaolo Rodola' +# Copyright (C) 2009-2014 Giampaolo Rodola' # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby @@ -43,34 +42,34 @@ Works with both python 2.X and 3.X. """ from __future__ import with_statement -import socket -import os -import errno -import timeit -import time import atexit -import sys -import optparse -import threading +import contextlib +import errno import itertools +import optparse +import os import signal -import contextlib +import socket +import sys +import threading +import time +import timeit from multiprocessing import Process from sendfile import sendfile - - # overridable defaults HOST = "127.0.0.1" PORT = 8022 BIGFILE = "$testfile1" -BIGFILE_SIZE = 1024 * 1024 * 1024 # 1 GB +BIGFILE_SIZE = 1024 * 1024 * 1024 # 1 GB BUFFER_SIZE = 65536 + # python 3 compatibility layer def b(s): - return bytes(s, 'ascii') if sys.version_info >= (3,) else s + return bytes(s, 'ascii') if sys.version_info >= (3, ) else s + # python 2.5 compatibility try: @@ -79,6 +78,7 @@ except NameError: def next(iterator): return iterator.next() + def print_(s, hilite=False): if hilite: bold = '1' @@ -86,6 +86,7 @@ def print_(s, hilite=False): sys.stdout.write(s + "\n") sys.stdout.flush() + def create_file(filename, size): with open(filename, 'wb') as f: bytes = 0 @@ -96,6 +97,7 @@ def create_file(filename, size): if bytes >= size: break + def safe_remove(file): try: os.remove(file) @@ -158,7 +160,8 @@ def start_server(use_sendfile, keep_sending=False): file = open(BIGFILE, 'rb') def on_exit(signum, fram): - file.close(); + file.close() + conn.close() sys.exit(0) signal.signal(signal.SIGTERM, on_exit) @@ -213,7 +216,7 @@ def main(): print_("starting benchmark...") # CPU time: use sendfile() - server = Process(target=start_server, kwargs={"use_sendfile":True}) + server = Process(target=start_server, kwargs={"use_sendfile": True}) server.start() time.sleep(0.1) t1 = timeit.Timer(setup="from __main__ import Client; client = Client()", @@ -222,7 +225,7 @@ def main(): server.join() # CPU time: use send() - server = Process(target=start_server, kwargs={"use_sendfile":False}) + server = Process(target=start_server, kwargs={"use_sendfile": False}) server.start() time.sleep(0.1) t2 = timeit.Timer(setup="from __main__ import Client; client = Client()", @@ -231,8 +234,8 @@ def main(): server.join() # MB/sec: use sendfile() - server = Process(target=start_server, kwargs={"use_sendfile":True, - "keep_sending":True}) + server = Process(target=start_server, kwargs={"use_sendfile": True, + "keep_sending": True}) server.start() time.sleep(0.1) client = Client() @@ -241,8 +244,8 @@ def main(): server.join() # MB/sec: use sendfile() - server = Process(target=start_server, kwargs={"use_sendfile":False, - "keep_sending":True}) + server = Process(target=start_server, kwargs={"use_sendfile": False, + "keep_sending": True}) server.start() time.sleep(0.1) client = Client() @@ -259,6 +262,7 @@ def main(): print_(" cpu: %7.2f usec/pass" % (1000000 * t1 / 100000)) print_(" rate: %7.2f MB/sec" % round(bytes1 / 1024.0 / 1024.0, 2)) + if __name__ == '__main__': s = Spinner() s.start() diff --git a/test/test_sendfile.py b/test/test_sendfile.py index 01ab5fc..ce223b1 100644 --- a/test/test_sendfile.py +++ b/test/test_sendfile.py @@ -1,12 +1,9 @@ #!/usr/bin/env python -# -# $Id$ -# # ====================================================================== # This software is distributed under the MIT license reproduced below: # -# Copyright (C) 2009-2012 Giampaolo Rodola' +# Copyright (C) 2009-2014 Giampaolo Rodola' # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby @@ -35,22 +32,23 @@ during tests. from __future__ import with_statement -import unittest +import asynchat +import asyncore +import atexit +import errno +import optparse import os -import sys import socket -import asyncore -import asynchat +import sys import threading -import errno import time -import atexit +import unittest import warnings -import optparse import sendfile -PY3 = sys.version_info >= (3,) +PY3 = sys.version_info >= (3, ) + def b(x): if PY3: @@ -72,12 +70,14 @@ except TypeError: except Exception: SUPPORT_HEADER_TRAILER = True + def safe_remove(file): try: os.remove(file) except OSError: pass + def has_large_file_support(): # taken from Python's Lib/test/test_largefile.py with open(TESTFN, 'wb', buffering=0) as f: @@ -150,7 +150,6 @@ class Server(asyncore.dispatcher, threading.Thread): self.handler_instance = None self._active = False self._active_lock = threading.Lock() - # --- public API @property @@ -174,7 +173,6 @@ class Server(asyncore.dispatcher, threading.Thread): while not getattr(self.handler_instance, "closed", True): time.sleep(0.001) self.stop() - # --- internals def run(self): @@ -201,7 +199,8 @@ class Server(asyncore.dispatcher, threading.Thread): raise -def sendfile_wrapper(sock, file, offset, nbytes=BUFFER_LEN, header="", trailer=""): +def sendfile_wrapper(sock, file, offset, nbytes=BUFFER_LEN, header="", + trailer=""): """A higher level wrapper representing how an application is supposed to use sendfile(). """ @@ -289,7 +288,8 @@ class TestSendfile(unittest.TestCase): def test_header(self): total_sent = 0 header = b("x") * 512 - sent = sendfile.sendfile(self.sockno, self.fileno, 0, header=header) + sent = sendfile.sendfile(self.sockno, self.fileno, 0, + header=header) total_sent += sent offset = BUFFER_LEN while 1: @@ -358,7 +358,6 @@ class TestSendfile(unittest.TestCase): err = sys.exc_info()[1] if err.errno not in (errno.EBUSY, errno.EAGAIN): raise - # --- corner cases def test_offset_overflow(self): @@ -485,13 +484,15 @@ class TestLargeFile(unittest.TestCase): sys.stdout.flush() def create_file(self): - if os.path.isfile(TESTFN3) and os.path.getsize(TESTFN3) >= BIGFILE_SIZE: + if (os.path.isfile(TESTFN3) and + os.path.getsize(TESTFN3) >= BIGFILE_SIZE): return f = open(TESTFN3, 'wb') chunk_len = 65536 chunk = b('x' * chunk_len) total = 0 - timer = RepeatedTimer(1, lambda: self.print_percent(total, BIGFILE_SIZE)) + timer = RepeatedTimer(1, lambda: self.print_percent(total, + BIGFILE_SIZE)) timer.start() try: while 1: @@ -538,6 +539,13 @@ class TestLargeFile(unittest.TestCase): self.assertEqual(file_size, data_len) +def cleanup(): + safe_remove(TESTFN) + safe_remove(TESTFN2) + +atexit.register(cleanup) + + def test_main(): parser = optparse.OptionParser() parser.add_option('-k', '--keepfile', action="store_true", default=False, @@ -546,22 +554,25 @@ def test_main(): if not options.keepfile: atexit.register(lambda: safe_remove(TESTFN3)) - def cleanup(): - safe_remove(TESTFN) - safe_remove(TESTFN2) - - atexit.register(cleanup) - test_suite = unittest.TestSuite() test_suite.addTest(unittest.makeSuite(TestSendfile)) if has_large_file_support(): test_suite.addTest(unittest.makeSuite(TestLargeFile)) else: - atexit.register(warnings.warn, "large files unsupported", RuntimeWarning) + atexit.register(warnings.warn, "large files unsupported", + RuntimeWarning) cleanup() with open(TESTFN, "wb") as f: f.write(DATA) - unittest.TextTestRunner(verbosity=2).run(test_suite) + result = unittest.TextTestRunner(verbosity=2).run(test_suite) + return result.wasSuccessful() + if __name__ == '__main__': - test_main() + try: + if not test_main(): + sys.exit(1) + except (KeyboardInterrupt, SystemExit): + # this will make the threaded server exit immediately + asyncore.socket_map.clear() + raise -- cgit v1.2.1