diff options
| author | Michael Vrhel <michael.vrhel@artifex.com> | 2020-07-02 15:38:27 -0700 |
|---|---|---|
| committer | Michael Vrhel <michael.vrhel@artifex.com> | 2020-07-02 15:38:27 -0700 |
| commit | 20757da77706d4227cd7268113bbffbe7716a6fc (patch) | |
| tree | e9e951cc620fd7a7ad820af2b285f82c744d0cff /toolbin | |
| parent | ade938cc74549ffc8d58b0c1fd5e9be7f5429855 (diff) | |
| download | ghostpdl-20757da77706d4227cd7268113bbffbe7716a6fc.tar.gz | |
Introduce demos folder with csharp and python API examples
Current csharp demo shows creation of wpf viewer. Goal will
be to next show a Linux viewer using mono and the same API file.
Python demo/API brought over from toolbin.
Diffstat (limited to 'toolbin')
| -rwxr-xr-x | toolbin/gsapi.py | 538 | ||||
| -rwxr-xr-x | toolbin/gsapiwrap.py | 699 | ||||
| -rw-r--r-- | toolbin/jlib.py | 1354 |
3 files changed, 0 insertions, 2591 deletions
diff --git a/toolbin/gsapi.py b/toolbin/gsapi.py deleted file mode 100755 index a2d67dbd7..000000000 --- a/toolbin/gsapi.py +++ /dev/null @@ -1,538 +0,0 @@ -#! /usr/bin/env python3 - -''' -Python version of the C API in psi/iapi.h, using ctypes. - -Overview: - - All functions have the same name as the C function that they wrap. - - All functions return an integer error code unless otherwise stated. [The - exceptions are usually for out-parameters, which are returned directly as - part of a tuple.] - -Usage: - - make sodebug - LD_LIBRARY_PATH=sodebugbin ./toolbin/gsapi.py - -Requirements: - - Requires python3.7 or later. - -Limitations as of 2020-06-18: - - Only tested on Linux. - - Only very limited testing on has been done. - - We don't provide gsapi_add_fs() or gsapi_remove_fs(). - -''' - -import collections -import ctypes -import sys - - - -gsapi_revision_t = collections.namedtuple('gsapi_revision_t', - 'product copyright revision revisiondate' - ) - - -def gsapi_revision(): - ''' - Returns (e, r) where <r> is a gsapi_revision_t. - ''' - _r = _gsapi_revision_t() - e = _libgs.gsapi_revision(ctypes.byref(_r), ctypes.sizeof(_r)) - if e: - return e, None - r = gsapi_revision_t( - _r.product.decode('latin-1'), - _r.copyright.decode('latin-1'), - _r.revision, - _r.revisiondate, - ) - return e, r - - -def gsapi_new_instance(caller_handle): - ''' - Returns (e, instance). - ''' - instance = ctypes.c_void_p() - e = _libgs.gsapi_new_instance(ctypes.byref(instance), ctypes.c_void_p(caller_handle)) - return e, instance - - -def gsapi_delete_instance(instance): - e = _libgs.gsapi_delete_instance(instance) - return e - - -def gsapi_set_stdio(instance, stdin_fn, stdout_fn, stderr_fn): - stdin_fn2 = _stdio_fn(stdin_fn) if stdin_fn else None - stdout_fn2 = _stdio_fn(stdout_fn) if stdout_fn else None - stderr_fn2 = _stdio_fn(stderr_fn) if stderr_fn else None - e = _libgs.gsapi_set_stdio(instance, stdout_fn2, stdout_fn2, stdout_fn2) - if not e: - # Need to keep references to call-back functions. - global _gsapi_set_stdio_refs - _gsapi_set_stdio_refs = stdin_fn2, stdout_fn2, stderr_fn2 - return e - - -def gsapi_set_poll(instance, poll_fn): - poll_fn2 = _poll_fn(poll_fn) - e = _libgs.gsapi_set_poll(instance, poll_fn2) - if not e: - global _gsapi_set_poll_refs - _gsapi_set_poll_refs = poll_fn2 - return e - - -display_callback = collections.namedtuple('display_callback', - ' size' - ' version_major' - ' version_minor' - ' display_open' - ' display_preclose' - ' display_close' - ' display_presize' - ' display_size' - ' display_sync' - ' display_page' - ' display_update' - ' display_memalloc' - ' display_memfree' - ' display_separation' - , - defaults=[0]*14, - ) - - -def gsapi_set_display_callback(instance, callback): - assert isinstance(callback, display_callback) - callback2 = _display_callback() - - # Copy from <callback> into <callback2>. - for name, type_ in _display_callback._fields_: - value = getattr(callback, name) - value2 = type_(value) - setattr(callback2, name, value2) - - e = _libgs.gsapi_set_display_callback(instance, ctypes.byref(callback2)) - if not e: - # Ensure that we keep references to callbacks. - global _gsapi_set_display_callback_refs - _gsapi_set_display_callback_refs = callback2 - return e - - -def gsapi_set_default_device_list(instance, list_): - assert isinstance(list_, str) - e = _libgs.gsapi_set_default_device_list( - instance, - list_.encode('latin-1'), - len(list_), - ) - return e - - -def gsapi_get_default_device_list(instance): - ''' - Returns (e, list) where <list> is a string. - ''' - list_ = ctypes.POINTER(ctypes.c_char)() - len_ = ctypes.c_int() - e = _libgs.gsapi_get_default_device_list( - instance, - ctypes.byref(list_), - ctypes.byref(len_), - ) - if e: - return e, '' - return e, list_[:len_.value] - - -GS_ARG_ENCODING_LOCAL = 0 -GS_ARG_ENCODING_UTF8 = 1 -GS_ARG_ENCODING_UTF16LE = 2 - - -def gsapi_set_arg_encoding(instance, encoding): - e = _libgs.gsapi_set_arg_encoding(instance, encoding) - return e - - -def gsapi_init_with_args(instance, args): - # Create copy of args in format expected by C. - argc = len(args) - argv = (_pchar * (argc + 1))() - for i, arg in enumerate(args): - enc_arg = arg.encode('utf-8') - argv[i] = ctypes.create_string_buffer(enc_arg) - argv[argc] = None - - e = _libgs.gsapi_init_with_args(instance, argc, argv) - return e - - -def gsapi_run_string_begin(instance, user_errors): - ''' - Returns (e, exit_code). - ''' - pexit_code = ctypes.c_int() - e = _libgs.gsapi_run_string_begin(instance, user_errors, ctypes.byref(pexit_code)) - return e, pexit_code.value - - -def gsapi_run_string_continue(instance, str_, user_errors): - ''' - Returns (e, exit_code). - ''' - pexit_code = ctypes.c_int() - e = _libgs.gsapi_run_string_continue( - instance, - str_, - len(str_), - user_errors, - ctypes.byref(pexit_code), - ) - return e, pexit_code.value - - -def gsapi_run_string_end(instance, user_errors): - ''' - Returns (e, exit_code). - ''' - pexit_code = ctypes.c_int() - e = _libgs.gsapi_run_string_end(instance, user_errors, ctypes.byref(pexit_code)) - return e, pexit_code.value - - -def gsapi_run_string_with_length(instance, str_, length, user_errors): - ''' - Returns (e, exit_code). - ''' - pexit_code = ctypes.c_int() - e = _libgs.gsapi_run_string_with_length( - instance, - str_, - length, - user_errors, - ctypes.byref(pexit_code), - ) - return e, pexit_code.value - - -def gsapi_run_string(instance, str_, user_errors): - ''' - Returns (e, exit_code). - ''' - pexit_code = ctypes.c_int() - e = _libgs.gsapi_run_string(instance, str_, user_errors, ctypes.byref(pexit_code)) - return e, pexit_code.value - - -def gsapi_run_file(instance, filename, user_errors): - ''' - Returns (e, exit_code). - ''' - pexit_code = ctypes.c_int() - e = _libgs.gsapi_run_file(instance, filename, user_errors, ctypes.byref(pexit_code)) - return e, pexit_code.value - - -def gsapi_exit(instance): - e = _libgs.gsapi_exit(instance) - return e - - -gs_spt_invalid = -1 -gs_spt_null = 0 # void * is NULL. -gs_spt_bool = 1 # void * is NULL (false) or non-NULL (true). -gs_spt_int = 2 # void * is a pointer to an int. -gs_spt_float = 3 # void * is a float *. -gs_spt_name = 4 # void * is a char *. -gs_spt_string = 5 # void * is a char *. -gs_spt_long = 6 # void * is a long *. -gs_spt_i64 = 7 # void * is an int64_t *. -gs_spt_size_t = 8 # void * is a size_t *. - - -def gsapi_set_param(instance, param, value): - param2 = param.encode('latin-1') - if 0: pass - elif isinstance(value, bool): - type2 = gs_spt_bool - value2 = ctypes.byref(ctypes.c_bool(value)) - elif isinstance(value, int): - type2 = gs_spt_i64 - value2 = ctypes.byref(ctypes.c_longlong(value)) - elif isinstance(value, float): - type2 = gs_spt_float - value2 = ctypes.byref(ctypes.c_float(value)) - elif isinstance(value, str): - # We use gs_spt_string, not psapi_spt_name, because the latter doesn't - # copy the string. - type2 = gs_spt_string - value2 = ctypes.c_char_p(value.encode('latin-1')) - else: - assert 0, 'unrecognised type: %s' % type(value) - e = _libgs.gsapi_set_param(instance, type2, param2, value2) - return e - - -GS_PERMIT_FILE_READING = 0 -GS_PERMIT_FILE_WRITING = 1 -GS_PERMIT_FILE_CONTROL = 2 - - -def gsapi_add_control_path(instance, type_, path): - e = _libgs.gsapi_add_control_path(instance, type_, path) - return e - - -def gsapi_remove_control_path(instance, type_, path): - e = _libgs.gsapi_remove_control_path(instance, type_, path) - return e - - -def gsapi_purge_control_paths(instance, type_): - e = _libgs.gsapi_purge_control_paths(instance, type_) - return e - - -def gsapi_activate_path_control(instance, enable): - e = _libgs.gsapi_activate_path_control(instance, enable) - return e - - -def gsapi_is_path_control_active(instance): - e = gsapi.gsapi_is_path_control_active(instance) - return e - - - -# Implementation details. -# - - -_libgs = ctypes.CDLL('libgs.so') - - -class _gsapi_revision_t(ctypes.Structure): - _fields_ = [ - ('product', ctypes.c_char_p), - ('copyright', ctypes.c_char_p), - ('revision', ctypes.c_long), - ('revisiondate', ctypes.c_long), - ] - - -_stdio_fn = ctypes.CFUNCTYPE( - ctypes.c_int, # return - ctypes.c_void_p, # caller_handle - ctypes.POINTER(ctypes.c_char), # str - ctypes.c_int, # len - ) - -_gsapi_set_stdio_refs = None - - -# ctypes representation of int (*poll_fn)(void* caller_handle). -# -_poll_fn = ctypes.CFUNCTYPE( - ctypes.c_int, # return - ctypes.c_void_p, # caller_handle - ) - -_gsapi_set_poll_refs = None - - -# ctypes representation of display_callback. -# -class _display_callback(ctypes.Structure): - _fields_ = [ - ('size', ctypes.c_int), - ('version_major', ctypes.c_int), - ('version_minor', ctypes.c_int), - ('display_open', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - )), - ('display_preclose', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - )), - ('display_close', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - )), - ('display_presize', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - ctypes.c_int, # width - ctypes.c_int, # height - ctypes.c_int, # raster - ctypes.c_uint, # format - )), - ('display_size', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - ctypes.c_int, # width - ctypes.c_int, # height - ctypes.c_int, # raster - ctypes.c_uint, # format - ctypes.c_char_p, # pimage - )), - ('display_sync', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - )), - ('display_page', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - ctypes.c_int, # copies - ctypes.c_int, # flush - )), - ('display_update', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - ctypes.c_int, # x - ctypes.c_int, # y - ctypes.c_int, # w - ctypes.c_int, # h - )), - ('display_memalloc', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - ctypes.c_ulong, # size - )), - ('display_memfree', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - ctypes.c_void_p, # mem - )), - ('display_separation', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_void_p, # handle - ctypes.c_void_p, # device - ctypes.c_int, # component - ctypes.c_char_p, # component_name - ctypes.c_ushort, # c - ctypes.c_ushort, # m - ctypes.c_ushort, # y - ctypes.c_ushort, # k - )), - ] - - -_libgs.gsapi_set_display_callback.argtypes = ( - ctypes.c_void_p, # instance - ctypes.POINTER(_display_callback), # callback - ) - - -_gsapi_set_display_callback_refs = None - - -# See: -# -# https://stackoverflow.com/questions/58598012/ctypes-errors-with-argv -# -_pchar = ctypes.POINTER(ctypes.c_char) -_ppchar = ctypes.POINTER(_pchar) - -_libgs.gsapi_init_with_args.argtypes = ( - ctypes.c_void_p, # instance - ctypes.c_int, # argc - _ppchar, # argv - ) - - -if 0: - # Not implemented yet: - # gsapi_add_fs() - # gsapi_remove_fs() - # - class gsapi_fs_t(ctypes.Structure): - _fields_ = [ - ('open_file', - ctypes.CFUNCTYPE(ctypes.c_int, - ctypes.c_pvoid, # const gs_memory_t *mem - ctypes.c_pvoid, # secret - ctypes.c_char_p, # fname - ctypes.c_char_p, # mode - )), - ] - - - -if __name__ == '__main__': - - # test - # - - print('Running some very simple and incomplete tests...') - - print('libgs: %s' % _libgs) - - e, revision = gsapi_revision() - print('libgs.gsapi_revision => e=%s revision=%s' % (e, revision)) - assert not e - - e, revision = gsapi_revision() - print('gsapi_revision() => e=%s: %s' % (e, revision)) - assert not e - - - e, instance = gsapi_new_instance(0) - print('gsapi_new_instance => e=%s: %s' % (e, instance)) - assert not e - - e = gsapi_set_arg_encoding(instance, GS_ARG_ENCODING_UTF8) - print('gsapi_set_arg_encoding => e=%s' % e) - assert not e - - def stdout_fn(caller_handle, str_, len_): - sys.stdout.write(str_[:len_].decode('latin-1').replace('\n', '\n*** ')) - return len_ - e = gsapi_set_stdio(instance, None, stdout_fn, None) - print('gsapi_set_stdio => e=%s' % e) - assert not e - - d = display_callback() - e = gsapi_set_display_callback(instance, d) - print('gsapi_set_display_callback => e=%s' % e) - assert not e - - e = gsapi_set_default_device_list(instance, 'bmp256 bmp32b bmpgray bmpmono bmpsep1 bmpsep8 ccr cdeskjet cdj1600 cdj500') - print('gsapi_set_default_device_list => e=%s' % e) - assert not e - - e, l = gsapi_get_default_device_list(instance) - print('gsapi_get_default_device_list => e=%s l=%s' % (e, l)) - assert not e - - e = gsapi_init_with_args(instance, ['gs',]) - print('gsapi_init_with_args => e=%s' % e) - assert not e - - for value in 32, True, 3.14, 'hello world': - e = gsapi_set_param(instance, "foo", value); - print('gsapi_set_param %s => e=%s' % (value, e)) - assert not e diff --git a/toolbin/gsapiwrap.py b/toolbin/gsapiwrap.py deleted file mode 100755 index 0ef0bb088..000000000 --- a/toolbin/gsapiwrap.py +++ /dev/null @@ -1,699 +0,0 @@ -#! /usr/bin/env python3 - -''' -Use Swig to build wrappers for gsapi. - -Example usage: - - Note that we use mupdf's scripts/jlib.py, and assume that there is a mupdf - checkout in the parent directory of the ghostpdl checkout - see 'import - jlib' below. - - ./toolbin/gsapiwrap.py --python -l -0 -1 -t - Build python wrapper for gsapi and run simple test. - - ./toolbin/gsapiwrap.py --csharp -l -0 -1 -t - Build C# wrapper for gsapi and run simple test. - -Args: - - -c: - Clean language-specific out-dir. - - -l: - Build libgs.so (by running make). - - -0: - Run swig to generate language-specific files. - - -1: - Generate language wrappers by compiling/linking the files generated by - -0. - - --csharp: - Generate C# wrappers (requires Mono on Linux). Should usually be first - param. - - --python - Generate Python wrappers. Should usually be first param. - - --swig <swig> - Set location of swig binary. - - -t - Run simple test of language wrappers generated by -1. - -Status: - As of 2020-05-22: - Some python wrappers seem to work ok. - - C# wrappers are not implemented for gsapi_set_poll() and - gsapi_set_stdio(). -''' - -import os -import re -import sys -import textwrap - -import jlib - - -def devpython_info(): - ''' - Use python3-config to find libpython.so and python-dev include path etc. - ''' - python_configdir = jlib.system( 'python3-config --configdir', out='return') - libpython_so = os.path.join( - python_configdir.strip(), - f'libpython{sys.version_info[0]}.{sys.version_info[1]}.so', - ) - assert os.path.isfile( libpython_so), f'cannot find libpython_so={libpython_so}' - - python_includes = jlib.system( 'python3-config --includes', out='return') - python_includes = python_includes.strip() - return python_includes, libpython_so - -def swig_version( swig='swig'): - t = jlib.system( f'{swig} -version', out='return') - m = re.search( 'SWIG Version ([0-9]+)[.]([0-9]+)[.]([0-9]+)', t) - assert m - swig_major = int( m.group(1)) - return swig_major - - -dir_ghostpdl = os.path.abspath( f'{__file__}/../../') + '/' - - -def out_dir( language): - if language == 'python': - return 'gsapiwrap/python/' - if language == 'csharp': - return 'gsapiwrap/csharp/' - assert 0 - -def out_so( language): - ''' - Returns name of .so that implements language-specific wrapper. I think - these names have to match what the language runtime requires. - - For python, Swig generates a module foo.py which does 'import _foo'. - - Similarly C# assumes a file called 'libfoo.so'. - ''' - if language == 'python': - return f'{out_dir(language)}_gsapi.so' - if language == 'csharp': - return f'{out_dir(language)}libgsapi.so' - assert 0 - -def lib_gs_info(): - return f'{dir_ghostpdl}sodebugbin/libgs.so', 'make sodebug' - return f'{dir_ghostpdl}sobin/libgs.so', 'make so' - -def lib_gs(): - ''' - Returns name of the gs shared-library. - ''' - return lib_gs_info()[0] - - -def swig_i( swig, language): - ''' - Returns text for a swig .i file for psi/iapi.h. - ''' - swig_major = swig_version( swig) - - - # We need to redeclare or wrap some functions, e.g. to add OUTPUT - # annotations. We use #define, %ignore and #undef to hide the original - # declarations in the .h file. - # - fns_redeclare = ( - 'gsapi_run_file', - 'gsapi_run_string', - 'gsapi_run_string_begin', - 'gsapi_run_string_continue', - 'gsapi_run_string_end', - 'gsapi_run_string_with_length', - 'gsapi_set_poll', - 'gsapi_set_poll_with_handle', - 'gsapi_set_stdio', - 'gsapi_set_stdio_with_handle', - 'gsapi_new_instance', - ) - - - swig_i_text = textwrap.dedent(f''' - %module(directors="1") gsapi - - %include cpointer.i - %pointer_functions(int, pint); - - // This seems to be necessary to make csharp handle OUTPUT args. - // - %include typemaps.i - - // For gsapi_init_with_args(). - %include argcargv.i - - %include cstring.i - - // Include type information in python doc strings. If we have - // swig-4, we can propogate comments from the C api instead, which - // is preferred. - // - {'%feature("autodoc", "3");' if swig_major < 4 else ''} - - %{{ - #include "psi/iapi.h" - //#include "base/gserrors.h" - - // Define wrapper functions that present a modified API that - // swig can cope with. - // - - // Swig cannot handle void** out-param. - // - static void* new_instance( void* caller_handle, int* out) - {{ - void* ret = NULL; - *out = gsapi_new_instance( &ret, caller_handle); - printf( "gsapi_new_instance() returned *out=%i ret=%p\\n", *out, ret); - fflush( stdout); - return ret; - }} - - // Swig cannot handle (const char* str, int strlen) args. - // - static int run_string_continue(void *instance, const char *str, int user_errors, int *pexit_code) {{ - - return gsapi_run_string_continue( instance, str, strlen(str), user_errors, pexit_code); - }} - %}} - - // Strip gsapi_ prefix from all generated names. - // - %rename("%(strip:[gsapi_])s") ""; - - // Tell Swig about gsapi_get_default_device_list()'s out-params, so - // it adds them to the returned object. - // - // I think the '(void) *$1' will ensure that swig code doesn't - // attempt to free() the returned string. - // - {'%cstring_output_allocate_size(char **list, int *listlen, (void) *$1);' if language == 'python' else ''} - - // Tell swig about the (argc,argv) args in gsapi_init_with_args(). - // - %apply (int ARGC, char **ARGV) {{ (int argc, char **argv) }} - - // Support for wrapping various functions that take function - // pointer args. For each, we define a wrapper function that, - // instead of having function pointer args, takes a class with - // virtual methods. This allows swig to wrap things - python/c# etc - // can create a derived class that implements these virtual methods - // in the python/c# world. - // - - // Wrap gsapi_set_stdio_with_handle(). - // - %feature("director") set_stdio_class; - - %inline {{ - struct set_stdio_class {{ - - virtual int stdin_fn( char* buf, int len) = 0; - virtual int stdout_fn( const char* buf, int len) = 0; - virtual int stderr_fn( const char* buf, int len) = 0; - - static int stdin_fn_wrap( void *caller_handle, char *buf, int len) {{ - return ((set_stdio_class*) caller_handle)->stdin_fn(buf, len); - }} - static int stdout_fn_wrap( void *caller_handle, const char *buf, int len) {{ - return ((set_stdio_class*) caller_handle)->stdout_fn(buf, len); - }} - static int stderr_fn_wrap( void *caller_handle, const char *buf, int len) {{ - return ((set_stdio_class*) caller_handle)->stderr_fn(buf, len); - }} - - virtual ~set_stdio_class() {{}} - }}; - - int set_stdio_with_class( void *instance, set_stdio_class* class_) {{ - return gsapi_set_stdio_with_handle( - instance, - set_stdio_class::stdin_fn_wrap, - set_stdio_class::stdout_fn_wrap, - set_stdio_class::stderr_fn_wrap, - (void*) class_ - ); - }} - - - }} - - // Wrap gsapi_set_poll(). - // - %feature("director") set_poll_class; - - %inline {{ - struct set_poll_class {{ - virtual int poll_fn() = 0; - - static int poll_fn_wrap( void* caller_handle) {{ - return ((set_poll_class*) caller_handle)->poll_fn(); - }} - - virtual ~set_poll_class() {{}} - }}; - - int set_poll_with_class( void* instance, set_poll_class* class_) {{ - return gsapi_set_poll_with_handle( - instance, - set_poll_class::poll_fn_wrap, - (void*) class_ - ); - }} - - }} - - // For functions that we re-declare (typically to specify OUTPUT on - // one or more args), use a macro to rename the declaration in the - // header file and tell swig to ignore these renamed declarations. - // - ''') - - for fn in fns_redeclare: - swig_i_text += f'#define {fn} {fn}0\n' - - for fn in fns_redeclare: - swig_i_text += f'%ignore {fn}0;\n' - - swig_i_text += textwrap.dedent(f''' - #include "psi/iapi.h" - //#include "base/gserrors.h" - ''') - - for fn in fns_redeclare: - swig_i_text += f'#undef {fn}\n' - - - swig_i_text += textwrap.dedent(f''' - // Tell swig about our wrappers and altered declarations. - // - - // Use swig's OUTPUT annotation for out-parameters. - // - int gsapi_run_file(void *instance, const char *file_name, int user_errors, int *OUTPUT); - int gsapi_run_string_begin(void *instance, int user_errors, int *OUTPUT); - int gsapi_run_string_end(void *instance, int user_errors, int *OUTPUT); - //int gsapi_run_string_with_length(void *instance, const char *str, unsigned int length, int user_errors, int *OUTPUT); - int gsapi_run_string(void *instance, const char *str, int user_errors, int *OUTPUT); - - // Declare functions defined above that we want swig to wrap. These - // don't have the gsapi_ prefix, so that they can internally call - // the wrapped gsapi_*() function. [We've told swig to strip the - // gsapi_ prefix on generated functions anyway, so this doesn't - // afffect the generated names.] - // - static int run_string_continue(void *instance, const char *str, int user_errors, int *OUTPUT); - static void* new_instance(void* caller_handle, int* OUTPUT); - ''') - - if language == 'python': - swig_i_text += textwrap.dedent(f''' - - // Define python code that is needed to handle functions with - // function-pointer args. - // - %pythoncode %{{ - - set_stdio_g = None - def set_stdio( instance, stdin, stdout, stderr): - class derived( set_stdio_class): - def stdin_fn( self): - return stdin() - def stdout_fn( self, text, len): - return stdout( text, len) - def stderr_fn( self, text, len): - return stderr( text) - - global set_stdio_g - set_stdio_g = derived() - return set_stdio_with_class( instance, set_stdio_g) - - set_poll_g = None - def set_poll( instance, fn): - class derived( set_poll_class): - def poll_fn( self): - return fn() - global set_poll_g - set_poll_g = derived() - return set_poll_with_class( instance, set_poll_g) - %}} - ''') - - return swig_i_text - - - -def run_swig( swig, language): - ''' - Runs swig using a generated .i file. - - The .i file modifies the gsapi API in places to allow specification of - out-parameters that swig understands - e.g. void** OUTPUT doesn't work. - ''' - os.makedirs( out_dir(language), exist_ok=True) - swig_major = swig_version( swig) - - swig_i_text = swig_i( swig, language) - swig_i_filename = f'{out_dir(language)}iapi.i' - jlib.update_file( swig_i_text, swig_i_filename) - - out_cpp = f'{out_dir(language)}gsapi.cpp' - - if language == 'python': - out_lang = f'{out_dir(language)}gsapi.py' - elif language == 'csharp': - out_lang = f'{out_dir(language)}gsapi.cs' - else: - assert 0 - - out_files = (out_cpp, out_lang) - - doxygen_arg = '' - if swig_major >= 4 and language == 'python': - doxygen_arg = '-doxygen' - - extra = '' - if language == 'csharp': - # Tell swig to put all generated csharp code into a single file. - extra = f'-outfile gsapi.cs' - - command = (textwrap.dedent(f''' - {swig} - -Wall - -c++ - -{language} - {doxygen_arg} - -module gsapi - -outdir {out_dir(language)} - -o {out_cpp} - {extra} - -includeall - -I{dir_ghostpdl} - -ignoremissing - {swig_i_filename} - ''').strip().replace( '\n', ' \\\n') - ) - - jlib.build( - (swig_i_filename,), - out_files, - command, - prefix=' ', - ) - - -def main( argv): - - swig = 'swig' - language = 'python' - - args = jlib.Args( sys.argv[1:]) - while 1: - try: - arg = args.next() - except StopIteration: - break - - if 0: - pass - - elif arg == '-c': - jlib.system( f'rm {out_dir(language)}* || true', verbose=1, prefix=' ') - - elif arg == '-l': - command = lib_gs_info()[1] - jlib.system( command, verbose=1, prefix=' ') - - elif arg == '-0': - run_swig( swig, language) - - elif arg == '-1': - - libs = [lib_gs()] - includes = [dir_ghostpdl] - file_cpp = f'{out_dir(language)}gsapi.cpp' - - if language == 'python': - python_includes, libpython_so = devpython_info() - libs.append( libpython_so) - includes.append( python_includes) - - includes_text = '' - for i in includes: - includes_text += f' -I{i}' - command = textwrap.dedent(f''' - g++ - -g - -Wall -W - -o {out_so(language)} - -fPIC - -shared - {includes_text} - {jlib.link_l_flags(libs)} - {file_cpp} - ''').strip().replace( '\n', ' \\\n') - jlib.build( - (file_cpp, lib_gs(), 'psi/iapi.h'), - (out_so(language),), - command, - prefix=' ', - ) - - elif arg == '--csharp': - language = 'csharp' - - elif arg == '--python': - language = 'python' - - elif arg == '--swig': - swig = args.next() - - elif arg == '-t': - - if language == 'python': - text = textwrap.dedent(''' - #!/usr/bin/env python3 - - import os - import sys - - import gsapi - - gsapi.gs_error_Quit = -101 - - def main(): - minst, code = gsapi.new_instance(None) - print( f'minst={minst} code={code}') - - if 1: - def stdin_local(len): - # Not sure whether this is right. - return sys.stdin.read(len) - def stdout_local(text, l): - sys.stdout.write(text[:l]) - return l - def stderr_local(text, l): - sys.stderr.write(text[:l]) - return l - gsapi.set_stdio( minst, None, stdout_local, stderr_local); - - if 1: - def poll_fn(): - return 0 - gsapi.set_poll(minst, poll_fn) - if 1: - s = 'display x11alpha x11 bbox' - gsapi.set_default_device_list( minst, s, len(s)) - - e, text = gsapi.get_default_device_list( minst) - print( f'gsapi.get_default_device_list() returned e={e} text={text!r}') - - out = 'out.pdf' - if os.path.exists( out): - os.remove( out) - assert not os.path.exists( out) - - gsargv = [''] - gsargv += f'-dNOPAUSE -dBATCH -dSAFER -sDEVICE=pdfwrite -sOutputFile={out} contrib/pcl3/ps/levels-test.ps'.split() - print( f'gsargv={gsargv}') - code = gsapi.set_arg_encoding(minst, gsapi.GS_ARG_ENCODING_UTF8) - if code == 0: - code = gsapi.init_with_args(minst, gsargv) - - code, exit_code = gsapi.run_string_begin( minst, 0) - print( f'gsapi.run_string_begin() returned code={code} exit_code={exit_code}') - assert code == 0 - assert exit_code == 0 - - gsapi.run_string - - code1 = gsapi.exit(minst) - if (code == 0 or code == gsapi.gs_error_Quit): - code = code1 - gsapi.delete_instance(minst) - assert os.path.isfile( out) - if code == 0 or code == gsapi.gs_error_Quit: - return 0 - return 1 - - if __name__ == '__main__': - code = main() - assert code == 0 - sys.exit( code) - ''') - text = text[1:] # skip leading \n. - test_py = f'{out_dir(language)}test.py' - jlib.update_file( text, test_py) - os.chmod( test_py, 0o744) - - jlib.system( - f'LD_LIBRARY_PATH={os.path.abspath( f"{lib_gs()}/..")}' - f' PYTHONPATH={out_dir(language)}' - f' {test_py}' - , - verbose = 1, - prefix=' ', - ) - - elif language == 'csharp': - # See: https://github.com/swig/swig/blob/master/Lib/csharp/typemaps.i - # - text = textwrap.dedent(''' - using System; - public class runme { - static void Main() { - int code; - SWIGTYPE_p_void instance; - Console.WriteLine("hello world"); - instance = gsapi.new_instance(null, out code); - Console.WriteLine("code is: " + code); - gsapi.add_control_path(instance, 0, "hello"); - } - } - ''') - test_cs = f'{out_dir(language)}test.cs' - jlib.update_file( text, test_cs) - files_in = f'{out_dir(language)}gsapi.cs', test_cs - file_out = f'{out_dir(language)}test.exe' - command = f'mono-csc -debug+ -out:{file_out} {" ".join(files_in)}' - jlib.build( files_in, (file_out,), command, prefix=' ') - - ld_library_path = f'{dir_ghostpdl}sobin' - jlib.system( f'LD_LIBRARY_PATH={ld_library_path} {file_out}', verbose=jlib.log, prefix=' ') - - elif arg == '--tt': - # small swig test case. - os.makedirs( 'swig-tt', exist_ok=True) - i = textwrap.dedent(f''' - %include cpointer.i - %include cstring.i - %feature("autodoc", "3"); - %cstring_output_allocate_size(char **list, int *listlen, (void) *$1); - %inline {{ - static inline int gsapi_get_default_device_list(void *instance, char **list, int *listlen) - {{ - *list = (char*) "hello world"; - *listlen = 6; - return 0; - }} - }} - ''') - jlib.update_file(i, 'swig-tt/tt.i') - jlib.system('swig -c++ -python -module tt -outdir swig-tt -o swig-tt/tt.cpp swig-tt/tt.i', verbose=1) - p = textwrap.dedent(f''' - #!/usr/bin/env python3 - import tt - print( tt.gsapi_get_default_device_list(None)) - ''')[1:] - jlib.update_file( p, 'swig-tt/test.py') - python_includes, python_so = devpython_info() - includes = f'-I {python_includes}' - link_flags = jlib.link_l_flags( [python_so]) - jlib.system( f'g++ -shared -fPIC {includes} {link_flags} -o swig-tt/_tt.so swig-tt/tt.cpp', verbose=1) - jlib.system( f'cd swig-tt; python3 test.py', verbose=1) - - elif arg == '-T': - # Very simple test that we can create c# wrapper for trivial code. - os.makedirs( 'swig-cs-test', exist_ok=True) - example_cpp = textwrap.dedent(''' - #include <time.h> - double My_variable = 3.0; - - int fact(int n) { - if (n <= 1) return 1; - else return n*fact(n-1); - } - - int my_mod(int x, int y) { - return (x%y); - } - - char *get_time() - { - time_t ltime; - time(<ime); - return ctime(<ime); - } - ''') - jlib.update_file( example_cpp, 'swig-cs-test/example.cpp') - - example_i = textwrap.dedent(''' - %module example - %{ - /* Put header files here or function declarations like below */ - extern double My_variable; - extern int fact(int n); - extern int my_mod(int x, int y); - extern char *get_time(); - %} - - extern double My_variable; - extern int fact(int n); - extern int my_mod(int x, int y); - extern char *get_time(); - ''') - jlib.update_file( example_i, 'swig-cs-test/example.i') - - runme_cs = textwrap.dedent(''' - using System; - public class runme { - static void Main() { - Console.WriteLine(example.My_variable); - Console.WriteLine(example.fact(5)); - Console.WriteLine(example.get_time()); - } - } - ''') - jlib.update_file( runme_cs, 'swig-cs-test/runme.cs') - jlib.system( 'g++ -g -fPIC -shared -o swig-cs-test/libfoo.so swig-cs-test/example.cpp', verbose=1) - jlib.system( 'swig -c++ -csharp -module example -outdir swig-cs-test -o swig-cs-test/example_wrap.cpp -outfile example.cs swig-cs-test/example.i', verbose=1) - jlib.system( 'g++ -g -fPIC -shared -L swig-cs-test -l foo swig-cs-test/example_wrap.cpp -o swig-cs-test/libexample.so', verbose=1) - jlib.system( 'cd swig-cs-test; mono-csc -out:runme.exe example.cs runme.cs', verbose=1) - jlib.system( 'cd swig-cs-test; LD_LIBRARY_PATH=`pwd` ./runme.exe', verbose=1) - jlib.system( 'ls -l swig-cs-test', verbose=1) - - - else: - raise Exception( f'unrecognised arg: {arg}') - -if __name__ == '__main__': - try: - main( sys.argv) - except Exception as e: - jlib.exception_info( out=sys.stdout) - sys.exit(1) diff --git a/toolbin/jlib.py b/toolbin/jlib.py deleted file mode 100644 index 54ee0a222..000000000 --- a/toolbin/jlib.py +++ /dev/null @@ -1,1354 +0,0 @@ -from __future__ import print_function - -import codecs -import inspect -import io -import os -import shutil -import subprocess -import sys -import time -import traceback - - -def place( frame_record): - ''' - Useful debugging function - returns representation of source position of - caller. - ''' - filename = frame_record.filename - line = frame_record.lineno - function = frame_record.function - ret = os.path.split( filename)[1] + ':' + str( line) + ':' + function + ':' - if 0: - tid = str( threading.currentThread()) - ret = '[' + tid + '] ' + ret - return ret - - -def expand_nv( text, caller): - ''' - Returns <text> with special handling of {<expression>} items. - - text: - String containing {<expression>} items. - caller: - If an int, the number of frames to step up when looking for file:line - information or evaluating expressions. - - Otherwise should be a frame record as returned by inspect.stack()[]. - - <expression> is evaluated in <caller>'s context using eval(), and expanded - to <expression> or <expression>=<value>. - - If <expression> ends with '=', this character is removed and we prefix the - result with <expression>=. - - E.g.: - x = 45 - y = 'hello' - expand_nv( 'foo {x} {y=}') - returns: - foo 45 y=hello - - <expression> can also use ':' and '!' to control formatting, like - str.format(). - ''' - if isinstance( caller, int): - frame_record = inspect.stack()[ caller] - else: - frame_record = caller - frame = frame_record.frame - try: - def get_items(): - ''' - Yields (pre, item), where <item> is contents of next {...} or None, - and <pre> is preceding text. - ''' - pos = 0 - pre = '' - while 1: - if pos == len( text): - yield pre, None - break - rest = text[ pos:] - if rest.startswith( '{{') or rest.startswith( '}}'): - pre += rest[0] - pos += 2 - elif text[ pos] == '{': - close = text.find( '}', pos) - if close < 0: - raise Exception( 'After "{" at offset %s, cannot find closing "}". text is: %r' % ( - pos, text)) - yield pre, text[ pos+1 : close] - pre = '' - pos = close + 1 - else: - pre += text[ pos] - pos += 1 - - ret = '' - for pre, item in get_items(): - ret += pre - nv = False - if item: - if item.endswith( '='): - nv = True - item = item[:-1] - expression, tail = split_first_of( item, '!:') - try: - value = eval( expression, frame.f_globals, frame.f_locals) - value_text = ('{0%s}' % tail).format( value) - except Exception as e: - value_text = '{??Failed to evaluate %r in context %s:%s because: %s??}' % ( - expression, - frame_record.filename, - frame_record.lineno, - e, - ) - if nv: - ret += '%s=' % expression - ret += value_text - - return ret - - finally: - del frame - - -class LogPrefixTime: - def __init__( self, date=False, time_=True, elapsed=False): - self.date = date - self.time = time_ - self.elapsed = elapsed - self.t0 = time.time() - def __call__( self): - ret = '' - if self.date: - ret += time.strftime( ' %F') - if self.time: - ret += time.strftime( ' %T') - if self.elapsed: - ret += ' (+%s)' % time_duration( time.time() - self.t0, s_format='%.1f') - if ret: - ret = ret.strip() + ': ' - return ret - -class LogPrefixFileLine: - def __call__( self, caller): - if isinstance( caller, int): - caller = inspect.stack()[ caller] - return place( caller) + ' ' - -class LogPrefixScopes: - ''' - Internal use only. - ''' - def __init__( self): - self.items = [] - def __call__( self): - ret = '' - for item in self.items: - if callable( item): - item = item() - ret += item - return ret - - -class LogPrefixScope: - ''' - Can be used to insert scoped prefix to log output. - ''' - def __init__( self, prefix): - g_log_prefixe_scopes.items.append( prefix) - def __enter__( self): - pass - def __exit__( self, exc_type, exc_value, traceback): - global g_log_prefix - g_log_prefixe_scopes.items.pop() - - -g_log_delta = 0 - -class LogDeltaScope: - ''' - Can be used to temporarily change verbose level of logging. - - E.g to temporarily increase logging: - - with jlib.LogDeltaScope(-1): - ... - ''' - def __init__( self, delta): - self.delta = delta - global g_log_delta - g_log_delta += self.delta - def __enter__( self): - pass - def __exit__( self, exc_type, exc_value, traceback): - global g_log_delta - g_log_delta -= self.delta - -# Special item that can be inserted into <g_log_prefixes> to enable -# temporary addition of text into log prefixes. -# -g_log_prefixe_scopes = LogPrefixScopes() - -# List of items that form prefix for all output from log(). -# -g_log_prefixes = [] - - -def log_text( text=None, caller=1, nv=True): - ''' - Returns log text, prepending all lines with text from g_log_prefixes. - - text: - The text to output. Each line is prepended with prefix text. - caller: - If an int, the number of frames to step up when looking for file:line - information or evaluating expressions. - - Otherwise should be a frame record as returned by inspect.stack()[]. - nv: - If true, we expand {...} in <text> using expand_nv(). - ''' - if isinstance( caller, int): - caller += 1 - prefix = '' - for p in g_log_prefixes: - if callable( p): - if isinstance( p, LogPrefixFileLine): - p = p(caller) - else: - p = p() - prefix += p - - if text is None: - return prefix - - if nv: - text = expand_nv( text, caller) - - if text.endswith( '\n'): - text = text[:-1] - lines = text.split( '\n') - - text = '' - for line in lines: - text += prefix + line + '\n' - return text - - - -s_log_levels_cache = dict() -s_log_levels_items = [] - -def log_levels_find( caller): - if not s_log_levels_items: - return 0 - - tb = traceback.extract_stack( None, 1+caller) - if len(tb) == 0: - return 0 - filename, line, function, text = tb[0] - - key = function, filename, line, - delta = s_log_levels_cache.get( key) - - if delta is None: - # Calculate and populate cache. - delta = 0 - for item_function, item_filename, item_delta in s_log_levels_items: - if item_function and not function.startswith( item_function): - continue - if item_filename and not filename.startswith( item_filename): - continue - delta = item_delta - break - - s_log_levels_cache[ key] = delta - - return delta - - -def log_levels_add( delta, filename_prefix, function_prefix): - ''' - log() calls from locations with filenames starting with <filename_prefix> - and/or function names starting with <function_prefix> will have <delta> - added to their level. - - Use -ve delta to increase verbosity from particular filename or function - prefixes. - ''' - log( 'adding level: {filename_prefix=!r} {function_prefix=!r}') - - # Sort in reverse order so that long functions and filename specs come - # first. - # - s_log_levels_items.append( (function_prefix, filename_prefix, delta)) - s_log_levels_items.sort( reverse=True) - - -def log( text, level=0, caller=1, nv=True, out=None): - ''' - Writes log text, with special handling of {<expression>} items in <text> - similar to python3's f-strings. - - text: - The text to output. - caller: - How many frames to step up to get caller's context when evaluating - file:line information and/or expressions. Or frame record as returned - by inspect.stack()[]. - nv: - If true, we expand {...} in <text> using expand_nv(). - out: - Where to send output. If None we use sys.stdout. - - <expression> is evaluated in our caller's context (<n> stack frames up) - using eval(), and expanded to <expression> or <expression>=<value>. - - If <expression> ends with '=', this character is removed and we prefix the - result with <expression>=. - - E.g.: - x = 45 - y = 'hello' - expand_nv( 'foo {x} {y=}') - returns: - foo 45 y=hello - - <expression> can also use ':' and '!' to control formatting, like - str.format(). - ''' - if out is None: - out = sys.stdout - level += g_log_delta - if isinstance( caller, int): - caller += 1 - level += log_levels_find( caller) - if level <= 0: - text = log_text( text, caller, nv=nv) - out.write( text) - out.flush() - - -def log0( text, caller=1, nv=True, out=None): - ''' - Most verbose log. Same as log(). - ''' - log( text, level=0, caller=caller+1, nv=nv, out=out) - -def log1( text, caller=1, nv=True, out=None): - log( text, level=1, caller=caller+1, nv=nv, out=out) - -def log2( text, caller=1, nv=True, out=None): - log( text, level=2, caller=caller+1, nv=nv, out=out) - -def log3( text, caller=1, nv=True, out=None): - log( text, level=3, caller=caller+1, nv=nv, out=out) - -def log4( text, caller=1, nv=True, out=None): - log( text, level=4, caller=caller+1, nv=nv, out=out) - -def log5( text, caller=1, nv=True, out=None): - ''' - Least verbose log. - ''' - log( text, level=5, caller=caller+1, nv=nv, out=out) - -def logx( text, caller=1, nv=True, out=None): - ''' - Does nothing, useful when commenting out a log(). - ''' - pass - - -def log_levels_add_env( name='JLIB_log_levels'): - ''' - Added log levels encoded in an environmental variable. - ''' - t = os.environ.get( name) - if t: - for ffll in t.split( ','): - ffl, delta = ffll.split( '=', 1) - delta = int( delta) - ffl = ffl.split( ':') - if 0: - pass - elif len( ffl) == 1: - filename = ffl - function = None - elif len( ffl) == 2: - filename, function = ffl - else: - assert 0 - log_levels_add( delta, filename, function) - - -def strpbrk( text, substrings): - ''' - Finds first occurrence of any item in <substrings> in <text>. - - Returns (pos, substring) or (len(text), None) if not found. - ''' - ret_pos = len( text) - ret_substring = None - for substring in substrings: - pos = text.find( substring) - if pos >= 0 and pos < ret_pos: - ret_pos = pos - ret_substring = substring - return ret_pos, ret_substring - - -def split_first_of( text, substrings): - ''' - Returns (pre, post), where <pre> doesn't contain any item in <substrings> - and <post> is empty or starts with an item in <substrings>. - ''' - pos, _ = strpbrk( text, substrings) - return text[ :pos], text[ pos:] - - - -log_levels_add_env() - - -def force_line_buffering(): - ''' - Ensure sys.stdout and sys.stderr are line-buffered. E.g. makes things work - better if output is piped to a file via 'tee'. - - Returns original out,err streams. - ''' - stdout0 = sys.stdout - stderr0 = sys.stderr - sys.stdout = os.fdopen( os.dup( sys.stdout.fileno()), 'w', 1) - sys.stderr = os.fdopen( os.dup( sys.stderr.fileno()), 'w', 1) - return stdout0, stderr0 - - -def exception_info( exception=None, limit=None, out=None, prefix='', oneline=False): - ''' - General replacement for traceback.* functions that print/return information - about exceptions. This function provides a simple way of getting the - functionality provided by these traceback functions: - - traceback.format_exc() - traceback.format_exception() - traceback.print_exc() - traceback.print_exception() - - Returns: - A string containing description of specified exception and backtrace. - - Inclusion of outer frames: - We improve upon traceback.* in that we also include stack frames above - the point at which an exception was caught - frames from the top-level - <module> or thread creation fn to the try..catch block, which makes - backtraces much more useful. - - Google 'sys.exc_info backtrace incomplete' for more details. - - We deliberately leave a slightly curious pair of items in the backtrace - - the point in the try: block that ended up raising an exception, and - the point in the associated except: block from which we were called. - - For clarity, we insert an empty frame in-between these two items, so - that one can easily distinguish the two parts of the backtrace. - - So the backtrace looks like this: - - root (e.g. <module> or /usr/lib/python2.7/threading.py:778:__bootstrap(): - ... - file:line in the except: block where the exception was caught. - ::(): marker - file:line in the try: block. - ... - file:line where the exception was raised. - - The items after the ::(): marker are the usual items that traceback.* - shows for an exception. - - Also the backtraces that are generated are more concise than those provided - by traceback.* - just one line per frame instead of two - and filenames are - output relative to the current directory if applicatble. And one can easily - prefix all lines with a specified string, e.g. to indent the text. - - Returns a string containing backtrace and exception information, and sends - returned string to <out> if specified. - - exception: - None, or a (type, value, traceback) tuple, e.g. from sys.exc_info(). If - None, we call sys.exc_info() and use its return value. - limit: - None or maximum number of stackframes to output. - out: - None or callable taking single <text> parameter or object with a - 'write' member that takes a single <text> parameter. - prefix: - Used to prefix all lines of text. - ''' - if exception is None: - exception = sys.exc_info() - etype, value, tb = exception - - if sys.version_info[0] == 2: - out2 = io.BytesIO() - else: - out2 = io.StringIO() - try: - - frames = [] - - # Get frames above point at which exception was caught - frames - # starting at top-level <module> or thread creation fn, and ending - # at the point in the catch: block from which we were called. - # - # These frames are not included explicitly in sys.exc_info()[2] and are - # also omitted by traceback.* functions, which makes for incomplete - # backtraces that miss much useful information. - # - for f in reversed(inspect.getouterframes(tb.tb_frame)): - ff = f[1], f[2], f[3], f[4][0].strip() - frames.append(ff) - - if 1: - # It's useful to see boundary between upper and lower frames. - frames.append( None) - - # Append frames from point in the try: block that caused the exception - # to be raised, to the point at which the exception was thrown. - # - # [One can get similar information using traceback.extract_tb(tb): - # for f in traceback.extract_tb(tb): - # frames.append(f) - # ] - for f in inspect.getinnerframes(tb): - ff = f[1], f[2], f[3], f[4][0].strip() - frames.append(ff) - - cwd = os.getcwd() + os.sep - if oneline: - if etype and value: - # The 'exception_text' variable below will usually be assigned - # something like '<ExceptionType>: <ExceptionValue>', unless - # there was no explanatory text provided (e.g. "raise Exception()"). - # In this case, str(value) will evaluate to ''. - exception_text = traceback.format_exception_only(etype, value)[0].strip() - filename, line, fnname, text = frames[-1] - if filename.startswith(cwd): - filename = filename[len(cwd):] - if not str(value): - # The exception doesn't have any useful explanatory text - # (for example, maybe it was raised by an expression like - # "assert <expression>" without a subsequent comma). In - # the absence of anything more helpful, print the code that - # raised the exception. - exception_text += ' (%s)' % text - line = '%s%s at %s:%s:%s()' % (prefix, exception_text, filename, line, fnname) - out2.write(line) - else: - out2.write( '%sBacktrace:\n' % prefix) - for frame in frames: - if frame is None: - out2.write( '%s ^except raise:\n' % prefix) - continue - filename, line, fnname, text = frame - if filename.startswith( cwd): - filename = filename[ len(cwd):] - if filename.startswith( './'): - filename = filename[ 2:] - out2.write( '%s %s:%s:%s(): %s\n' % ( - prefix, filename, line, fnname, text)) - - if etype and value: - out2.write( '%sException:\n' % prefix) - lines = traceback.format_exception_only( etype, value) - for line in lines: - out2.write( '%s %s' % ( prefix, line)) - - text = out2.getvalue() - - # Write text to <out> if specified. - out = getattr( out, 'write', out) - if callable( out): - out( text) - return text - - finally: - # clear things to avoid cycles. - exception = None - etype = None - value = None - tb = None - frames = None - - -def number_sep( s): - ''' - Simple number formatter, adds commas in-between thousands. <s> can - be a number or a string. Returns a string. - ''' - if not isinstance( s, str): - s = str( s) - c = s.find( '.') - if c==-1: c = len(s) - end = s.find('e') - if end == -1: end = s.find('E') - if end == -1: end = len(s) - ret = '' - for i in range( end): - ret += s[i] - if i<c-1 and (c-i-1)%3==0: - ret += ',' - elif i>c and i<end-1 and (i-c)%3==0: - ret += ',' - ret += s[end:] - return ret - -assert number_sep(1)=='1' -assert number_sep(12)=='12' -assert number_sep(123)=='123' -assert number_sep(1234)=='1,234' -assert number_sep(12345)=='12,345' -assert number_sep(123456)=='123,456' -assert number_sep(1234567)=='1,234,567' - - -class Stream: - ''' - Base layering abstraction for streams - abstraction for things like - sys.stdout to allow prefixing of all output, e.g. with a timestamp. - ''' - def __init__( self, stream): - self.stream = stream - def write( self, text): - self.stream.write( text) - -class StreamPrefix: - ''' - Prefixes output with a prefix, which can be a string or a callable that - takes no parameters and return a string. - ''' - def __init__( self, stream, prefix): - self.stream = stream - self.at_start = True - if callable(prefix): - self.prefix = prefix - else: - self.prefix = lambda : prefix - - def write( self, text): - if self.at_start: - text = self.prefix() + text - self.at_start = False - append_newline = False - if text.endswith( '\n'): - text = text[:-1] - self.at_start = True - append_newline = True - text = text.replace( '\n', '\n%s' % self.prefix()) - if append_newline: - text += '\n' - self.stream.write( text) - - def flush( self): - self.stream.flush() - - -def debug( text): - if callable(text): - text = text() - print( text) - -debug_periodic_t0 = [0] -def debug_periodic( text, override=0): - interval = 10 - t = time.time() - if t - debug_periodic_t0[0] > interval or override: - debug_periodic_t0[0] = t - debug(text) - - -def time_duration( seconds, verbose=False, s_format='%i'): - ''' - Returns string expressing an interval. - - seconds: - The duration in seconds - verbose: - If true, return like '4 days 1 hour 2 mins 23 secs', otherwise as - '4d3h2m23s'. - s_format: - If specified, use as printf-style format string for seconds. - ''' - x = abs(seconds) - ret = '' - i = 0 - for div, text in [ - ( 60, 'sec'), - ( 60, 'min'), - ( 24, 'hour'), - ( None, 'day'), - ]: - force = ( x == 0 and i == 0) - if div: - remainder = x % div - x = int( x/div) - else: - remainder = x - if not verbose: - text = text[0] - if remainder or force: - if verbose and remainder > 1: - # plural. - text += 's' - if verbose: - text = ' %s ' % text - if i == 0: - remainder = s_format % remainder - ret = '%s%s%s' % ( remainder, text, ret) - i += 1 - ret = ret.strip() - if ret == '': - ret = '0s' - if seconds < 0: - ret = '-%s' % ret - return ret - -assert time_duration( 303333) == '3d12h15m33s' -assert time_duration( 303333.33, s_format='%.1f') == '3d12h15m33.3s' -assert time_duration( 303333, verbose=True) == '3 days 12 hours 15 mins 33 secs' -assert time_duration( 303333.33, verbose=True, s_format='%.1f') == '3 days 12 hours 15 mins 33.3 secs' - -assert time_duration( 0) == '0s' -assert time_duration( 0, verbose=True) == '0 sec' - - -def date_time( t=None): - if t is None: - t = time.time() - return time.strftime( "%F-%T", time.gmtime( t)) - -def stream_prefix_time( stream): - ''' - Returns StreamPrefix that prefixes lines with time and elapsed time. - ''' - t_start = time.time() - def prefix_time(): - return '%s (+%s): ' % ( - time.strftime( '%T'), - time_duration( time.time() - t_start, s_format='0.1f'), - ) - return StreamPrefix( stream, prefix_time) - -def stdout_prefix_time(): - ''' - Changes sys.stdout to prefix time and elapsed time; returns original - sys.stdout. - ''' - ret = sys.stdout - sys.stdout = stream_prefix_time( sys.stdout) - return ret - - -def make_stream( out): - ''' - If <out> already has a .write() member, returns <out>. - - Otherwise a stream-like object with a .write() method that writes to <out>. - - out: - Where output is sent. - If None, output is lost. - Otherwise if an integer, we do: os.write( out, text) - Otherwise if callable, we do: out( text) - Otherwise we assume <out> is python stream or similar already. - ''' - if getattr( out, 'write', None): - return out - class Ret: - def flush(): - pass - ret = Ret() - if out is None: - ret.write = lambda text: None - elif isinstance( out, int): - ret.write = lambda text: os.write( out, text) - elif callable( out): - ret.write = out - else: - ret.write = lambda text: out.write( text) - return ret - - -def system_raw( - command, - out=None, - shell=True, - encoding='latin_1', - errors='strict', - buffer_len=-1, - ): - ''' - Runs command, writing output to <out> which can be an int fd, a python - stream or a Stream object. - - Args: - command: - The command to run. - out: - Where output is sent. - If None, output is lost. - If -1, output is sent to stdout and stderr. - Otherwise if an integer, we do: os.write( out, text) - Otherwise if callable, we do: out( text) - Otherwise we assume <out> is python stream or similar, and do: out.write(text) - shell: - Whether to run command inside a shell (see subprocess.Popen). - encoding: - Sepecify the encoding used to translate the command's output - to characters. - - Note that if <encoding> is None and we are being run by python3, - <out> will be passed bytes, not a string. - - Note that latin_1 will never raise a UnicodeDecodeError. - errors: - How to handle encoding errors; see docs for codecs module for - details. - buffer_len: - The number of bytes we attempt to read at a time. If -1 we read - output one line at a time. - - Returns: - subprocess's <returncode>, i.e. -N means killed by signal N, otherwise - the exit value (e.g. 12 if command terminated with exit(12)). - ''' - if out == -1: - stdin = 0 - stdout = 1 - stderr = 2 - else: - stdin = None - stdout = subprocess.PIPE - stderr = subprocess.STDOUT - child = subprocess.Popen( - command, - shell=shell, - stdin=stdin, - stdout=stdout, - stderr=stderr, - close_fds=True, - #encoding=encoding - only python-3.6+. - ) - - child_out = child.stdout - if encoding: - child_out = codecs.getreader( encoding)( child_out, errors) - - out = make_stream( out) - - if stdout == subprocess.PIPE: - if buffer_len == -1: - for line in child_out: - out.write( line) - else: - while 1: - text = child_out.read( buffer_len) - if not text: - break - out.write( text) - #decode( lambda : os.read( child_out.fileno(), 100), outfn, encoding) - - return child.wait() - -if __name__ == '__main__': - - if os.getenv( 'jtest_py_system_raw_test') == '1': - out = io.StringIO() - system_raw( - 'jtest_py_system_raw_test=2 python jlib.py', - sys.stdout, - encoding='utf-8', - #'latin_1', - errors='replace', - ) - print( repr( out.getvalue())) - - elif os.getenv( 'jtest_py_system_raw_test') == '2': - for i in range(256): - sys.stdout.write( chr(i)) - - -def system( - command, - verbose=None, - raise_errors=True, - out=None, - prefix=None, - rusage=False, - shell=True, - encoding=None, - errors='replace', - buffer_len=-1, - ): - ''' - Runs a command like os.system() or subprocess.*, but with more flexibility. - - We give control over where the command's output is sent, whether to return - the output and/or exit code, and whether to raise an exception if the - command fails. - - We also support the use of /usr/bin/time to gather rusage information. - - command: - The command to run. - verbose: - If true, we output information about the command that we run, and - its result. - - If callable or something with a .write() method, information is - sent to <verbose> itself. Otherwise it is sent to <out> (without - applying <prefix>). - raise_errors: - If true, we raise an exception if the command fails, otherwise we - return the failing error code or zero. - out: - Python stream, fd, callable or Stream instance to which output is - sent. - - If <out> is 'return', we buffer the output and return (e, - <output>). Note that if raise_errors is true, we only return if <e> - is zero. - - If -1, output is sent to stdout and stderr. - prefix: - If not None, should be prefix string or callable used to prefix - all output. [This is for convenience to avoid the need to do - out=StreamPrefix(...).] - rusage: - If true, we run via /usr/bin/time and return rusage string - containing information on execution. <raise_errors> and - out='return' are ignored. - shell: - Passed to underlying subprocess.Popen() call. - encoding: - Sepecify the encoding used to translate the command's output - to characters. Defaults to utf-8. - errors: - How to handle encoding errors; see docs for codecs module - for details. Defaults to 'replace' so we never raise a - UnicodeDecodeError. - buffer_len: - The number of bytes we attempt to read at a time. If -1 we read - output one line at a time. - - Returns: - If <rusage> is true, we return the rusage text. - - Else if raise_errors is true: - If the command failed, we raise an exception. - Else if <out> is 'return' we return the text output from the command. - Else we return None - - Else if <out> is 'return', we return (e, text) where <e> is the - command's exit code and <text> is the output from the command. - - Else we return <e>, the command's exit code. - ''' - if encoding is None: - if sys.version_info[0] == 2: - # python-2 doesn't seem to implement 'replace' properly. - encoding = None - errors = None - else: - encoding = 'utf-8' - errors = 'replace' - - out_original = out - if out is None: - out = sys.stdout - elif out == 'return': - # Store the output ourselves so we can return it. - out = io.StringIO() - else: - out = make_stream( out) - - if verbose: - if getattr( verbose, 'write', None): - pass - elif callable( verbose): - verbose = make_stream( verbose) - else: - verbose = out - - if prefix: - out = StreamPrefix( out, prefix) - - if verbose: - verbose.write( 'running: %s\n' % command) - - if rusage: - command2 = '' - command2 += '/usr/bin/time -o ubt-out -f "D=%D E=%D F=%F I=%I K=%K M=%M O=%O P=%P R=%r S=%S U=%U W=%W X=%X Z=%Z c=%c e=%e k=%k p=%p r=%r s=%s t=%t w=%w x=%x C=%C"' - command2 += ' ' - command2 += command - system_raw( command2, out, shell, encoding, errors, buffer_len=buffer_len) - with open('ubt-out') as f: - rusage_text = f.read() - #print 'have read rusage output: %r' % rusage_text - if rusage_text.startswith( 'Command '): - # Annoyingly, /usr/bin/time appears to write 'Command - # exited with ...' or 'Command terminated by ...' to the - # output file before the rusage info if command doesn't - # exit 0. - nl = rusage_text.find('\n') - rusage_text = rusage_text[ nl+1:] - return rusage_text - else: - e = system_raw( command, out, shell, encoding, errors, buffer_len=buffer_len) - - if verbose: - verbose.write( '[returned e=%s]\n' % e) - - if raise_errors: - if e: - raise Exception( 'command failed: %s' % command) - if out_original == 'return': - return out.getvalue() - return - - if out_original == 'return': - return e, out.getvalue() - return e - -def get_gitfiles( directory, submodules=False): - ''' - Returns list of all files known to git in <directory>; <directory> must be - somewhere within a git checkout. - - Returned names are all relative to <directory>. - - If .git directory, we also create <directory>/jtest-git-files. Otherwise we - assume a this file already exists. - ''' - if os.path.isdir( '%s/.git' % directory): - command = 'cd ' + directory + ' && git ls-files' - if submodules: - command += ' --recurse-submodules' - command += ' > jtest-git-files' - system( command, verbose=sys.stdout) - - with open( '%s/jtest-git-files' % directory, 'r') as f: - text = f.read() - ret = text.split( '\n') - return ret - -def get_git_id_raw( directory): - if not os.path.isdir( '%s/.git' % directory): - return - text = system( - f'cd {directory} && (PAGER= git show --pretty=oneline|head -n 1 && git diff)', - out='return', - ) - return text - -def get_git_id( directory, allow_none=False): - ''' - Returns text where first line is '<git-sha> <commit summary>' and remaining - lines contain output from 'git diff' in <directory>. - - directory: - Root of git checkout. - allow_none: - If true, we return None if <directory> is not a git checkout and - jtest-git-id file does not exist. - ''' - filename = f'{directory}/jtest-git-id' - text = get_git_id_raw( directory) - if text: - with open( filename, 'w') as f: - f.write( text) - elif os.path.isfile( filename): - with open( filename) as f: - text = f.read() - else: - if not allow_none: - raise Exception( f'Not in git checkout, and no file {filename}.') - text = None - return text - -class Args: - ''' - Iterates over argv items. Does getopt-style splitting of args starting with - single '-' character. - ''' - def __init__( self, argv): - self.argv = argv - self.pos = 0 - self.pos_sub = None - def next( self): - while 1: - if self.pos >= len(self.argv): - raise StopIteration() - arg = self.argv[self.pos] - if (not self.pos_sub - and arg.startswith('-') - and not arg.startswith('--') - ): - # Start splitting current arg. - self.pos_sub = 1 - if self.pos_sub and self.pos_sub >= len(arg): - # End of '-' sub-arg. - self.pos += 1 - self.pos_sub = None - continue - if self.pos_sub: - # Return '-' sub-arg. - ret = arg[self.pos_sub] - self.pos_sub += 1 - return f'-{ret}' - # Return normal arg. - self.pos += 1 - return arg - -def update_file( text, filename): - ''' - Writes <text> to <filename>. Does nothing if contents of <filename> are - already <text>. - ''' - try: - with open( filename) as f: - text0 = f.read() - except OSError: - text0 = None - if text == text0: - log( 'Unchanged: ' + filename) - else: - log( 'Updating: ' + filename) - # Write to temp file and rename, to ensure we are atomic. - filename_temp = f'{filename}-jlib-temp' - with open( filename_temp, 'w') as f: - f.write( text) - os.rename( filename_temp, filename) - - -def mtime( filename, default=0): - ''' - Returns mtime of file, or <default> if error - e.g. doesn't exist. - ''' - try: - return os.path.getmtime( filename) - except OSError: - return default - -def get_filenames( paths): - ''' - Yields each file in <paths>, walking any directories. - ''' - if isinstance( paths, str): - paths = (paths,) - for name in paths: - if os.path.isdir( name): - for dirpath, dirnames, filenames in os.walk( name): - for filename in filenames: - path = os.path.join( dirpath, filename) - yield path - else: - yield name - -def remove( path): - ''' - Removes file or directory, without raising exception if it doesn't exist. - - We assert-fail if the path still exists when we return, in case of - permission problems etc. - ''' - try: - os.remove( path) - except Exception: - pass - shutil.rmtree( path, ignore_errors=1) - assert not os.path.exists( path) - - -# Things for figuring out whether files need updating, using mtimes. -# -def newest( names): - ''' - Returns mtime of newest file in <filenames>. Returns 0 if no file exists. - ''' - assert isinstance( names, (list, tuple)) - assert names - ret_t = 0 - ret_name = None - for filename in get_filenames( names): - t = mtime( filename) - if t > ret_t: - ret_t = t - ret_name = filename - return ret_t, ret_name - -def oldest( names): - ''' - Returns mtime of oldest file in <filenames> or 0 if no file exists. - ''' - assert isinstance( names, (list, tuple)) - assert names - ret_t = None - ret_name = None - for filename in get_filenames( names): - t = mtime( filename) - if ret_t is None or t < ret_t: - ret_t = t - ret_name = filename - if ret_t is None: - ret_t = 0 - return ret_t, ret_name - -def update_needed( infiles, outfiles): - ''' - If any file in <infiles> is newer than any file in <outfiles>, returns - string description. Otherwise returns None. - ''' - in_tmax, in_tmax_name = newest( infiles) - out_tmin, out_tmin_name = oldest( outfiles) - if in_tmax > out_tmin: - text = f'{in_tmax_name} is newer than {out_tmin_name}' - return text - -def build( - infiles, - outfiles, - command, - force_rebuild=False, - out=None, - all_reasons=False, - verbose=True, - prefix=None, - ): - ''' - Ensures that <outfiles> are up to date using enhanced makefile-like - determinism of dependencies. - - Rebuilds <outfiles> by running <command> if we determine that any of them - are out of date. - - infiles: - Names of files that are read by <command>. Can be a single filename. If - an item is a directory, we expand to all filenames in the directory's - tree. - outfiles: - Names of files that are written by <command>. Can also be a single - filename. - command: - Command to run. - force_rebuild: - If true, we always re-run the command. - out: - A callable, passed to jlib.system(). If None, we use jlib.log() with - our caller's stack record. - all_reasons: - If true we check all ways for a build being needed, even if we already - know a build is needed; this only affects the diagnostic that we - output. - verbose: - Passed to jlib.system(). - prefix: - Passed to jlib.system(). - - We compare mtimes of <infiles> and <outfiles>, and we also detect changes - to the command itself. - - If any of infiles are newer than any of outfiles, or <command> is - different to contents of commandfile '<outfile[0]>.cmd, then truncates - commandfile and runs <command>. If <command> succeeds we writes <command> - to commandfile. - ''' - if isinstance( infiles, str): - infiles = (infiles,) - if isinstance( outfiles, str): - infiles = (outfiles,) - - if not out: - out_frame_record = inspect.stack()[1] - out = lambda text: log( text, nv=0, caller=out_frame_record) - - command_filename = f'{outfiles[0]}.cmd' - - reasons = [] - - if not reasons or all_reasons: - if force_rebuild: - reasons.append( 'force_rebuild was specified') - - if not reasons or all_reasons: - try: - with open( command_filename) as f: - command0 = f.read() - except Exception: - command0 = None - if command != command0: - if command0: - reasons.append( 'command has changed') - else: - reasons.append( 'no previous command') - - if not reasons or all_reasons: - reason = update_needed( infiles, outfiles) - if reason: - reasons.append( reason) - - if not reasons: - out( 'Already up to date: ' + ' '.join(outfiles)) - return - - if out: - out( 'Rebuilding because %s: %s' % ( - ', and '.join( reasons), - ' '.join(outfiles), - )) - - # Empty <command_filename) while we run the command so that if command - # fails but still creates target(s), then next time we will know target(s) - # are not up to date. - # - with open( command_filename, 'w') as f: - pass - - system( command, out=out, verbose=verbose, prefix=prefix) - - with open( command_filename, 'w') as f: - f.write( command) - - -def link_l_flags( sos): - ''' - Returns flags needed to link with items in <sos>. For each unique item we - use -L with parent directory, and -l with embedded name (without leading - 'lib' or trailing '.co'). - ''' - dirs = set() - names = [] - if isinstance( sos, str): - sos = (sos,) - for so in sos: - dir_ = os.path.dirname( so) - name = os.path.basename( so) - assert name.startswith( 'lib') - assert name.endswith ( '.so') - name = name[3:-3] - dirs.add( dir_) - names.append( name) - ret = '' - # Important to use sorted() here, otherwise ordering from set() is - # arbitrary causing occasional spurious rebuilds by jlib.build(). - for dir_ in sorted(dirs): - ret += f' -L {dir_}' - for name in names: - ret += f' -l {name}' - return ret |