Added experimental swig wrapper for gsapi.

2 files changed, 2053 insertions, 0 deletions

diff --git a/toolbin/gsapiwrap.py b/toolbin/gsapiwrap.py
new file mode 100755
index 000000000..0ef0bb088
--- /dev/null
+++ b/toolbin/gsapiwrap.py
@@ -0,0 +1,699 @@
+#! /usr/bin/env python3
+
+'''
+Use Swig to build wrappers for gsapi.
+
+Example usage:
+
+    Note that we use mupdf's scripts/jlib.py, and assume that there is a mupdf
+    checkout in the parent directory of the ghostpdl checkout - see 'import
+    jlib' below.
+
+    ./toolbin/gsapiwrap.py --python -l -0 -1 -t
+        Build python wrapper for gsapi and run simple test.
+
+    ./toolbin/gsapiwrap.py --csharp -l -0 -1 -t
+        Build C# wrapper for gsapi and run simple test.
+
+Args:
+
+    -c:
+        Clean language-specific out-dir.
+
+    -l:
+        Build libgs.so (by running make).
+
+    -0:
+        Run swig to generate language-specific files.
+
+    -1:
+        Generate language wrappers by compiling/linking the files generated by
+        -0.
+
+    --csharp:
+        Generate C# wrappers (requires Mono on Linux). Should usually be first
+        param.
+
+    --python
+        Generate Python wrappers. Should usually be first param.
+
+    --swig <swig>
+        Set location of swig binary.
+
+    -t
+        Run simple test of language wrappers generated by -1.
+
+Status:
+    As of 2020-05-22:
+        Some python wrappers seem to work ok.
+
+        C# wrappers are not implemented for gsapi_set_poll() and
+        gsapi_set_stdio().
+'''
+
+import os
+import re
+import sys
+import textwrap
+
+import jlib
+
+
+def devpython_info():
+    '''
+    Use python3-config to find libpython.so and python-dev include path etc.
+    '''
+    python_configdir = jlib.system( 'python3-config --configdir', out='return')
+    libpython_so = os.path.join(
+            python_configdir.strip(),
+            f'libpython{sys.version_info[0]}.{sys.version_info[1]}.so',
+            )
+    assert os.path.isfile( libpython_so), f'cannot find libpython_so={libpython_so}'
+
+    python_includes = jlib.system( 'python3-config --includes', out='return')
+    python_includes = python_includes.strip()
+    return python_includes, libpython_so
+
+def swig_version( swig='swig'):
+    t = jlib.system( f'{swig} -version', out='return')
+    m = re.search( 'SWIG Version ([0-9]+)[.]([0-9]+)[.]([0-9]+)', t)
+    assert m
+    swig_major = int( m.group(1))
+    return swig_major
+
+
+dir_ghostpdl = os.path.abspath( f'{__file__}/../../') + '/'
+
+
+def out_dir( language):
+    if language == 'python':
+        return 'gsapiwrap/python/'
+    if language == 'csharp':
+        return 'gsapiwrap/csharp/'
+    assert 0
+
+def out_so( language):
+    '''
+    Returns name of .so that implements language-specific wrapper. I think
+    these names have to match what the language runtime requires.
+
+    For python, Swig generates a module foo.py which does 'import _foo'.
+
+    Similarly C# assumes a file called 'libfoo.so'.
+    '''
+    if language == 'python':
+        return f'{out_dir(language)}_gsapi.so'
+    if language == 'csharp':
+        return f'{out_dir(language)}libgsapi.so'
+    assert 0
+
+def lib_gs_info():
+    return f'{dir_ghostpdl}sodebugbin/libgs.so', 'make sodebug'
+    return f'{dir_ghostpdl}sobin/libgs.so', 'make so'
+
+def lib_gs():
+        '''
+        Returns name of the gs shared-library.
+        '''
+        return lib_gs_info()[0]
+
+
+def swig_i( swig, language):
+    '''
+    Returns text for a swig .i file for psi/iapi.h.
+    '''
+    swig_major = swig_version( swig)
+
+
+    # We need to redeclare or wrap some functions, e.g. to add OUTPUT
+    # annotations. We use #define, %ignore and #undef to hide the original
+    # declarations in the .h file.
+    #
+    fns_redeclare = (
+            'gsapi_run_file',
+            'gsapi_run_string',
+            'gsapi_run_string_begin',
+            'gsapi_run_string_continue',
+            'gsapi_run_string_end',
+            'gsapi_run_string_with_length',
+            'gsapi_set_poll',
+            'gsapi_set_poll_with_handle',
+            'gsapi_set_stdio',
+            'gsapi_set_stdio_with_handle',
+            'gsapi_new_instance',
+            )
+
+
+    swig_i_text = textwrap.dedent(f'''
+            %module(directors="1") gsapi
+
+            %include cpointer.i
+            %pointer_functions(int, pint);
+
+            // This seems to be necessary to make csharp handle OUTPUT args.
+            //
+            %include typemaps.i
+
+            // For gsapi_init_with_args().
+            %include argcargv.i
+
+            %include cstring.i
+
+            // Include type information in python doc strings. If we have
+            // swig-4, we can propogate comments from the C api instead, which
+            // is preferred.
+            //
+            {'%feature("autodoc", "3");' if swig_major < 4 else ''}
+
+            %{{
+                #include "psi/iapi.h"
+                //#include "base/gserrors.h"
+
+                // Define wrapper functions that present a modified API that
+                // swig can cope with.
+                //
+
+                // Swig cannot handle void** out-param.
+                //
+                static void* new_instance( void* caller_handle, int* out)
+                {{
+                    void*   ret = NULL;
+                    *out = gsapi_new_instance( &ret, caller_handle);
+                    printf( "gsapi_new_instance() returned *out=%i ret=%p\\n", *out, ret);
+                    fflush( stdout);
+                    return ret;
+                }}
+
+                // Swig cannot handle (const char* str, int strlen) args.
+                //
+                static int run_string_continue(void *instance, const char *str, int user_errors, int *pexit_code) {{
+
+                    return gsapi_run_string_continue( instance, str, strlen(str), user_errors, pexit_code);
+                }}
+            %}}
+
+            // Strip gsapi_ prefix from all generated names.
+            //
+            %rename("%(strip:[gsapi_])s") "";
+
+            // Tell Swig about gsapi_get_default_device_list()'s out-params, so
+            // it adds them to the returned object.
+            //
+            // I think the '(void) *$1' will ensure that swig code doesn't
+            // attempt to free() the returned string.
+            //
+            {'%cstring_output_allocate_size(char **list, int *listlen, (void) *$1);' if language == 'python' else ''}
+
+            // Tell swig about the (argc,argv) args in gsapi_init_with_args().
+            //
+            %apply (int ARGC, char **ARGV) {{ (int argc, char **argv) }}
+
+            // Support for wrapping various functions that take function
+            // pointer args. For each, we define a wrapper function that,
+            // instead of having function pointer args, takes a class with
+            // virtual methods. This allows swig to wrap things - python/c# etc
+            // can create a derived class that implements these virtual methods
+            // in the python/c# world.
+            //
+
+            // Wrap gsapi_set_stdio_with_handle().
+            //
+            %feature("director") set_stdio_class;
+
+            %inline {{
+                struct set_stdio_class {{
+
+                    virtual int stdin_fn( char* buf, int len) = 0;
+                    virtual int stdout_fn( const char* buf, int len) = 0;
+                    virtual int stderr_fn( const char* buf, int len) = 0;
+
+                    static int stdin_fn_wrap( void *caller_handle, char *buf, int len) {{
+                        return ((set_stdio_class*) caller_handle)->stdin_fn(buf, len);
+                    }}
+                    static int stdout_fn_wrap( void *caller_handle, const char *buf, int len) {{
+                        return ((set_stdio_class*) caller_handle)->stdout_fn(buf, len);
+                    }}
+                    static int stderr_fn_wrap( void *caller_handle, const char *buf, int len) {{
+                        return ((set_stdio_class*) caller_handle)->stderr_fn(buf, len);
+                    }}
+
+                    virtual ~set_stdio_class() {{}}
+                }};
+
+                int set_stdio_with_class( void *instance, set_stdio_class* class_) {{
+                    return gsapi_set_stdio_with_handle(
+                            instance,
+                            set_stdio_class::stdin_fn_wrap,
+                            set_stdio_class::stdout_fn_wrap,
+                            set_stdio_class::stderr_fn_wrap,
+                            (void*) class_
+                            );
+                }}
+
+
+            }}
+
+            // Wrap gsapi_set_poll().
+            //
+            %feature("director") set_poll_class;
+
+            %inline {{
+                struct set_poll_class {{
+                    virtual int poll_fn() = 0;
+
+                    static int poll_fn_wrap( void* caller_handle) {{
+                        return ((set_poll_class*) caller_handle)->poll_fn();
+                    }}
+
+                    virtual ~set_poll_class() {{}}
+                }};
+
+                int set_poll_with_class( void* instance, set_poll_class* class_) {{
+                    return gsapi_set_poll_with_handle(
+                            instance,
+                            set_poll_class::poll_fn_wrap,
+                            (void*) class_
+                            );
+                }}
+
+            }}
+
+            // For functions that we re-declare (typically to specify OUTPUT on
+            // one or more args), use a macro to rename the declaration in the
+            // header file and tell swig to ignore these renamed declarations.
+            //
+            ''')
+
+    for fn in fns_redeclare:
+        swig_i_text += f'#define {fn} {fn}0\n'
+
+    for fn in fns_redeclare:
+        swig_i_text += f'%ignore {fn}0;\n'
+
+    swig_i_text += textwrap.dedent(f'''
+            #include "psi/iapi.h"
+            //#include "base/gserrors.h"
+            ''')
+
+    for fn in fns_redeclare:
+        swig_i_text += f'#undef {fn}\n'
+
+
+    swig_i_text += textwrap.dedent(f'''
+            // Tell swig about our wrappers and altered declarations.
+            //
+
+            // Use swig's OUTPUT annotation for out-parameters.
+            //
+            int gsapi_run_file(void *instance, const char *file_name, int user_errors, int *OUTPUT);
+            int gsapi_run_string_begin(void *instance, int user_errors, int *OUTPUT);
+            int gsapi_run_string_end(void *instance, int user_errors, int *OUTPUT);
+            //int gsapi_run_string_with_length(void *instance, const char *str, unsigned int length, int user_errors, int *OUTPUT);
+            int gsapi_run_string(void *instance, const char *str, int user_errors, int *OUTPUT);
+
+            // Declare functions defined above that we want swig to wrap. These
+            // don't have the gsapi_ prefix, so that they can internally call
+            // the wrapped gsapi_*() function. [We've told swig to strip the
+            // gsapi_ prefix on generated functions anyway, so this doesn't
+            // afffect the generated names.]
+            //
+            static int run_string_continue(void *instance, const char *str, int user_errors, int *OUTPUT);
+            static void* new_instance(void* caller_handle, int* OUTPUT);
+            ''')
+
+    if language == 'python':
+        swig_i_text += textwrap.dedent(f'''
+
+                // Define python code that is needed to handle functions with
+                // function-pointer args.
+                //
+                %pythoncode %{{
+
+                    set_stdio_g = None
+                    def set_stdio( instance, stdin, stdout, stderr):
+                        class derived( set_stdio_class):
+                            def stdin_fn( self):
+                                return stdin()
+                            def stdout_fn( self, text, len):
+                                return stdout( text, len)
+                            def stderr_fn( self, text, len):
+                                return stderr( text)
+
+                        global set_stdio_g
+                        set_stdio_g = derived()
+                        return set_stdio_with_class( instance, set_stdio_g)
+
+                    set_poll_g = None
+                    def set_poll( instance, fn):
+                        class derived( set_poll_class):
+                            def poll_fn( self):
+                                return fn()
+                        global set_poll_g
+                        set_poll_g = derived()
+                        return set_poll_with_class( instance, set_poll_g)
+                %}}
+                ''')
+
+    return swig_i_text
+
+
+
+def run_swig( swig, language):
+    '''
+    Runs swig using a generated .i file.
+
+    The .i file modifies the gsapi API in places to allow specification of
+    out-parameters that swig understands - e.g. void** OUTPUT doesn't work.
+    '''
+    os.makedirs( out_dir(language), exist_ok=True)
+    swig_major = swig_version( swig)
+
+    swig_i_text = swig_i( swig, language)
+    swig_i_filename = f'{out_dir(language)}iapi.i'
+    jlib.update_file( swig_i_text, swig_i_filename)
+
+    out_cpp = f'{out_dir(language)}gsapi.cpp'
+
+    if language == 'python':
+        out_lang = f'{out_dir(language)}gsapi.py'
+    elif language == 'csharp':
+        out_lang = f'{out_dir(language)}gsapi.cs'
+    else:
+        assert 0
+
+    out_files = (out_cpp, out_lang)
+
+    doxygen_arg = ''
+    if swig_major >= 4 and language == 'python':
+        doxygen_arg = '-doxygen'
+
+    extra = ''
+    if language == 'csharp':
+        # Tell swig to put all generated csharp code into a single file.
+        extra = f'-outfile gsapi.cs'
+
+    command = (textwrap.dedent(f'''
+            {swig}
+                -Wall
+                -c++
+                -{language}
+                {doxygen_arg}
+                -module gsapi
+                -outdir {out_dir(language)}
+                -o {out_cpp}
+                {extra}
+                -includeall
+                -I{dir_ghostpdl}
+                -ignoremissing
+                {swig_i_filename}
+            ''').strip().replace( '\n', ' \\\n')
+            )
+
+    jlib.build(
+            (swig_i_filename,),
+            out_files,
+            command,
+            prefix='    ',
+            )
+
+
+def main( argv):
+
+    swig = 'swig'
+    language = 'python'
+
+    args = jlib.Args( sys.argv[1:])
+    while 1:
+        try:
+            arg = args.next()
+        except StopIteration:
+            break
+
+        if 0:
+            pass
+
+        elif arg == '-c':
+            jlib.system( f'rm {out_dir(language)}* || true', verbose=1, prefix='    ')
+
+        elif arg == '-l':
+            command = lib_gs_info()[1]
+            jlib.system( command, verbose=1, prefix='    ')
+
+        elif arg == '-0':
+            run_swig( swig, language)
+
+        elif arg == '-1':
+
+            libs = [lib_gs()]
+            includes = [dir_ghostpdl]
+            file_cpp = f'{out_dir(language)}gsapi.cpp'
+
+            if language == 'python':
+                python_includes, libpython_so = devpython_info()
+                libs.append( libpython_so)
+                includes.append( python_includes)
+
+            includes_text = ''
+            for i in includes:
+                includes_text += f' -I{i}'
+            command = textwrap.dedent(f'''
+                    g++
+                        -g
+                        -Wall -W
+                        -o {out_so(language)}
+                        -fPIC
+                        -shared
+                        {includes_text}
+                        {jlib.link_l_flags(libs)}
+                        {file_cpp}
+                    ''').strip().replace( '\n', ' \\\n')
+            jlib.build(
+                    (file_cpp, lib_gs(), 'psi/iapi.h'),
+                    (out_so(language),),
+                    command,
+                    prefix='    ',
+                    )
+
+        elif arg == '--csharp':
+            language = 'csharp'
+
+        elif arg == '--python':
+            language = 'python'
+
+        elif arg == '--swig':
+            swig = args.next()
+
+        elif arg == '-t':
+
+            if language == 'python':
+                text = textwrap.dedent('''
+                        #!/usr/bin/env python3
+
+                        import os
+                        import sys
+
+                        import gsapi
+
+                        gsapi.gs_error_Quit = -101
+
+                        def main():
+                            minst, code = gsapi.new_instance(None)
+                            print( f'minst={minst} code={code}')
+
+                            if 1:
+                                def stdin_local(len):
+                                    # Not sure whether this is right.
+                                    return sys.stdin.read(len)
+                                def stdout_local(text, l):
+                                    sys.stdout.write(text[:l])
+                                    return l
+                                def stderr_local(text, l):
+                                    sys.stderr.write(text[:l])
+                                    return l
+                                gsapi.set_stdio( minst, None, stdout_local, stderr_local);
+
+                            if 1:
+                                def poll_fn():
+                                    return 0
+                                gsapi.set_poll(minst, poll_fn)
+                            if 1:
+                                s = 'display x11alpha x11 bbox'
+                                gsapi.set_default_device_list( minst, s, len(s))
+
+                            e, text = gsapi.get_default_device_list( minst)
+                            print( f'gsapi.get_default_device_list() returned e={e} text={text!r}')
+
+                            out = 'out.pdf'
+                            if os.path.exists( out):
+                                os.remove( out)
+                            assert not os.path.exists( out)
+
+                            gsargv = ['']
+                            gsargv += f'-dNOPAUSE -dBATCH -dSAFER -sDEVICE=pdfwrite -sOutputFile={out} contrib/pcl3/ps/levels-test.ps'.split()
+                            print( f'gsargv={gsargv}')
+                            code = gsapi.set_arg_encoding(minst, gsapi.GS_ARG_ENCODING_UTF8)
+                            if code == 0:
+                                code = gsapi.init_with_args(minst, gsargv)
+
+                            code, exit_code = gsapi.run_string_begin( minst, 0)
+                            print( f'gsapi.run_string_begin() returned code={code} exit_code={exit_code}')
+                            assert code == 0
+                            assert exit_code == 0
+
+                            gsapi.run_string
+
+                            code1 = gsapi.exit(minst)
+                            if (code == 0 or code == gsapi.gs_error_Quit):
+                                code = code1
+                            gsapi.delete_instance(minst)
+                            assert os.path.isfile( out)
+                            if code == 0 or code == gsapi.gs_error_Quit:
+                                return 0
+                            return 1
+
+                        if __name__ == '__main__':
+                            code = main()
+                            assert code == 0
+                            sys.exit( code)
+                        ''')
+                text = text[1:] # skip leading \n.
+                test_py = f'{out_dir(language)}test.py'
+                jlib.update_file( text, test_py)
+                os.chmod( test_py, 0o744)
+
+                jlib.system(
+                        f'LD_LIBRARY_PATH={os.path.abspath( f"{lib_gs()}/..")}'
+                            f' PYTHONPATH={out_dir(language)}'
+                            f' {test_py}'
+                            ,
+                        verbose = 1,
+                        prefix='    ',
+                        )
+
+            elif language == 'csharp':
+                # See: https://github.com/swig/swig/blob/master/Lib/csharp/typemaps.i
+                #
+                text = textwrap.dedent('''
+                        using System;
+                        public class runme {
+                            static void Main() {
+                                int code;
+                                SWIGTYPE_p_void instance;
+                                Console.WriteLine("hello world");
+                                instance = gsapi.new_instance(null, out code);
+                                Console.WriteLine("code is: " + code);
+                                gsapi.add_control_path(instance, 0, "hello");
+                            }
+                        }
+                        ''')
+                test_cs = f'{out_dir(language)}test.cs'
+                jlib.update_file( text, test_cs)
+                files_in = f'{out_dir(language)}gsapi.cs', test_cs
+                file_out = f'{out_dir(language)}test.exe'
+                command = f'mono-csc -debug+ -out:{file_out} {" ".join(files_in)}'
+                jlib.build( files_in, (file_out,), command, prefix='    ')
+
+                ld_library_path = f'{dir_ghostpdl}sobin'
+                jlib.system( f'LD_LIBRARY_PATH={ld_library_path} {file_out}', verbose=jlib.log, prefix='    ')
+
+        elif arg == '--tt':
+            # small swig test case.
+            os.makedirs( 'swig-tt', exist_ok=True)
+            i = textwrap.dedent(f'''
+                    %include cpointer.i
+                    %include cstring.i
+                    %feature("autodoc", "3");
+                    %cstring_output_allocate_size(char **list, int *listlen, (void) *$1);
+                    %inline {{
+                    static inline int gsapi_get_default_device_list(void *instance, char **list, int *listlen)
+                    {{
+                        *list = (char*) "hello world";
+                        *listlen = 6;
+                        return 0;
+                    }}
+                    }}
+                    ''')
+            jlib.update_file(i, 'swig-tt/tt.i')
+            jlib.system('swig -c++ -python -module tt -outdir swig-tt -o swig-tt/tt.cpp swig-tt/tt.i', verbose=1)
+            p = textwrap.dedent(f'''
+                    #!/usr/bin/env python3
+                    import tt
+                    print( tt.gsapi_get_default_device_list(None))
+                    ''')[1:]
+            jlib.update_file( p, 'swig-tt/test.py')
+            python_includes, python_so = devpython_info()
+            includes = f'-I {python_includes}'
+            link_flags = jlib.link_l_flags( [python_so])
+            jlib.system( f'g++ -shared -fPIC {includes} {link_flags} -o swig-tt/_tt.so swig-tt/tt.cpp', verbose=1)
+            jlib.system( f'cd swig-tt; python3 test.py', verbose=1)
+
+        elif arg == '-T':
+            # Very simple test that we can create c# wrapper for trivial code.
+            os.makedirs( 'swig-cs-test', exist_ok=True)
+            example_cpp = textwrap.dedent('''
+                    #include <time.h>
+                    double My_variable = 3.0;
+
+                    int fact(int n) {
+                        if (n <= 1) return 1;
+                        else return n*fact(n-1);
+                    }
+
+                    int my_mod(int x, int y) {
+                        return (x%y);
+                    }
+
+                    char *get_time()
+                    {
+                        time_t ltime;
+                        time(&ltime);
+                        return ctime(&ltime);
+                    }
+                    ''')
+            jlib.update_file( example_cpp, 'swig-cs-test/example.cpp')
+
+            example_i = textwrap.dedent('''
+                    %module example
+                    %{
+                    /* Put header files here or function declarations like below */
+                    extern double My_variable;
+                    extern int fact(int n);
+                    extern int my_mod(int x, int y);
+                    extern char *get_time();
+                    %}
+
+                    extern double My_variable;
+                    extern int fact(int n);
+                    extern int my_mod(int x, int y);
+                    extern char *get_time();
+                    ''')
+            jlib.update_file( example_i, 'swig-cs-test/example.i')
+
+            runme_cs = textwrap.dedent('''
+                    using System;
+                    public class runme {
+                        static void Main() {
+                            Console.WriteLine(example.My_variable);
+                            Console.WriteLine(example.fact(5));
+                            Console.WriteLine(example.get_time());
+                        }
+                    }
+                    ''')
+            jlib.update_file( runme_cs, 'swig-cs-test/runme.cs')
+            jlib.system( 'g++ -g -fPIC -shared -o swig-cs-test/libfoo.so swig-cs-test/example.cpp', verbose=1)
+            jlib.system( 'swig -c++ -csharp -module example -outdir swig-cs-test -o swig-cs-test/example_wrap.cpp -outfile example.cs swig-cs-test/example.i', verbose=1)
+            jlib.system( 'g++ -g -fPIC -shared -L swig-cs-test -l foo swig-cs-test/example_wrap.cpp -o swig-cs-test/libexample.so', verbose=1)
+            jlib.system( 'cd swig-cs-test; mono-csc -out:runme.exe example.cs runme.cs', verbose=1)
+            jlib.system( 'cd swig-cs-test; LD_LIBRARY_PATH=`pwd` ./runme.exe', verbose=1)
+            jlib.system( 'ls -l swig-cs-test', verbose=1)
+
+
+        else:
+            raise Exception( f'unrecognised arg: {arg}')
+
+if __name__ == '__main__':
+    try:
+        main( sys.argv)
+    except Exception as e:
+        jlib.exception_info( out=sys.stdout)
+        sys.exit(1)
diff --git a/toolbin/jlib.py b/toolbin/jlib.py
new file mode 100644
index 000000000..54ee0a222
--- /dev/null
+++ b/toolbin/jlib.py
@@ -0,0 +1,1354 @@
+from __future__ import print_function
+
+import codecs
+import inspect
+import io
+import os
+import shutil
+import subprocess
+import sys
+import time
+import traceback
+
+
+def place( frame_record):
+    '''
+    Useful debugging function - returns representation of source position of
+    caller.
+    '''
+    filename    = frame_record.filename
+    line        = frame_record.lineno
+    function    = frame_record.function
+    ret = os.path.split( filename)[1] + ':' + str( line) + ':' + function + ':'
+    if 0:
+        tid = str( threading.currentThread())
+        ret = '[' + tid + '] ' + ret
+    return ret
+
+
+def expand_nv( text, caller):
+    '''
+    Returns <text> with special handling of {<expression>} items.
+
+    text:
+        String containing {<expression>} items.
+    caller:
+        If an int, the number of frames to step up when looking for file:line
+        information or evaluating expressions.
+
+        Otherwise should be a frame record as returned by inspect.stack()[].
+
+    <expression> is evaluated in <caller>'s context using eval(), and expanded
+    to <expression> or <expression>=<value>.
+
+    If <expression> ends with '=', this character is removed and we prefix the
+    result with <expression>=.
+
+    E.g.:
+        x = 45
+        y = 'hello'
+        expand_nv( 'foo {x} {y=}')
+    returns:
+        foo 45 y=hello
+
+    <expression> can also use ':' and '!' to control formatting, like
+    str.format().
+    '''
+    if isinstance( caller, int):
+        frame_record = inspect.stack()[ caller]
+    else:
+        frame_record = caller
+    frame = frame_record.frame
+    try:
+        def get_items():
+            '''
+            Yields (pre, item), where <item> is contents of next {...} or None,
+            and <pre> is preceding text.
+            '''
+            pos = 0
+            pre = ''
+            while 1:
+                if pos == len( text):
+                    yield pre, None
+                    break
+                rest = text[ pos:]
+                if rest.startswith( '{{') or rest.startswith( '}}'):
+                    pre += rest[0]
+                    pos += 2
+                elif text[ pos] == '{':
+                    close = text.find( '}', pos)
+                    if close < 0:
+                        raise Exception( 'After "{" at offset %s, cannot find closing "}". text is: %r' % (
+                                pos, text))
+                    yield pre, text[ pos+1 : close]
+                    pre = ''
+                    pos = close + 1
+                else:
+                    pre += text[ pos]
+                    pos += 1
+
+        ret = ''
+        for pre, item in get_items():
+            ret += pre
+            nv = False
+            if item:
+                if item.endswith( '='):
+                    nv = True
+                    item = item[:-1]
+                expression, tail = split_first_of( item, '!:')
+                try:
+                    value = eval( expression, frame.f_globals, frame.f_locals)
+                    value_text = ('{0%s}' % tail).format( value)
+                except Exception as e:
+                    value_text = '{??Failed to evaluate %r in context %s:%s because: %s??}' % (
+                            expression,
+                            frame_record.filename,
+                            frame_record.lineno,
+                            e,
+                            )
+                if nv:
+                    ret += '%s=' % expression
+                ret += value_text
+
+        return ret
+
+    finally:
+        del frame
+
+
+class LogPrefixTime:
+    def __init__( self, date=False, time_=True, elapsed=False):
+        self.date = date
+        self.time = time_
+        self.elapsed = elapsed
+        self.t0 = time.time()
+    def __call__( self):
+        ret = ''
+        if self.date:
+            ret += time.strftime( ' %F')
+        if self.time:
+            ret += time.strftime( ' %T')
+        if self.elapsed:
+            ret += ' (+%s)' % time_duration( time.time() - self.t0, s_format='%.1f')
+        if ret:
+            ret = ret.strip() + ': '
+        return ret
+
+class LogPrefixFileLine:
+    def __call__( self, caller):
+        if isinstance( caller, int):
+            caller = inspect.stack()[ caller]
+        return place( caller) + ' '
+
+class LogPrefixScopes:
+    '''
+    Internal use only.
+    '''
+    def __init__( self):
+        self.items = []
+    def __call__( self):
+        ret = ''
+        for item in self.items:
+            if callable( item):
+                item = item()
+            ret += item
+        return ret
+
+
+class LogPrefixScope:
+    '''
+    Can be used to insert scoped prefix to log output.
+    '''
+    def __init__( self, prefix):
+        g_log_prefixe_scopes.items.append( prefix)
+    def __enter__( self):
+        pass
+    def __exit__( self, exc_type, exc_value, traceback):
+        global g_log_prefix
+        g_log_prefixe_scopes.items.pop()
+
+
+g_log_delta = 0
+
+class LogDeltaScope:
+    '''
+    Can be used to temporarily change verbose level of logging.
+
+    E.g to temporarily increase logging:
+
+        with jlib.LogDeltaScope(-1):
+            ...
+    '''
+    def __init__( self, delta):
+        self.delta = delta
+        global g_log_delta
+        g_log_delta += self.delta
+    def __enter__( self):
+        pass
+    def __exit__( self, exc_type, exc_value, traceback):
+        global g_log_delta
+        g_log_delta -= self.delta
+
+# Special item that can be inserted into <g_log_prefixes> to enable
+# temporary addition of text into log prefixes.
+#
+g_log_prefixe_scopes = LogPrefixScopes()
+
+# List of items that form prefix for all output from log().
+#
+g_log_prefixes = []
+
+
+def log_text( text=None, caller=1, nv=True):
+    '''
+    Returns log text, prepending all lines with text from g_log_prefixes.
+
+    text:
+        The text to output. Each line is prepended with prefix text.
+    caller:
+        If an int, the number of frames to step up when looking for file:line
+        information or evaluating expressions.
+
+        Otherwise should be a frame record as returned by inspect.stack()[].
+    nv:
+        If true, we expand {...} in <text> using expand_nv().
+    '''
+    if isinstance( caller, int):
+        caller += 1
+    prefix = ''
+    for p in g_log_prefixes:
+        if callable( p):
+            if isinstance( p, LogPrefixFileLine):
+                p = p(caller)
+            else:
+                p = p()
+        prefix += p
+
+    if text is None:
+        return prefix
+
+    if nv:
+        text = expand_nv( text, caller)
+
+    if text.endswith( '\n'):
+        text = text[:-1]
+    lines = text.split( '\n')
+
+    text = ''
+    for line in lines:
+        text += prefix + line + '\n'
+    return text
+
+
+
+s_log_levels_cache = dict()
+s_log_levels_items = []
+
+def log_levels_find( caller):
+    if not s_log_levels_items:
+        return 0
+
+    tb = traceback.extract_stack( None, 1+caller)
+    if len(tb) == 0:
+        return 0
+    filename, line, function, text = tb[0]
+
+    key = function, filename, line,
+    delta = s_log_levels_cache.get( key)
+
+    if delta is None:
+        # Calculate and populate cache.
+        delta = 0
+        for item_function, item_filename, item_delta in s_log_levels_items:
+            if item_function and not function.startswith( item_function):
+                continue
+            if item_filename and not filename.startswith( item_filename):
+                continue
+            delta = item_delta
+            break
+
+        s_log_levels_cache[ key] = delta
+
+    return delta
+
+
+def log_levels_add( delta, filename_prefix, function_prefix):
+    '''
+    log() calls from locations with filenames starting with <filename_prefix>
+    and/or function names starting with <function_prefix> will have <delta>
+    added to their level.
+
+    Use -ve delta to increase verbosity from particular filename or function
+    prefixes.
+    '''
+    log( 'adding level: {filename_prefix=!r} {function_prefix=!r}')
+
+    # Sort in reverse order so that long functions and filename specs come
+    # first.
+    #
+    s_log_levels_items.append( (function_prefix, filename_prefix, delta))
+    s_log_levels_items.sort( reverse=True)
+
+
+def log( text, level=0, caller=1, nv=True, out=None):
+    '''
+    Writes log text, with special handling of {<expression>} items in <text>
+    similar to python3's f-strings.
+
+    text:
+        The text to output.
+    caller:
+        How many frames to step up to get caller's context when evaluating
+        file:line information and/or expressions. Or frame record as returned
+        by inspect.stack()[].
+    nv:
+        If true, we expand {...} in <text> using expand_nv().
+    out:
+        Where to send output. If None we use sys.stdout.
+
+    <expression> is evaluated in our caller's context (<n> stack frames up)
+    using eval(), and expanded to <expression> or <expression>=<value>.
+
+    If <expression> ends with '=', this character is removed and we prefix the
+    result with <expression>=.
+
+    E.g.:
+        x = 45
+        y = 'hello'
+        expand_nv( 'foo {x} {y=}')
+    returns:
+        foo 45 y=hello
+
+    <expression> can also use ':' and '!' to control formatting, like
+    str.format().
+    '''
+    if out is None:
+        out = sys.stdout
+    level += g_log_delta
+    if isinstance( caller, int):
+        caller += 1
+    level += log_levels_find( caller)
+    if level <= 0:
+        text = log_text( text, caller, nv=nv)
+        out.write( text)
+        out.flush()
+
+
+def log0( text, caller=1, nv=True, out=None):
+    '''
+    Most verbose log. Same as log().
+    '''
+    log( text, level=0, caller=caller+1, nv=nv, out=out)
+
+def log1( text, caller=1, nv=True, out=None):
+    log( text, level=1, caller=caller+1, nv=nv, out=out)
+
+def log2( text, caller=1, nv=True, out=None):
+    log( text, level=2, caller=caller+1, nv=nv, out=out)
+
+def log3( text, caller=1, nv=True, out=None):
+    log( text, level=3, caller=caller+1, nv=nv, out=out)
+
+def log4( text, caller=1, nv=True, out=None):
+    log( text, level=4, caller=caller+1, nv=nv, out=out)
+
+def log5( text, caller=1, nv=True, out=None):
+    '''
+    Least verbose log.
+    '''
+    log( text, level=5, caller=caller+1, nv=nv, out=out)
+
+def logx( text, caller=1, nv=True, out=None):
+    '''
+    Does nothing, useful when commenting out a log().
+    '''
+    pass
+
+
+def log_levels_add_env( name='JLIB_log_levels'):
+    '''
+    Added log levels encoded in an environmental variable.
+    '''
+    t = os.environ.get( name)
+    if t:
+        for ffll in t.split( ','):
+            ffl, delta = ffll.split( '=', 1)
+            delta = int( delta)
+            ffl = ffl.split( ':')
+            if 0:
+                pass
+            elif len( ffl) == 1:
+                filename = ffl
+                function = None
+            elif len( ffl) == 2:
+                filename, function = ffl
+            else:
+                assert 0
+            log_levels_add( delta, filename, function)
+
+
+def strpbrk( text, substrings):
+    '''
+    Finds first occurrence of any item in <substrings> in <text>.
+
+    Returns (pos, substring) or (len(text), None) if not found.
+    '''
+    ret_pos = len( text)
+    ret_substring = None
+    for substring in substrings:
+        pos = text.find( substring)
+        if pos >= 0 and pos < ret_pos:
+            ret_pos = pos
+            ret_substring = substring
+    return ret_pos, ret_substring
+
+
+def split_first_of( text, substrings):
+    '''
+    Returns (pre, post), where <pre> doesn't contain any item in <substrings>
+    and <post> is empty or starts with an item in <substrings>.
+    '''
+    pos, _ = strpbrk( text, substrings)
+    return text[ :pos], text[ pos:]
+
+
+
+log_levels_add_env()
+
+
+def force_line_buffering():
+    '''
+    Ensure sys.stdout and sys.stderr are line-buffered. E.g. makes things work
+    better if output is piped to a file via 'tee'.
+
+    Returns original out,err streams.
+    '''
+    stdout0 = sys.stdout
+    stderr0 = sys.stderr
+    sys.stdout = os.fdopen( os.dup( sys.stdout.fileno()), 'w', 1)
+    sys.stderr = os.fdopen( os.dup( sys.stderr.fileno()), 'w', 1)
+    return stdout0, stderr0
+
+
+def exception_info( exception=None, limit=None, out=None, prefix='', oneline=False):
+    '''
+    General replacement for traceback.* functions that print/return information
+    about exceptions. This function provides a simple way of getting the
+    functionality provided by these traceback functions:
+
+        traceback.format_exc()
+        traceback.format_exception()
+        traceback.print_exc()
+        traceback.print_exception()
+
+    Returns:
+        A string containing description of specified exception and backtrace.
+
+    Inclusion of outer frames:
+        We improve upon traceback.* in that we also include stack frames above
+        the point at which an exception was caught - frames from the top-level
+        <module> or thread creation fn to the try..catch block, which makes
+        backtraces much more useful.
+
+        Google 'sys.exc_info backtrace incomplete' for more details.
+
+        We deliberately leave a slightly curious pair of items in the backtrace
+        - the point in the try: block that ended up raising an exception, and
+        the point in the associated except: block from which we were called.
+
+        For clarity, we insert an empty frame in-between these two items, so
+        that one can easily distinguish the two parts of the backtrace.
+
+        So the backtrace looks like this:
+
+            root (e.g. <module> or /usr/lib/python2.7/threading.py:778:__bootstrap():
+            ...
+            file:line in the except: block where the exception was caught.
+            ::(): marker
+            file:line in the try: block.
+            ...
+            file:line where the exception was raised.
+
+        The items after the ::(): marker are the usual items that traceback.*
+        shows for an exception.
+
+    Also the backtraces that are generated are more concise than those provided
+    by traceback.* - just one line per frame instead of two - and filenames are
+    output relative to the current directory if applicatble. And one can easily
+    prefix all lines with a specified string, e.g. to indent the text.
+
+    Returns a string containing backtrace and exception information, and sends
+    returned string to <out> if specified.
+
+    exception:
+        None, or a (type, value, traceback) tuple, e.g. from sys.exc_info(). If
+        None, we call sys.exc_info() and use its return value.
+    limit:
+        None or maximum number of stackframes to output.
+    out:
+        None or callable taking single <text> parameter or object with a
+        'write' member that takes a single <text> parameter.
+    prefix:
+        Used to prefix all lines of text.
+    '''
+    if exception is None:
+        exception = sys.exc_info()
+    etype, value, tb = exception
+
+    if sys.version_info[0] == 2:
+        out2 = io.BytesIO()
+    else:
+        out2 = io.StringIO()
+    try:
+
+        frames = []
+
+        # Get frames above point at which exception was caught - frames
+        # starting at top-level <module> or thread creation fn, and ending
+        # at the point in the catch: block from which we were called.
+        #
+        # These frames are not included explicitly in sys.exc_info()[2] and are
+        # also omitted by traceback.* functions, which makes for incomplete
+        # backtraces that miss much useful information.
+        #
+        for f in reversed(inspect.getouterframes(tb.tb_frame)):
+            ff = f[1], f[2], f[3], f[4][0].strip()
+            frames.append(ff)
+
+        if 1:
+            # It's useful to see boundary between upper and lower frames.
+            frames.append( None)
+
+        # Append frames from point in the try: block that caused the exception
+        # to be raised, to the point at which the exception was thrown.
+        #
+        # [One can get similar information using traceback.extract_tb(tb):
+        #   for f in traceback.extract_tb(tb):
+        #       frames.append(f)
+        # ]
+        for f in inspect.getinnerframes(tb):
+            ff = f[1], f[2], f[3], f[4][0].strip()
+            frames.append(ff)
+
+        cwd = os.getcwd() + os.sep
+        if oneline:
+            if etype and value:
+                # The 'exception_text' variable below will usually be assigned
+                # something like '<ExceptionType>: <ExceptionValue>', unless
+                # there was no explanatory text provided (e.g. "raise Exception()").
+                # In this case, str(value) will evaluate to ''.
+                exception_text = traceback.format_exception_only(etype, value)[0].strip()
+                filename, line, fnname, text = frames[-1]
+                if filename.startswith(cwd):
+                    filename = filename[len(cwd):]
+                if not str(value):
+                    # The exception doesn't have any useful explanatory text
+                    # (for example, maybe it was raised by an expression like
+                    # "assert <expression>" without a subsequent comma).  In
+                    # the absence of anything more helpful, print the code that
+                    # raised the exception.
+                    exception_text += ' (%s)' % text
+                line = '%s%s at %s:%s:%s()' % (prefix, exception_text, filename, line, fnname)
+                out2.write(line)
+        else:
+            out2.write( '%sBacktrace:\n' % prefix)
+            for frame in frames:
+                if frame is None:
+                    out2.write( '%s    ^except raise:\n' % prefix)
+                    continue
+                filename, line, fnname, text = frame
+                if filename.startswith( cwd):
+                    filename = filename[ len(cwd):]
+                if filename.startswith( './'):
+                    filename = filename[ 2:]
+                out2.write( '%s    %s:%s:%s(): %s\n' % (
+                        prefix, filename, line, fnname, text))
+
+            if etype and value:
+                out2.write( '%sException:\n' % prefix)
+                lines = traceback.format_exception_only( etype, value)
+                for line in lines:
+                    out2.write( '%s    %s' % ( prefix, line))
+
+        text = out2.getvalue()
+
+        # Write text to <out> if specified.
+        out = getattr( out, 'write', out)
+        if callable( out):
+            out( text)
+        return text
+
+    finally:
+        # clear things to avoid cycles.
+        exception = None
+        etype = None
+        value = None
+        tb = None
+        frames = None
+
+
+def number_sep( s):
+    '''
+    Simple number formatter, adds commas in-between thousands. <s> can
+    be a number or a string. Returns a string.
+    '''
+    if not isinstance( s, str):
+        s = str( s)
+    c = s.find( '.')
+    if c==-1:   c = len(s)
+    end = s.find('e')
+    if end == -1:   end = s.find('E')
+    if end == -1:   end = len(s)
+    ret = ''
+    for i in range( end):
+        ret += s[i]
+        if i<c-1 and (c-i-1)%3==0:
+            ret += ','
+        elif i>c and i<end-1 and (i-c)%3==0:
+            ret += ','
+    ret += s[end:]
+    return ret
+
+assert number_sep(1)=='1'
+assert number_sep(12)=='12'
+assert number_sep(123)=='123'
+assert number_sep(1234)=='1,234'
+assert number_sep(12345)=='12,345'
+assert number_sep(123456)=='123,456'
+assert number_sep(1234567)=='1,234,567'
+
+
+class Stream:
+    '''
+    Base layering abstraction for streams - abstraction for things like
+    sys.stdout to allow prefixing of all output, e.g. with a timestamp.
+    '''
+    def __init__( self, stream):
+        self.stream = stream
+    def write( self, text):
+        self.stream.write( text)
+
+class StreamPrefix:
+    '''
+    Prefixes output with a prefix, which can be a string or a callable that
+    takes no parameters and return a string.
+    '''
+    def __init__( self, stream, prefix):
+        self.stream = stream
+        self.at_start = True
+        if callable(prefix):
+            self.prefix = prefix
+        else:
+            self.prefix = lambda : prefix
+
+    def write( self, text):
+        if self.at_start:
+            text = self.prefix() + text
+            self.at_start = False
+        append_newline = False
+        if text.endswith( '\n'):
+            text = text[:-1]
+            self.at_start = True
+            append_newline = True
+        text = text.replace( '\n', '\n%s' % self.prefix())
+        if append_newline:
+            text += '\n'
+        self.stream.write( text)
+
+    def flush( self):
+        self.stream.flush()
+
+
+def debug( text):
+    if callable(text):
+        text = text()
+    print( text)
+
+debug_periodic_t0 = [0]
+def debug_periodic( text, override=0):
+    interval = 10
+    t = time.time()
+    if t - debug_periodic_t0[0] > interval or override:
+        debug_periodic_t0[0] = t
+        debug(text)
+
+
+def time_duration( seconds, verbose=False, s_format='%i'):
+    '''
+    Returns string expressing an interval.
+
+    seconds:
+        The duration in seconds
+    verbose:
+        If true, return like '4 days 1 hour 2 mins 23 secs', otherwise as
+        '4d3h2m23s'.
+    s_format:
+        If specified, use as printf-style format string for seconds.
+    '''
+    x = abs(seconds)
+    ret = ''
+    i = 0
+    for div, text in [
+            ( 60, 'sec'),
+            ( 60, 'min'),
+            ( 24, 'hour'),
+            ( None, 'day'),
+            ]:
+        force = ( x == 0 and i == 0)
+        if div:
+            remainder = x % div
+            x = int( x/div)
+        else:
+            remainder = x
+        if not verbose:
+            text = text[0]
+        if remainder or force:
+            if verbose and remainder > 1:
+                # plural.
+                text += 's'
+            if verbose:
+                text = ' %s ' % text
+            if i == 0:
+                remainder = s_format % remainder
+            ret = '%s%s%s' % ( remainder, text, ret)
+        i += 1
+    ret = ret.strip()
+    if ret == '':
+        ret = '0s'
+    if seconds < 0:
+        ret = '-%s' % ret
+    return ret
+
+assert time_duration( 303333) == '3d12h15m33s'
+assert time_duration( 303333.33, s_format='%.1f') == '3d12h15m33.3s'
+assert time_duration( 303333, verbose=True) == '3 days 12 hours 15 mins 33 secs'
+assert time_duration( 303333.33, verbose=True, s_format='%.1f') == '3 days 12 hours 15 mins 33.3 secs'
+
+assert time_duration( 0) == '0s'
+assert time_duration( 0, verbose=True) == '0 sec'
+
+
+def date_time( t=None):
+    if t is None:
+        t = time.time()
+    return time.strftime( "%F-%T", time.gmtime( t))
+
+def stream_prefix_time( stream):
+    '''
+    Returns StreamPrefix that prefixes lines with time and elapsed time.
+    '''
+    t_start = time.time()
+    def prefix_time():
+        return '%s (+%s): ' % (
+                time.strftime( '%T'),
+                time_duration( time.time() - t_start, s_format='0.1f'),
+                )
+    return StreamPrefix( stream, prefix_time)
+
+def stdout_prefix_time():
+    '''
+    Changes sys.stdout to prefix time and elapsed time; returns original
+    sys.stdout.
+    '''
+    ret = sys.stdout
+    sys.stdout = stream_prefix_time( sys.stdout)
+    return ret
+
+
+def make_stream( out):
+    '''
+    If <out> already has a .write() member, returns <out>.
+
+    Otherwise a stream-like object with a .write() method that writes to <out>.
+
+    out:
+        Where output is sent.
+        If None, output is lost.
+        Otherwise if an integer, we do: os.write( out, text)
+        Otherwise if callable, we do: out( text)
+        Otherwise we assume <out> is python stream or similar already.
+    '''
+    if getattr( out, 'write', None):
+        return out
+    class Ret:
+        def flush():
+            pass
+    ret = Ret()
+    if out is None:
+        ret.write = lambda text: None
+    elif isinstance( out, int):
+        ret.write = lambda text: os.write( out, text)
+    elif callable( out):
+        ret.write = out
+    else:
+        ret.write = lambda text: out.write( text)
+    return ret
+
+
+def system_raw(
+        command,
+        out=None,
+        shell=True,
+        encoding='latin_1',
+        errors='strict',
+        buffer_len=-1,
+        ):
+    '''
+    Runs command, writing output to <out> which can be an int fd, a python
+    stream or a Stream object.
+
+    Args:
+        command:
+            The command to run.
+        out:
+            Where output is sent.
+            If None, output is lost.
+            If -1, output is sent to stdout and stderr.
+            Otherwise if an integer, we do: os.write( out, text)
+            Otherwise if callable, we do: out( text)
+            Otherwise we assume <out> is python stream or similar, and do: out.write(text)
+        shell:
+            Whether to run command inside a shell (see subprocess.Popen).
+        encoding:
+            Sepecify the encoding used to translate the command's output
+            to characters.
+
+            Note that if <encoding> is None and we are being run by python3,
+            <out> will be passed bytes, not a string.
+
+            Note that latin_1 will never raise a UnicodeDecodeError.
+        errors:
+            How to handle encoding errors; see docs for codecs module for
+            details.
+        buffer_len:
+            The number of bytes we attempt to read at a time. If -1 we read
+            output one line at a time.
+
+    Returns:
+        subprocess's <returncode>, i.e. -N means killed by signal N, otherwise
+        the exit value (e.g. 12 if command terminated with exit(12)).
+    '''
+    if out == -1:
+        stdin = 0
+        stdout = 1
+        stderr = 2
+    else:
+        stdin = None
+        stdout = subprocess.PIPE
+        stderr = subprocess.STDOUT
+    child = subprocess.Popen(
+            command,
+            shell=shell,
+            stdin=stdin,
+            stdout=stdout,
+            stderr=stderr,
+            close_fds=True,
+            #encoding=encoding - only python-3.6+.
+            )
+
+    child_out = child.stdout
+    if encoding:
+        child_out = codecs.getreader( encoding)( child_out, errors)
+
+    out = make_stream( out)
+
+    if stdout == subprocess.PIPE:
+        if buffer_len == -1:
+            for line in child_out:
+                out.write( line)
+        else:
+            while 1:
+                text = child_out.read( buffer_len)
+                if not text:
+                    break
+                out.write( text)
+    #decode( lambda : os.read( child_out.fileno(), 100), outfn, encoding)
+
+    return child.wait()
+
+if __name__ == '__main__':
+
+    if os.getenv( 'jtest_py_system_raw_test') == '1':
+        out = io.StringIO()
+        system_raw(
+                'jtest_py_system_raw_test=2 python jlib.py',
+                sys.stdout,
+                encoding='utf-8',
+                #'latin_1',
+                errors='replace',
+                )
+        print( repr( out.getvalue()))
+
+    elif os.getenv( 'jtest_py_system_raw_test') == '2':
+        for i in range(256):
+            sys.stdout.write( chr(i))
+
+
+def system(
+        command,
+        verbose=None,
+        raise_errors=True,
+        out=None,
+        prefix=None,
+        rusage=False,
+        shell=True,
+        encoding=None,
+        errors='replace',
+        buffer_len=-1,
+        ):
+    '''
+    Runs a command like os.system() or subprocess.*, but with more flexibility.
+
+    We give control over where the command's output is sent, whether to return
+    the output and/or exit code, and whether to raise an exception if the
+    command fails.
+
+    We also support the use of /usr/bin/time to gather rusage information.
+
+        command:
+            The command to run.
+        verbose:
+            If true, we output information about the command that we run, and
+            its result.
+
+            If callable or something with a .write() method, information is
+            sent to <verbose> itself. Otherwise it is sent to <out> (without
+            applying <prefix>).
+        raise_errors:
+            If true, we raise an exception if the command fails, otherwise we
+            return the failing error code or zero.
+        out:
+            Python stream, fd, callable or Stream instance to which output is
+            sent.
+
+            If <out> is 'return', we buffer the output and return (e,
+            <output>). Note that if raise_errors is true, we only return if <e>
+            is zero.
+
+            If -1, output is sent to stdout and stderr.
+        prefix:
+            If not None, should be prefix string or callable used to prefix
+            all output. [This is for convenience to avoid the need to do
+            out=StreamPrefix(...).]
+        rusage:
+            If true, we run via /usr/bin/time and return rusage string
+            containing information on execution. <raise_errors> and
+            out='return' are ignored.
+        shell:
+            Passed to underlying subprocess.Popen() call.
+        encoding:
+            Sepecify the encoding used to translate the command's output
+            to characters. Defaults to utf-8.
+        errors:
+            How to handle encoding errors; see docs for codecs module
+            for details. Defaults to 'replace' so we never raise a
+            UnicodeDecodeError.
+        buffer_len:
+            The number of bytes we attempt to read at a time. If -1 we read
+            output one line at a time.
+
+    Returns:
+        If <rusage> is true, we return the rusage text.
+
+        Else if raise_errors is true:
+            If the command failed, we raise an exception.
+            Else if <out> is 'return' we return the text output from the command.
+            Else we return None
+
+        Else if <out> is 'return', we return (e, text) where <e> is the
+        command's exit code and <text> is the output from the command.
+
+        Else we return <e>, the command's exit code.
+    '''
+    if encoding is None:
+        if sys.version_info[0] == 2:
+            # python-2 doesn't seem to implement 'replace' properly.
+            encoding = None
+            errors = None
+        else:
+            encoding = 'utf-8'
+            errors = 'replace'
+
+    out_original = out
+    if out is None:
+        out = sys.stdout
+    elif out == 'return':
+        # Store the output ourselves so we can return it.
+        out = io.StringIO()
+    else:
+        out = make_stream( out)
+
+    if verbose:
+        if getattr( verbose, 'write', None):
+            pass
+        elif callable( verbose):
+            verbose = make_stream( verbose)
+        else:
+            verbose = out
+
+    if prefix:
+        out = StreamPrefix( out, prefix)
+
+    if verbose:
+        verbose.write( 'running: %s\n' % command)
+
+    if rusage:
+        command2 = ''
+        command2 += '/usr/bin/time -o ubt-out -f "D=%D E=%D F=%F I=%I K=%K M=%M O=%O P=%P R=%r S=%S U=%U W=%W X=%X Z=%Z c=%c e=%e k=%k p=%p r=%r s=%s t=%t w=%w x=%x C=%C"'
+        command2 += ' '
+        command2 += command
+        system_raw( command2, out, shell, encoding, errors, buffer_len=buffer_len)
+        with open('ubt-out') as f:
+            rusage_text = f.read()
+        #print 'have read rusage output: %r' % rusage_text
+        if rusage_text.startswith( 'Command '):
+            # Annoyingly, /usr/bin/time appears to write 'Command
+            # exited with ...' or 'Command terminated by ...' to the
+            # output file before the rusage info if command doesn't
+            # exit 0.
+            nl = rusage_text.find('\n')
+            rusage_text = rusage_text[ nl+1:]
+        return rusage_text
+    else:
+        e = system_raw( command, out, shell, encoding, errors, buffer_len=buffer_len)
+
+        if verbose:
+            verbose.write( '[returned e=%s]\n' % e)
+
+        if raise_errors:
+            if e:
+                raise Exception( 'command failed: %s' % command)
+            if out_original == 'return':
+                return out.getvalue()
+            return
+
+        if out_original == 'return':
+            return e, out.getvalue()
+        return e
+
+def get_gitfiles( directory, submodules=False):
+    '''
+    Returns list of all files known to git in <directory>; <directory> must be
+    somewhere within a git checkout.
+
+    Returned names are all relative to <directory>.
+
+    If .git directory, we also create <directory>/jtest-git-files. Otherwise we
+    assume a this file already exists.
+    '''
+    if os.path.isdir( '%s/.git' % directory):
+        command = 'cd ' + directory + ' && git ls-files'
+        if submodules:
+            command += ' --recurse-submodules'
+        command += ' > jtest-git-files'
+        system( command, verbose=sys.stdout)
+
+    with open( '%s/jtest-git-files' % directory, 'r') as f:
+        text = f.read()
+    ret = text.split( '\n')
+    return ret
+
+def get_git_id_raw( directory):
+    if not os.path.isdir( '%s/.git' % directory):
+        return
+    text = system(
+            f'cd {directory} && (PAGER= git show --pretty=oneline|head -n 1 && git diff)',
+            out='return',
+            )
+    return text
+
+def get_git_id( directory, allow_none=False):
+    '''
+    Returns text where first line is '<git-sha> <commit summary>' and remaining
+    lines contain output from 'git diff' in <directory>.
+
+    directory:
+        Root of git checkout.
+    allow_none:
+        If true, we return None if <directory> is not a git checkout and
+        jtest-git-id file does not exist.
+    '''
+    filename = f'{directory}/jtest-git-id'
+    text = get_git_id_raw( directory)
+    if text:
+        with open( filename, 'w') as f:
+            f.write( text)
+    elif os.path.isfile( filename):
+        with open( filename) as f:
+            text = f.read()
+    else:
+        if not allow_none:
+            raise Exception( f'Not in git checkout, and no file {filename}.')
+        text = None
+    return text
+
+class Args:
+    '''
+    Iterates over argv items. Does getopt-style splitting of args starting with
+    single '-' character.
+    '''
+    def __init__( self, argv):
+        self.argv = argv
+        self.pos = 0
+        self.pos_sub = None
+    def next( self):
+        while 1:
+            if self.pos >= len(self.argv):
+                raise StopIteration()
+            arg = self.argv[self.pos]
+            if (not self.pos_sub
+                    and arg.startswith('-')
+                    and not arg.startswith('--')
+                    ):
+                # Start splitting current arg.
+                self.pos_sub = 1
+            if self.pos_sub and self.pos_sub >= len(arg):
+                # End of '-' sub-arg.
+                self.pos += 1
+                self.pos_sub = None
+                continue
+            if self.pos_sub:
+                # Return '-' sub-arg.
+                ret = arg[self.pos_sub]
+                self.pos_sub += 1
+                return f'-{ret}'
+            # Return normal arg.
+            self.pos += 1
+            return arg
+
+def update_file( text, filename):
+    '''
+    Writes <text> to <filename>. Does nothing if contents of <filename> are
+    already <text>.
+    '''
+    try:
+        with open( filename) as f:
+            text0 = f.read()
+    except OSError:
+        text0 = None
+    if text == text0:
+        log( 'Unchanged: ' + filename)
+    else:
+        log( 'Updating:  ' + filename)
+        # Write to temp file and rename, to ensure we are atomic.
+        filename_temp = f'{filename}-jlib-temp'
+        with open( filename_temp, 'w') as f:
+            f.write( text)
+        os.rename( filename_temp, filename)
+
+
+def mtime( filename, default=0):
+    '''
+    Returns mtime of file, or <default> if error - e.g. doesn't exist.
+    '''
+    try:
+        return os.path.getmtime( filename)
+    except OSError:
+        return default
+
+def get_filenames( paths):
+    '''
+    Yields each file in <paths>, walking any directories.
+    '''
+    if isinstance( paths, str):
+        paths = (paths,)
+    for name in paths:
+        if os.path.isdir( name):
+            for dirpath, dirnames, filenames in os.walk( name):
+                for filename in filenames:
+                    path = os.path.join( dirpath, filename)
+                    yield path
+        else:
+            yield name
+
+def remove( path):
+    '''
+    Removes file or directory, without raising exception if it doesn't exist.
+
+    We assert-fail if the path still exists when we return, in case of
+    permission problems etc.
+    '''
+    try:
+        os.remove( path)
+    except Exception:
+        pass
+    shutil.rmtree( path, ignore_errors=1)
+    assert not os.path.exists( path)
+
+
+# Things for figuring out whether files need updating, using mtimes.
+#
+def newest( names):
+    '''
+    Returns mtime of newest file in <filenames>. Returns 0 if no file exists.
+    '''
+    assert isinstance( names, (list, tuple))
+    assert names
+    ret_t = 0
+    ret_name = None
+    for filename in get_filenames( names):
+        t = mtime( filename)
+        if t > ret_t:
+            ret_t = t
+            ret_name = filename
+    return ret_t, ret_name
+
+def oldest( names):
+    '''
+    Returns mtime of oldest file in <filenames> or 0 if no file exists.
+    '''
+    assert isinstance( names, (list, tuple))
+    assert names
+    ret_t = None
+    ret_name = None
+    for filename in get_filenames( names):
+        t = mtime( filename)
+        if ret_t is None or t < ret_t:
+            ret_t = t
+            ret_name = filename
+    if ret_t is None:
+        ret_t = 0
+    return ret_t, ret_name
+
+def update_needed( infiles, outfiles):
+    '''
+    If any file in <infiles> is newer than any file in <outfiles>, returns
+    string description. Otherwise returns None.
+    '''
+    in_tmax, in_tmax_name = newest( infiles)
+    out_tmin, out_tmin_name = oldest( outfiles)
+    if in_tmax > out_tmin:
+        text = f'{in_tmax_name} is newer than {out_tmin_name}'
+        return text
+
+def build(
+        infiles,
+        outfiles,
+        command,
+        force_rebuild=False,
+        out=None,
+        all_reasons=False,
+        verbose=True,
+        prefix=None,
+        ):
+    '''
+    Ensures that <outfiles> are up to date using enhanced makefile-like
+    determinism of dependencies.
+
+    Rebuilds <outfiles> by running <command> if we determine that any of them
+    are out of date.
+
+    infiles:
+        Names of files that are read by <command>. Can be a single filename. If
+        an item is a directory, we expand to all filenames in the directory's
+        tree.
+    outfiles:
+        Names of files that are written by <command>. Can also be a single
+        filename.
+    command:
+        Command to run.
+    force_rebuild:
+        If true, we always re-run the command.
+    out:
+        A callable, passed to jlib.system(). If None, we use jlib.log() with
+        our caller's stack record.
+    all_reasons:
+        If true we check all ways for a build being needed, even if we already
+        know a build is needed; this only affects the diagnostic that we
+        output.
+    verbose:
+        Passed to jlib.system().
+    prefix:
+        Passed to jlib.system().
+
+    We compare mtimes of <infiles> and <outfiles>, and we also detect changes
+    to the command itself.
+
+    If any of infiles are newer than any of outfiles, or <command> is
+    different to contents of commandfile '<outfile[0]>.cmd, then truncates
+    commandfile and runs <command>. If <command> succeeds we writes <command>
+    to commandfile.
+    '''
+    if isinstance( infiles, str):
+        infiles = (infiles,)
+    if isinstance( outfiles, str):
+        infiles = (outfiles,)
+
+    if not out:
+        out_frame_record = inspect.stack()[1]
+        out = lambda text: log( text, nv=0, caller=out_frame_record)
+
+    command_filename = f'{outfiles[0]}.cmd'
+
+    reasons = []
+
+    if not reasons or all_reasons:
+        if force_rebuild:
+            reasons.append( 'force_rebuild was specified')
+
+    if not reasons or all_reasons:
+        try:
+            with open( command_filename) as f:
+                command0 = f.read()
+        except Exception:
+            command0 = None
+        if command != command0:
+            if command0:
+                reasons.append( 'command has changed')
+            else:
+                reasons.append( 'no previous command')
+
+    if not reasons or all_reasons:
+        reason = update_needed( infiles, outfiles)
+        if reason:
+            reasons.append( reason)
+
+    if not reasons:
+        out( 'Already up to date: ' + ' '.join(outfiles))
+        return
+
+    if out:
+        out( 'Rebuilding because %s: %s' % (
+                ', and '.join( reasons),
+                ' '.join(outfiles),
+                ))
+
+    # Empty <command_filename) while we run the command so that if command
+    # fails but still creates target(s), then next time we will know target(s)
+    # are not up to date.
+    #
+    with open( command_filename, 'w') as f:
+        pass
+
+    system( command, out=out, verbose=verbose, prefix=prefix)
+
+    with open( command_filename, 'w') as f:
+        f.write( command)
+
+
+def link_l_flags( sos):
+    '''
+    Returns flags needed to link with items in <sos>. For each unique item we
+    use -L with parent directory, and -l with embedded name (without leading
+    'lib' or trailing '.co').
+    '''
+    dirs = set()
+    names = []
+    if isinstance( sos, str):
+        sos = (sos,)
+    for so in sos:
+        dir_ = os.path.dirname( so)
+        name = os.path.basename( so)
+        assert name.startswith( 'lib')
+        assert name.endswith ( '.so')
+        name = name[3:-3]
+        dirs.add( dir_)
+        names.append( name)
+    ret = ''
+    # Important to use sorted() here, otherwise ordering from set() is
+    # arbitrary causing occasional spurious rebuilds by jlib.build().
+    for dir_ in sorted(dirs):
+        ret += f' -L {dir_}'
+    for name in names:
+        ret += f' -l {name}'
+    return ret