Massive Python/Pyrex wrapper cleanup. We now use Pyxelator to generate

the C function and data structure wrappers for Pyrex.  We now require
Pyrex 0.9.5 to build the Python wrappers, as only >=0.9.5 has working
weakref support.  We actually need 0.9.5.1, but it's not yet released
(0.9.5 has a crash bug in enum wrapping that we trigger).

Pyxelator works a lot better with non-anonymous enums/structs, so libyasm
has been scrubbed for this.

Next step: full Yasm data structure inspection.

svn path=/trunk/yasm/; revision=1745

12 files changed, 158 insertions, 564 deletions

diff --git a/tools/python-yasm/Makefile.inc b/tools/python-yasm/Makefile.inc
index ed8d7df3..53c0c675 100644
--- a/tools/python-yasm/Makefile.inc
+++ b/tools/python-yasm/Makefile.inc
@@ -1,7 +1,6 @@
 # $Id$
 
 PYBINDING_DEPS  = tools/python-yasm/bytecode.pxi
-PYBINDING_DEPS += tools/python-yasm/coretype.pxi
 PYBINDING_DEPS += tools/python-yasm/errwarn.pxi
 PYBINDING_DEPS += tools/python-yasm/expr.pxi
 PYBINDING_DEPS += tools/python-yasm/floatnum.pxi
@@ -9,15 +8,43 @@ PYBINDING_DEPS += tools/python-yasm/intnum.pxi
 PYBINDING_DEPS += tools/python-yasm/symrec.pxi
 PYBINDING_DEPS += tools/python-yasm/value.pxi
 
+EXTRA_DIST += tools/python-yasm/pyxelator/cparse.py
+EXTRA_DIST += tools/python-yasm/pyxelator/genpyx.py
+EXTRA_DIST += tools/python-yasm/pyxelator/ir.py
+EXTRA_DIST += tools/python-yasm/pyxelator/lexer.py
+EXTRA_DIST += tools/python-yasm/pyxelator/node.py
+EXTRA_DIST += tools/python-yasm/pyxelator/parse_core.py
+EXTRA_DIST += tools/python-yasm/pyxelator/work_unit.py
+EXTRA_DIST += tools/python-yasm/pyxelator/wrap_yasm.py
 EXTRA_DIST += tools/python-yasm/setup.py
 EXTRA_DIST += tools/python-yasm/yasm.pyx
 EXTRA_DIST += $(PYBINDING_DEPS)
 
 if HAVE_PYTHON
 
-yasm_python.c: $(srcdir)/tools/python-yasm/yasm.pyx $(PYBINDING_DEPS)
+# Use Pyxelator to generate Pyrex function headers.
+_yasm.pxi: ${libyasm_a_SOURCES}
+	@rm -rf .tmp
+	@mkdir .tmp
+	$(PYTHON) $(srcdir)/tools/python-yasm/pyxelator/wrap_yasm.py \
+		"YASM_DIR=${srcdir}" "CPP=${CPP}" "CPPFLAGS=${CPPFLAGS}"
+	@rm -rf .tmp
+
+CLEANFILES += _yasm.pxi
+
+# Need to build a local copy of the main Pyrex input file to include _yasm.pxi
+# from the build directory.  Also need to fixup the other .pxi include paths.
+yasm.pyx: $(srcdir)/tools/python-yasm/yasm.pyx
+	cat $(srcdir)/tools/python-yasm/yasm.pyx \
+		| sed -e 's,^include "\([^_]\),include "${srcdir}/tools/python-yasm/\1,' \
+		> $@
+
+CLEANFILES += yasm.pyx
+
+# Actually run Pyrex
+yasm_python.c: yasm.pyx _yasm.pxi $(PYBINDING_DEPS)
 	$(PYTHON) -c "from Pyrex.Compiler.Main import main; main(command_line=1)" \
-		-o $@ `test -f tools/python-yasm/yasm.pyx || echo '$(srcdir)/'`tools/python-yasm/yasm.pyx
+		-o $@ yasm.pyx
 
 CLEANFILES += yasm_python.c
 
diff --git a/tools/python-yasm/bytecode.pxi b/tools/python-yasm/bytecode.pxi
index 86f38d8f..8a077b0d 100644
--- a/tools/python-yasm/bytecode.pxi
+++ b/tools/python-yasm/bytecode.pxi
@@ -23,132 +23,23 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-cdef extern from "libyasm/bytecode.h":
-    cdef struct yasm_effaddr
-
-    cdef struct yasm_effaddr_callback:
-        void (*destroy) (yasm_effaddr *ea)
-        void (*print_ "print") (yasm_effaddr *ea, FILE *f, int indent_level)
-
-    cdef struct yasm_effaddr:
-        yasm_effaddr_callback *callback
-        yasm_value disp
-        unsigned long segreg
-        unsigned int disp_len
-        unsigned int need_disp
-        unsigned int nosplit
-        unsigned int strong
-
-    cdef struct yasm_dataval
-    cdef struct yasm_datavalhead
-
-    cdef yasm_expr* yasm_ea_get_disp(yasm_effaddr *ea)
-    cdef void yasm_ea_set_len(yasm_effaddr *ea, unsigned int len)
-    cdef void yasm_ea_set_nosplit(yasm_effaddr *ea, unsigned int nosplit)
-    cdef void yasm_ea_set_strong(yasm_effaddr *ea, unsigned int strong)
-    cdef void yasm_ea_set_segreg(yasm_effaddr *ea, unsigned long segreg)
-    cdef void yasm_ea_destroy(yasm_effaddr *ea)
-    cdef void yasm_ea_print(yasm_effaddr *ea, FILE *f, int indent_level)
-
-    cdef void yasm_bc_set_multiple(yasm_bytecode *bc, yasm_expr *e)
-    cdef yasm_bytecode* yasm_bc_create_data(yasm_datavalhead *datahead,
-            unsigned int size, int append_zero, unsigned long line)
-    cdef yasm_bytecode* yasm_bc_create_leb128(yasm_datavalhead *datahead,
-            int sign, unsigned long line)
-    cdef yasm_bytecode* yasm_bc_create_reserve(yasm_expr *numitems,
-            unsigned int itemsize, unsigned long line)
-    cdef yasm_bytecode* yasm_bc_create_incbin(char *filename,
-            yasm_expr *start, yasm_expr *maxlen, yasm_linemap *linemap,
-            unsigned long line)
-    cdef yasm_bytecode* yasm_bc_create_align(yasm_expr *boundary,
-            yasm_expr *fill, yasm_expr *maxskip,
-            unsigned char **code_fill, unsigned long line)
-    cdef yasm_bytecode* yasm_bc_create_org(unsigned long start,
-            unsigned long line)
-    cdef yasm_bytecode* yasm_bc_create_insn(yasm_arch *arch,
-            unsigned long insn_data[4], int num_operands,
-            yasm_insn_operands *operands, unsigned long line)
-    cdef yasm_bytecode* yasm_bc_create_empty_insn(yasm_arch *arch,
-            unsigned long insn_data[4], int num_operands,
-            yasm_insn_operands *operands, unsigned long line)
-    cdef void yasm_bc_insn_add_prefix(yasm_bytecode *bc,
-            unsigned long prefix_data[4])
-    cdef void yasm_bc_insn_add_seg_prefix(yasm_bytecode *bc,
-            unsigned long segreg)
-    cdef yasm_section* yasm_bc_get_section(yasm_bytecode *bc)
-    cdef void yasm_bc__add_symrec(yasm_bytecode *bc, yasm_symrec *sym)
-    cdef void yasm_bc_destroy(yasm_bytecode *bc)
-    cdef void yasm_bc_print(yasm_bytecode *bc, FILE *f, int indent_level)
-    cdef void yasm_bc_finalize(yasm_bytecode *bc, yasm_bytecode *prev_bc)
-    cdef yasm_intnum *yasm_calc_bc_dist(yasm_bytecode *precbc1,
-            yasm_bytecode *precbc2)
-    ctypedef void (*yasm_bc_add_span_func) (void *add_span_data,
-            yasm_bytecode *bc, int id, yasm_value *value, long neg_thres,
-            long pos_thres)
-    cdef int yasm_bc_calc_len(yasm_bytecode *bc, yasm_bc_add_span_func add_span,
-                              void *add_span_data)
-    cdef int yasm_bc_expand(yasm_bytecode *bc, int span, long old_val,
-                            long new_val, long *neg_thres, long *pos_thres)
-    cdef unsigned char* yasm_bc_tobytes(yasm_bytecode *bc,
-            unsigned char *buf, unsigned long *bufsize, int *gap, void *d,
-            yasm_output_value_func output_value,
-            yasm_output_reloc_func output_reloc)
-    cdef int yasm_bc_get_multiple(yasm_bytecode *bc, unsigned long *multiple,
-                                  int calc_bc_dist)
-
-    cdef yasm_dataval* yasm_dv_create_expr(yasm_expr *expn)
-    cdef yasm_dataval* yasm_dv_create_string(char *contents, size_t len)
-
-    cdef void yasm_dvs_initialize(yasm_datavalhead *headp)
-    cdef void yasm_dvs_destroy(yasm_datavalhead *headp)
-    cdef yasm_dataval* yasm_dvs_append(yasm_datavalhead *headp,
-            yasm_dataval *dv)
-    cdef void yasm_dvs_print(yasm_datavalhead *headp, FILE *f,
-            int indent_level)
-
-cdef extern from "libyasm/bc-int.h":
-    cdef enum yasm_bc_special:
-        YASM_BC_SPECIAL_NONE
-        YASM_BC_SPECIAL_RESERVE
-        YASM_BC_SPECIAL_OFFSET
-
-    cdef struct yasm_bytecode_callback:
-        void (*destroy) (void *contents)
-        void (*c_print "print") (void *contents, FILE *f, int indent_level)
-        void (*finalize) (yasm_bytecode *bc, yasm_bytecode *prev_bc)
-        int (*calc_len) (yasm_bytecode *bc, yasm_bc_add_span_func add_span,
-		         void *add_span_data)
-        int (*expand) (yasm_bytecode *bc, int span, long old_val, long new_val,
-		       long *neg_thres, long *pos_thres)
-        int (*tobytes) (yasm_bytecode *bc, unsigned char **bufp, void *d,
-	                yasm_output_value_func output_value,
-		        yasm_output_reloc_func output_reloc)
-        yasm_bc_special special
-
-    cdef struct yasm_bytecode:
-        yasm_bytecode_callback *callback
-        yasm_section *section
-        yasm_expr *multiple
-        unsigned long len
-        unsigned long mult_int
-        unsigned long line
-        unsigned long offset
-        unsigned long bc_index
-        yasm_symrec **symrecs
-        void *contents
-
-    cdef yasm_bytecode *yasm_bc_create_common(yasm_bytecode_callback *callback,
-            void *contents, unsigned long line)
-
-    cdef void yasm_bc_transform(yasm_bytecode *bc,
-            yasm_bytecode_callback *callback, void *contents)
+cdef class Bytecode:
+    cdef yasm_bytecode *bc
 
-    cdef void yasm_bc_finalize_common(yasm_bytecode *bc, yasm_bytecode *prev_bc)
+    cdef object __weakref__     # make weak-referenceable
 
-    cdef yasm_bytecode *yasm_bc__next(yasm_bytecode *bc)
+    def __new__(self, bc):
+        self.bc = NULL
+        if PyCObject_Check(bc):
+            self.bc = <yasm_bytecode *>__get_voidp(bc, Bytecode)
+        else:
+            raise NotImplementedError
 
-cdef class Bytecode:
-    cdef yasm_bytecode *bc
+    def __dealloc__(self):
+        # Only free if we're not part of a section; if we're part of a section
+        # the section takes care of freeing the bytecodes.
+        if self.bc.section == NULL:
+            yasm_bc_destroy(self.bc)
 
     property len:
         def __get__(self): return self.bc.len
@@ -171,6 +62,8 @@ cdef class Bytecode:
         def __get__(self):
             cdef yasm_symrec *sym
             cdef int i
+            if self.bc.symrecs == NULL:
+                return []
             s = []
             i = 0
             sym = self.bc.symrecs[i]
@@ -179,3 +72,36 @@ cdef class Bytecode:
                 i = i+1
                 sym = self.bc.symrecs[i]
             return s
+
+#
+# Keep Bytecode reference paired with bc using weak references.
+# This is broken in Pyrex 0.9.4.1; Pyrex 0.9.5 has a working version.
+#
+
+from weakref import WeakValueDictionary as __weakvaldict
+__bytecode_map = __weakvaldict()
+#__bytecode_map = {}
+
+cdef object __make_bytecode(yasm_bytecode *bc):
+    __error_check()
+    vptr = PyCObject_FromVoidPtr(bc, NULL)
+    data = __bytecode_map.get(vptr, None)
+    if data:
+        return data
+    bcobj = Bytecode(__pass_voidp(bc, Bytecode))
+    __bytecode_map[vptr] = bcobj
+    return bcobj
+
+# Org bytecode
+def __org__new__(cls, start, line=0):
+    cdef yasm_bytecode *bc
+    bc = yasm_bc_create_org(start, line)
+    obj = Bytecode.__new__(cls, __pass_voidp(bc, Bytecode))
+    __bytecode_map[PyCObject_FromVoidPtr(bc, NULL)] = obj
+    return obj
+__org__new__ = staticmethod(__org__new__)
+class Org(Bytecode):
+    __new__ = __org__new__
+
+
+#cdef class Section:
diff --git a/tools/python-yasm/coretype.pxi b/tools/python-yasm/coretype.pxi
deleted file mode 100644
index 6146b908..00000000
--- a/tools/python-yasm/coretype.pxi
+++ /dev/null
@@ -1,122 +0,0 @@
-# Python bindings for Yasm: Pyrex input file for coretype.h
-#
-#  Copyright (C) 2006  Michael Urman, Peter Johnson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND OTHER CONTRIBUTORS ``AS IS''
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-cdef extern struct FILE:
-    int fileno
-
-ctypedef unsigned long size_t
-
-cdef extern struct va_list:
-    int arglist
-
-cdef extern from "libyasm/coretype.h":
-    cdef struct yasm_arch
-    cdef struct yasm_preproc
-    cdef struct yasm_parser
-    cdef struct yasm_optimizer
-    cdef struct yasm_objfmt
-    cdef struct yasm_dbgfmt
-    cdef struct yasm_listfmt
-
-    cdef struct yasm_assoc_data_callback:
-        void (*destroy) (void *data)
-        void (*print_ "print") (void *data, FILE *f, int indent_level)
-
-    cdef struct yasm_errwarns
-
-    cdef struct yasm_bytecode
-    cdef struct yasm_object
-    cdef struct yasm_section
-    cdef struct yasm_symtab
-    cdef struct yasm_symrec
-    cdef struct yasm_expr
-    cdef struct yasm_intnum
-    cdef struct yasm_floatnum
-
-    cdef struct yasm_value:
-        yasm_expr *abs
-        yasm_symrec *rel
-        yasm_symrec *wrt
-        unsigned int seg_of
-        unsigned int rshift
-        unsigned int curpos_rel
-        unsigned int ip_rel
-        unsigned int section_rel
-        unsigned int size
-
-    cdef struct yasm_linemap
-    cdef struct yasm_valparam
-    cdef struct yasm_valparamhead
-    cdef struct yasm_insn_operands
-
-    ctypedef enum yasm_expr_op:
-        YASM_EXPR_IDENT
-        YASM_EXPR_ADD
-        YASM_EXPR_SUB
-        YASM_EXPR_MUL
-        YASM_EXPR_DIV
-        YASM_EXPR_SIGNDIV
-        YASM_EXPR_MOD
-        YASM_EXPR_SIGNMOD
-        YASM_EXPR_NEG
-        YASM_EXPR_NOT
-        YASM_EXPR_OR
-        YASM_EXPR_AND
-        YASM_EXPR_XOR
-        YASM_EXPR_XNOR
-        YASM_EXPR_NOR
-        YASM_EXPR_SHL
-        YASM_EXPR_SHR
-        YASM_EXPR_LOR
-        YASM_EXPR_LAND
-        YASM_EXPR_LNOT
-        YASM_EXPR_LXOR
-        YASM_EXPR_LXNOR
-        YASM_EXPR_LNOR
-        YASM_EXPR_LT
-        YASM_EXPR_GT
-        YASM_EXPR_EQ
-        YASM_EXPR_LE
-        YASM_EXPR_GE
-        YASM_EXPR_NE
-        YASM_EXPR_NONNUM
-        YASM_EXPR_SEG
-        YASM_EXPR_WRT
-        YASM_EXPR_SEGOFF
-
-    ctypedef enum yasm_sym_vis:
-        YASM_SYM_LOCAL
-        YASM_SYM_GLOBAL
-        YASM_SYM_COMMON
-        YASM_SYM_EXTERN
-        YASM_SYM_DLOCAL
-
-    ctypedef int*(*yasm_output_value_func)(yasm_value *value, unsigned char
-            *buf, size_t destsize, unsigned long offset, yasm_bytecode *bc,
-            int warn, void *d)
-    ctypedef int(*yasm_output_reloc_func)(yasm_symrec *sym,
-            yasm_bytecode *bc, unsigned char *buf, size_t destsize,
-            size_t valsize, int warn, void *d)
-
diff --git a/tools/python-yasm/errwarn.pxi b/tools/python-yasm/errwarn.pxi
index 390391b2..9568cc07 100644
--- a/tools/python-yasm/errwarn.pxi
+++ b/tools/python-yasm/errwarn.pxi
@@ -23,112 +23,40 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-cdef extern from "libyasm/errwarn.h":
-    ctypedef enum yasm_warn_class:
-        YASM_WARN_NONE
-        YASM_WARN_GENERAL
-        YASM_WARN_UNREC_CHAR
-        YASM_WARN_PREPROC
-        YASM_WARN_ORPHAN_LABEL
-        YASM_WARN_UNINIT_CONTENTS
-
-    ctypedef enum yasm_error_class:
-        YASM_ERROR_NONE
-        YASM_ERROR_GENERAL
-        YASM_ERROR_ARITHMETIC
-        YASM_ERROR_OVERFLOW
-        YASM_ERROR_FLOATING_POINT
-        YASM_ERROR_ZERO_DIVISION
-        YASM_ERROR_ASSERTION
-        YASM_ERROR_VALUE
-        YASM_ERROR_NOT_ABSOLUTE
-        YASM_ERROR_TOO_COMPLEX
-        YASM_ERROR_NOT_CONSTANT
-        YASM_ERROR_IO
-        YASM_ERROR_NOT_IMPLEMENTED
-        YASM_ERROR_TYPE
-        YASM_ERROR_SYNTAX
-        YASM_ERROR_PARSE
-
-    void yasm_errwarn_initialize()
-    void yasm_errwarn_cleanup()
-    extern void (*yasm_internal_error_) (char *file, unsigned int line,
-                                         char *message)
-    void yasm_internal_error(char *message)
-    extern void (*yasm_fatal) (char *message, va_list va)
-    void yasm__fatal(char *message, ...)
-
-    void yasm_error_clear()
-    yasm_error_class yasm_error_occurred()
-    int yasm_error_matches(yasm_error_class eclass)
-
-    void yasm_error_set_va(yasm_error_class eclass, char *format, va_list va)
-    void yasm_error_set(yasm_error_class eclass, char *format, ...)
-    void yasm_error_set_xref_va(unsigned long xrefline, char *format,
-                                va_list va)
-    void yasm_error_set_xref(unsigned long xrefline, char *format, ...)
-    void yasm_error_fetch(yasm_error_class *eclass, char **str,
-                          unsigned long *xrefline, char **xrefstr)
-
-    void yasm_warn_clear()
-    void yasm_warn_set_va(yasm_warn_class wclass, char *format, va_list va)
-    void yasm_warn_set(yasm_warn_class wclass, char *format, ...)
-    void yasm_warn_fetch(yasm_warn_class *wclass, char **str)
-
-    void yasm_warn_enable(yasm_warn_class wclass)
-    void yasm_warn_disable(yasm_warn_class wclass)
-
-    void yasm_warn_disable_all()
-
-    yasm_errwarns *yasm_errwarns_create()
-    void yasm_errwarns_destroy(yasm_errwarns *errwarns)
-    void yasm_errwarn_propagate(yasm_errwarns *errwarns, unsigned long line)
-    unsigned int yasm_errwarns_num_errors(yasm_errwarns *errwarns,
-                                          int warning_as_error)
-
-    ctypedef void (*yasm_print_error_func) (char *fn, unsigned long line,
-                                            char *msg, unsigned long xrefline,
-                                            char *xrefmsg)
-    ctypedef void (*yasm_print_warning_func) (char *fn, unsigned long line,
-                                              char *msg)
-    void yasm_errwarns_output_all(yasm_errwarns *errwarns, yasm_linemap *lm,
-                                  int warning_as_error,
-                                  yasm_print_error_func print_error,
-                                  yasm_print_warning_func print_warning)
-
-    char *yasm__conv_unprint(int ch)
-
-    extern char * (*yasm_gettext_hook) (char *msgid)
-
 class YasmError(Exception): pass
 
-__errormap = [
-    # Order matters here. Go from most to least specific within a class
-    (YASM_ERROR_ZERO_DIVISION, ZeroDivisionError),
-    # Enable these once there are tests that need them.
-    #(YASM_ERROR_OVERFLOW, OverflowError),
-    #(YASM_ERROR_FLOATING_POINT, FloatingPointError),
-    #(YASM_ERROR_ARITHMETIC, ArithmeticError),
-    #(YASM_ERROR_ASSERTION, AssertionError),
-    #(YASM_ERROR_VALUE, ValueError), # include notabs, notconst, toocomplex
-    #(YASM_ERROR_IO, IOError),
-    #(YASM_ERROR_NOT_IMPLEMENTED, NotImplementedError),
-    #(YASM_ERROR_TYPE, TypeError),
-    #(YASM_ERROR_SYNTAX, SyntaxError), #include parse
-]
-
-cdef void __error_check() except *:
+cdef int __error_check() except 1:
     cdef yasm_error_class errclass
     cdef unsigned long xrefline
     cdef char *errstr, *xrefstr
 
     # short path for the common case
-    if not yasm_error_occurred(): return
+    if not <int>yasm_error_occurred():
+        return 0
 
     # look up our preferred python error, fall back to YasmError
-    for error_class, exception in __errormap:
-        if yasm_error_matches(error_class):
-            break
+    # Order matters here. Go from most to least specific within a class
+    if yasm_error_matches(YASM_ERROR_ZERO_DIVISION):
+        exception = ZeroDivisionError
+    # Enable these once there are tests that need them.
+    #elif yasm_error_matches(YASM_ERROR_OVERFLOW):
+    #   exception = OverflowError
+    #elif yasm_error_matches(YASM_ERROR_FLOATING_POINT):
+    #   exception = FloatingPointError
+    #elif yasm_error_matches(YASM_ERROR_ARITHMETIC):
+    #   exception = ArithmeticError
+    #elif yasm_error_matches(YASM_ERROR_ASSERTION):
+    #   exception = AssertionError
+    #elif yasm_error_matches(YASM_ERROR_VALUE):
+    #   exception = ValueError # include notabs, notconst, toocomplex
+    #elif yasm_error_matches(YASM_ERROR_IO):
+    #   exception = IOError
+    #elif yasm_error_matches(YASM_ERROR_NOT_IMPLEMENTED):
+    #   exception = NotImplementedError
+    #elif yasm_error_matches(YASM_ERROR_TYPE):
+    #   exception = TypeError
+    #elif yasm_error_matches(YASM_ERROR_SYNTAX):
+    #   exception = SyntaxError #include parse
     else:
         exception = YasmError
 
@@ -142,3 +70,4 @@ cdef void __error_check() except *:
 
     if xrefstr: free(xrefstr)
     free(errstr)
+    return 1
diff --git a/tools/python-yasm/expr.pxi b/tools/python-yasm/expr.pxi
index 34a62f96..dea75d78 100644
--- a/tools/python-yasm/expr.pxi
+++ b/tools/python-yasm/expr.pxi
@@ -23,74 +23,9 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-cdef extern from "libyasm/expr.h":
-    cdef struct yasm_expr__item
-
-    cdef yasm_expr *yasm_expr_create(yasm_expr_op op, yasm_expr__item *a,
-            yasm_expr__item *b, unsigned long line)
-    cdef yasm_expr__item *yasm_expr_sym(yasm_symrec *sym)
-    cdef yasm_expr__item *yasm_expr_expr(yasm_expr *e)
-    cdef yasm_expr__item *yasm_expr_int(yasm_intnum *intn)
-    cdef yasm_expr__item *yasm_expr_float(yasm_floatnum *flt)
-    cdef yasm_expr__item *yasm_expr_reg(unsigned long reg)
-    cdef yasm_expr *yasm_expr_create_tree(yasm_expr *l, yasm_expr_op op,
-            yasm_expr *r, unsigned long line)
-    cdef yasm_expr *yasm_expr_create_branch(yasm_expr_op op,
-            yasm_expr *r, unsigned long line)
-    cdef yasm_expr *yasm_expr_create_ident(yasm_expr *r, unsigned long line)
-    cdef yasm_expr *yasm_expr_copy(yasm_expr *e)
-    cdef void yasm_expr_destroy(yasm_expr *e)
-    cdef int yasm_expr_is_op(yasm_expr *e, yasm_expr_op op)
-    ctypedef yasm_expr * (*yasm_expr_xform_func) (yasm_expr *e, void *d)
-
-    cdef yasm_expr *yasm_expr__level_tree(yasm_expr *e, int fold_const,
-            int simplify_ident, int simplify_reg_mul, int calc_bc_dist,
-            yasm_expr_xform_func expr_xform_extra,
-            void *expr_xform_extra_data)
+cdef extern from *:
+    # Defined as a macro, so not automatically brought in by pyxelator
     cdef yasm_expr *yasm_expr_simplify(yasm_expr *e, int calc_bc_dist)
-    cdef yasm_expr *yasm_expr_extract_segoff(yasm_expr **ep)
-    cdef yasm_expr *yasm_expr_extract_wrt(yasm_expr **ep)
-    cdef yasm_intnum *yasm_expr_get_intnum(yasm_expr **ep, int calc_bc_dist)
-    cdef yasm_symrec *yasm_expr_get_symrec(yasm_expr **ep, int simplify)
-    cdef unsigned long *yasm_expr_get_reg(yasm_expr **ep, int simplify)
-    cdef void yasm_expr_print(yasm_expr *e, FILE *f)
-
-cdef extern from "libyasm/expr-int.h":
-    cdef enum yasm_expr__type:
-        YASM_EXPR_NONE
-        YASM_EXPR_REG
-        YASM_EXPR_INT
-        YASM_EXPR_FLOAT
-        YASM_EXPR_SYM
-        YASM_EXPR_EXPR
-
-    cdef union yasm_expr__type_data:
-        yasm_symrec *sym
-        yasm_expr *expn
-        yasm_intnum *intn
-        yasm_floatnum *flt
-        unsigned long reg
-
-    cdef struct yasm_expr__item:
-        yasm_expr__type type
-        yasm_expr__type_data data
-
-    cdef struct yasm_expr:
-        yasm_expr_op op
-        unsigned long line
-        int numterms
-        yasm_expr__item terms[2]
-
-    cdef int yasm_expr__traverse_leaves_in_const(yasm_expr *e,
-            void *d, int (*func) (yasm_expr__item *ei, void *d))
-    cdef int yasm_expr__traverse_leaves_in(yasm_expr *e, void *d,
-            int (*func) (yasm_expr__item *ei, void *d))
-
-    cdef void yasm_expr__order_terms(yasm_expr *e)
-
-    cdef yasm_expr *yasm_expr__copy_except(yasm_expr *e, int excpt)
-
-    cdef int yasm_expr__contains(yasm_expr *e, yasm_expr__type t)
 
 import operator
 __op = {}
diff --git a/tools/python-yasm/floatnum.pxi b/tools/python-yasm/floatnum.pxi
index 7e761985..3939056f 100644
--- a/tools/python-yasm/floatnum.pxi
+++ b/tools/python-yasm/floatnum.pxi
@@ -23,21 +23,6 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-cdef extern from "libyasm/floatnum.h":
-    cdef void yasm_floatnum_initialize()
-    cdef void yasm_floatnum_cleanup()
-    cdef yasm_floatnum* yasm_floatnum_create(char *str)
-    cdef yasm_floatnum* yasm_floatnum_copy(yasm_floatnum *flt)
-    cdef void yasm_floatnum_destroy(yasm_floatnum *flt)
-    cdef void yasm_floatnum_calc(yasm_floatnum *acc, yasm_expr_op op,
-            yasm_floatnum *operand)
-    cdef int yasm_floatnum_get_int(yasm_floatnum *flt, size_t *ret_val)
-    cdef int yasm_floatnum_get_sized(yasm_floatnum *flt, unsigned char *ptr,
-            size_t destsize, size_t valsize, size_t shift, int
-            bigendian, int warn)
-    cdef int yasm_floatnum_check_size(yasm_floatnum *flt, size_t size)
-    cdef void yasm_floatnum_print(yasm_floatnum *flt, FILE *f)
-
 cdef class FloatNum:
     cdef yasm_floatnum *flt
     def __new__(self, value):
diff --git a/tools/python-yasm/intnum.pxi b/tools/python-yasm/intnum.pxi
index 7f77af95..f99769e6 100644
--- a/tools/python-yasm/intnum.pxi
+++ b/tools/python-yasm/intnum.pxi
@@ -23,46 +23,6 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-cdef extern from "libyasm/intnum.h":
-    cdef void yasm_intnum_initialize()
-    cdef void yasm_intnum_cleanup()
-    cdef yasm_intnum *yasm_intnum_create_dec(char *str)
-    cdef yasm_intnum *yasm_intnum_create_bin(char *str)
-    cdef yasm_intnum *yasm_intnum_create_oct(char *str)
-    cdef yasm_intnum *yasm_intnum_create_hex(char *str)
-    cdef yasm_intnum *yasm_intnum_create_charconst_nasm(char *str)
-    cdef yasm_intnum *yasm_intnum_create_uint(unsigned long i)
-    cdef yasm_intnum *yasm_intnum_create_int(long i)
-    cdef yasm_intnum *yasm_intnum_create_leb128(unsigned char *ptr,
-            int sign, unsigned long *size)
-    cdef yasm_intnum *yasm_intnum_create_sized(unsigned char *ptr, int sign,
-            size_t srcsize, int bigendian)
-    cdef yasm_intnum *yasm_intnum_copy(yasm_intnum *intn)
-    cdef void yasm_intnum_destroy(yasm_intnum *intn)
-    cdef void yasm_intnum_calc(yasm_intnum *acc, yasm_expr_op op,
-            yasm_intnum *operand)
-    cdef void yasm_intnum_zero(yasm_intnum *intn)
-    cdef void yasm_intnum_set_uint(yasm_intnum *intn, unsigned long val)
-    cdef int yasm_intnum_is_zero(yasm_intnum *acc)
-    cdef int yasm_intnum_is_pos1(yasm_intnum *acc)
-    cdef int yasm_intnum_is_neg1(yasm_intnum *acc)
-    cdef int yasm_intnum_sign(yasm_intnum *acc)
-    cdef unsigned long yasm_intnum_get_uint(yasm_intnum *intn)
-    cdef long yasm_intnum_get_int(yasm_intnum *intn)
-    cdef void yasm_intnum_get_sized(yasm_intnum *intn, unsigned char *ptr,
-	    size_t destsize, size_t valsize, int shift, int bigendian, int warn)
-    cdef int yasm_intnum_check_size(yasm_intnum *intn, size_t size,
-            size_t rshift, int rangetype)
-    cdef unsigned long yasm_intnum_get_leb128(yasm_intnum *intn,
-            unsigned char *ptr, int sign)
-    cdef unsigned long yasm_intnum_size_leb128(yasm_intnum *intn,
-            int sign)
-    cdef unsigned long yasm_get_sleb128(long v, unsigned char *ptr)
-    cdef unsigned long yasm_size_sleb128(long v)
-    cdef unsigned long yasm_get_uleb128(unsigned long v, unsigned char *ptr)
-    cdef unsigned long yasm_size_uleb128(unsigned long v)
-    cdef void yasm_intnum_print(yasm_intnum *intn, FILE *f)
-
 cdef class IntNum
 
 cdef object __intnum_op_ex(object x, yasm_expr_op op, object y):
diff --git a/tools/python-yasm/pyxelator/wrap_yasm.py b/tools/python-yasm/pyxelator/wrap_yasm.py
index deb133fd..45ee6237 100755
--- a/tools/python-yasm/pyxelator/wrap_yasm.py
+++ b/tools/python-yasm/pyxelator/wrap_yasm.py
@@ -17,16 +17,15 @@ from work_unit import WorkUnit, get_syms
 import ir
 
 
-def mk_tao(CPPFLAGS = "", CPP = "gcc -E", modname = '_yasm', oname = None, **options):
+def mk_tao(CPPFLAGS = "", CPP = "gcc -E", modname = '_yasm', oname = None, YASM_DIR = ".", **options):
     if oname is None:
         oname = modname+'.pyx'
-    YASM_DIR = "../../.."
     CPPFLAGS += " -I"+YASM_DIR
     CPPFLAGS += " -DYASM_PYXELATOR"
     CPPFLAGS += " -DYASM_LIB_INTERNAL"
     CPPFLAGS += " -DYASM_BC_INTERNAL"
     CPPFLAGS += " -DYASM_EXPR_INTERNAL"
-    files = [ 'libyasm.h', 'libyasm/assocdat.h', 'libyasm/section-int.h', 'libyasm/symrec-int.h' ]
+    files = [ 'libyasm.h', 'libyasm/assocdat.h' ]
 
     syms = get_syms( ['yasm'], [YASM_DIR] )
     def cb(trans_unit, node, *args):
@@ -46,7 +45,7 @@ def main():
     options = {}
     for i,arg in enumerate(sys.argv[1:]):
         if arg.count('='):
-            key,val = arg.split('=')
+            key,val = arg.split('=', 1)
             options[key]=val
     mk_tao(**options)
 
diff --git a/tools/python-yasm/setup.py b/tools/python-yasm/setup.py
index ffc0e0c0..60e2364d 100644
--- a/tools/python-yasm/setup.py
+++ b/tools/python-yasm/setup.py
@@ -39,6 +39,9 @@ def ParseCPPFlags(flags):
     """parse the CPPFlags macro"""
     incl_dir = [x[2:] for x in flags.split() if x.startswith("-I")]
     cppflags = [x for x in flags.split() if not x.startswith("-I")]
+    cppflags.append("-DYASM_LIB_INTERNAL")
+    cppflags.append("-DYASM_BC_INTERNAL")
+    cppflags.append("-DYASM_EXPR_INTERNAL")
     return (incl_dir, cppflags)
 
 def ParseSources(src, srcdir):
diff --git a/tools/python-yasm/symrec.pxi b/tools/python-yasm/symrec.pxi
index 60f7a4d7..d91fe7f5 100644
--- a/tools/python-yasm/symrec.pxi
+++ b/tools/python-yasm/symrec.pxi
@@ -23,53 +23,6 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-cdef extern from "libyasm/symrec.h":
-    cdef yasm_symtab* yasm_symtab_create()
-    cdef void yasm_symtab_destroy(yasm_symtab *symtab)
-    cdef yasm_symrec* yasm_symtab_use(yasm_symtab *symtab, char *name,
-            unsigned long line)
-    cdef yasm_symrec* yasm_symtab_get(yasm_symtab *symtab, char *name)
-    cdef yasm_symrec* yasm_symtab_define_equ(yasm_symtab *symtab,
-            char *name, yasm_expr *e, unsigned long line)
-    cdef yasm_symrec* yasm_symtab_define_label(yasm_symtab *symtab,
-            char *name, yasm_bytecode *precbc, int in_table, unsigned long line)
-    cdef yasm_symrec* yasm_symtab_define_curpos(yasm_symtab *symtab, char *name,
-            yasm_bytecode *precbc, unsigned long line)
-    cdef yasm_symrec* yasm_symtab_define_special(yasm_symtab *symtab,
-            char *name, yasm_sym_vis vis)
-    cdef yasm_symrec* yasm_symtab_declare(yasm_symtab *symtab,
-            char *name, yasm_sym_vis vis, unsigned long line)
-    cdef void yasm_symrec_declare(yasm_symrec *symrec, yasm_sym_vis vis,
-            unsigned long line)
-
-    ctypedef int (*yasm_symtab_traverse_callback)(yasm_symrec *sym, void *d)
-    cdef int yasm_symtab_traverse(yasm_symtab *symtab, void *d,
-            yasm_symtab_traverse_callback func)
-
-    ctypedef struct yasm_symtab_iter
-    cdef yasm_symtab_iter *yasm_symtab_first(yasm_symtab *symtab)
-    cdef yasm_symtab_iter *yasm_symtab_next(yasm_symtab_iter *prev)
-    cdef yasm_symrec *yasm_symtab_iter_value(yasm_symtab_iter *cur)
-
-    cdef void yasm_symtab_parser_finalize(yasm_symtab *symtab,
-            int undef_extern, yasm_objfmt *objfmt)
-    cdef void yasm_symtab_print(yasm_symtab *symtab, FILE *f, int indent_level)
-    cdef char* yasm_symrec_get_name(yasm_symrec *sym)
-    cdef yasm_sym_vis yasm_symrec_get_visibility(yasm_symrec *sym)
-    cdef yasm_expr* yasm_symrec_get_equ(yasm_symrec *sym)
-
-    ctypedef yasm_bytecode *yasm_symrec_get_label_bytecodep
-
-    cdef int yasm_symrec_get_label(yasm_symrec *sym,
-            yasm_symrec_get_label_bytecodep *precbc)
-    cdef int yasm_symrec_is_special(yasm_symrec *sym)
-    cdef int yasm_symrec_is_curpos(yasm_symrec *sym)
-    cdef void* yasm_symrec_get_data(yasm_symrec *sym,
-            yasm_assoc_data_callback *callback)
-    cdef void yasm_symrec_add_data(yasm_symrec *sym,
-            yasm_assoc_data_callback *callback, void *data)
-    cdef void yasm_symrec_print(yasm_symrec *sym, FILE *f, int indent_level)
-
 cdef class Symbol:
     cdef yasm_symrec *sym
 
@@ -85,15 +38,30 @@ cdef class Symbol:
     property name:
         def __get__(self): return yasm_symrec_get_name(self.sym)
 
+    property status:
+        def __get__(self):
+            cdef yasm_sym_status status
+            s = set()
+            status = yasm_symrec_get_status(self.sym)
+            if <int>status & <int>SYM_USED: s.add('used')
+            if <int>status & <int>SYM_DEFINED: s.add('defined')
+            if <int>status & <int>SYM_VALUED: s.add('valued')
+            return s
+
+    property in_table:
+        def __get__(self):
+            return bool(<int>yasm_symrec_get_status(self.sym) &
+                        <int>SYM_NOTINTABLE)
+
     property visibility:
         def __get__(self):
             cdef yasm_sym_vis vis
             s = set()
             vis = yasm_symrec_get_visibility(self.sym)
-            if vis & YASM_SYM_GLOBAL: s.add('global')
-            if vis & YASM_SYM_COMMON: s.add('common')
-            if vis & YASM_SYM_EXTERN: s.add('extern')
-            if vis & YASM_SYM_DLOCAL: s.add('dlocal')
+            if <int>vis & <int>YASM_SYM_GLOBAL: s.add('global')
+            if <int>vis & <int>YASM_SYM_COMMON: s.add('common')
+            if <int>vis & <int>YASM_SYM_EXTERN: s.add('extern')
+            if <int>vis & <int>YASM_SYM_DLOCAL: s.add('dlocal')
             return s
 
     property equ:
@@ -128,15 +96,6 @@ cdef class Symbol:
 # Use associated data mechanism to keep Symbol reference paired with symrec.
 #
 
-cdef class __assoc_data_callback:
-    cdef yasm_assoc_data_callback *cb
-    def __new__(self, destroy, print_):
-        self.cb = <yasm_assoc_data_callback *>malloc(sizeof(yasm_assoc_data_callback))
-        self.cb.destroy = <void (*) (void *)>PyCObject_AsVoidPtr(destroy)
-        self.cb.print_ = <void (*) (void *, FILE *, int)>PyCObject_AsVoidPtr(print_)
-    def __dealloc__(self):
-        free(self.cb)
-
 cdef void __python_symrec_cb_destroy(void *data):
     Py_DECREF(<object>data)
 cdef void __python_symrec_cb_print(void *data, FILE *f, int indent_level):
@@ -218,7 +177,7 @@ cdef class SymbolTableItemIterator:
         self.iter = yasm_symtab_next(self.iter)
         return rv
 
-cdef yasm_sym_vis __parse_vis(vis) except -1:
+cdef int __parse_vis(vis) except -1:
     if not vis or vis == 'local': return YASM_SYM_LOCAL
     if vis == 'global': return YASM_SYM_GLOBAL
     if vis == 'common': return YASM_SYM_COMMON
@@ -253,12 +212,14 @@ cdef class SymbolTable:
                 (<Bytecode>precbc).bc, in_table, line))
 
     def define_special(self, name, vis):
-        return __make_symbol(yasm_symtab_define_special(self.symtab, name,
-                                                        __parse_vis(vis)))
+        return __make_symbol(
+            yasm_symtab_define_special(self.symtab, name,
+                                       <yasm_sym_vis>__parse_vis(vis)))
 
     def declare(self, name, vis, line):
-        return __make_symbol(yasm_symtab_declare(self.symtab, name,
-                                                 __parse_vis(vis), line))
+        return __make_symbol(
+            yasm_symtab_declare(self.symtab, name,
+                                <yasm_sym_vis>__parse_vis(vis), line))
 
     #
     # Methods to make SymbolTable behave like a dictionary of Symbols.
diff --git a/tools/python-yasm/value.pxi b/tools/python-yasm/value.pxi
index 98ba14ea..f2328dd4 100644
--- a/tools/python-yasm/value.pxi
+++ b/tools/python-yasm/value.pxi
@@ -23,21 +23,6 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-cdef extern from "libyasm/value.h":
-    cdef void yasm_value_initialize(yasm_value *value, yasm_expr *e,
-                                    unsigned int size)
-    cdef void yasm_value_init_sym(yasm_value *value, yasm_symrec *sym,
-                                  unsigned int size)
-    cdef void yasm_value_delete(yasm_value *value)
-    cdef int yasm_value_finalize(yasm_value *value, yasm_bytecode *precbc)
-    cdef int yasm_value_finalize_expr(yasm_value *value, yasm_expr *e,
-                                      yasm_bytecode *precbc, unsigned int size)
-    cdef yasm_intnum *yasm_value_get_intnum(yasm_value *value,
-                                            yasm_bytecode *bc)
-    cdef int yasm_value_output_basic(yasm_value *value, unsigned char *buf,
-            size_t destsize, yasm_bytecode *bc, int warn, yasm_arch *arch)
-    cdef void yasm_value_print(yasm_value *value, FILE *f, int indent_level)
-
 cdef class Value:
     cdef yasm_value value
     def __new__(self, value=None, size=None):
diff --git a/tools/python-yasm/yasm.pyx b/tools/python-yasm/yasm.pyx
index f0114d2e..f9c6ee4d 100644
--- a/tools/python-yasm/yasm.pyx
+++ b/tools/python-yasm/yasm.pyx
@@ -60,6 +60,12 @@ cdef extern from "Python.h":
     cdef void PyErr_SetString(object type, char *message)
     cdef object PyErr_Format(object type, char *format, ...)
 
+cdef extern from "stdlib.h":
+    cdef void *malloc(int n)
+    cdef void free(void *p)
+
+include "_yasm.pxi"
+
 cdef object __pass_voidp(void *obj, object forclass):
     return PyCObject_FromVoidPtrAndDesc(obj, <void *>forclass, NULL)
 
@@ -85,20 +91,25 @@ cdef void *__get_voidp(object obj, object forclass) except NULL:
 
     return PyCObject_AsVoidPtr(obj)
 
-cdef extern from "stdlib.h":
-    cdef void *malloc(int n)
-    cdef void free(void *p)
+#
+# Link to associated data mechanism to keep Python references paired with
+# yasm objects.
+#
+cdef class __assoc_data_callback:
+    cdef yasm_assoc_data_callback *cb
+    def __new__(self, destroy, print_):
+        self.cb = <yasm_assoc_data_callback *>malloc(sizeof(yasm_assoc_data_callback))
+        self.cb.destroy = <void (*) (void *)>PyCObject_AsVoidPtr(destroy)
+        #self.cb.print_ = <void (*) (void *, FILE *, int)>PyCObject_AsVoidPtr(print_)
+    def __dealloc__(self):
+        free(self.cb)
 
-cdef extern from "libyasm/compat-queue.h":
-    pass
 
 cdef class Register:
     cdef unsigned long reg
     def __new__(self, reg):
         self.reg = reg
 
-include "coretype.pxi"
-
 include "errwarn.pxi"
 include "intnum.pxi"
 include "floatnum.pxi"
@@ -108,12 +119,7 @@ include "value.pxi"
 
 include "bytecode.pxi"
 
-
-cdef extern from "libyasm/bitvect.h":
-    cdef void BitVector_Boot()
-    cdef void BitVector_Shutdown()
-
-def __initialize():
+cdef __initialize():
     BitVector_Boot()
     yasm_intnum_initialize()
     yasm_floatnum_initialize()