diff options
author | Stefan Behnel <stefan_ml@behnel.de> | 2020-05-30 09:50:33 +0200 |
---|---|---|
committer | Stefan Behnel <stefan_ml@behnel.de> | 2020-05-30 09:50:33 +0200 |
commit | 7ccc80805daa5c316b76eb2542e107c12cd75c61 (patch) | |
tree | 1ebd876c37e6010a029c966c2e0a2451b7053638 | |
parent | 78aa444dee046cc9c1ac28123ea5b2bac82d9093 (diff) | |
download | cython-7ccc80805daa5c316b76eb2542e107c12cd75c61.tar.gz |
Compile DFA.py and tie it into the other lexer classes.
-rw-r--r-- | Cython/Plex/DFA.pxd | 30 | ||||
-rw-r--r-- | Cython/Plex/DFA.py | 14 | ||||
-rw-r--r-- | Cython/Plex/Machines.pxd | 20 | ||||
-rw-r--r-- | Cython/Plex/Machines.py | 9 | ||||
-rw-r--r-- | Cython/Plex/Transitions.pxd | 22 | ||||
-rw-r--r-- | Cython/Plex/Transitions.py | 14 | ||||
-rwxr-xr-x | setup.py | 1 |
7 files changed, 78 insertions, 32 deletions
diff --git a/Cython/Plex/DFA.pxd b/Cython/Plex/DFA.pxd new file mode 100644 index 000000000..063460db9 --- /dev/null +++ b/Cython/Plex/DFA.pxd @@ -0,0 +1,30 @@ +# cython: + +cimport cython + +from . cimport Machines +from .Transitions cimport TransitionMap + + +@cython.final +cdef class StateMap: + cdef Machines.FastMachine new_machine + cdef dict old_to_new_dict + cdef dict new_to_old_dict + + cdef old_to_new(self, dict old_state_set) + + @cython.locals(state=Machines.Node) + cdef highest_priority_action(self, dict state_set) + + cdef make_key(self, dict state_set) + + +@cython.locals(new_machine=Machines.FastMachine, transitions=TransitionMap) +cpdef nfa_to_dfa(Machines.Machine old_machine, debug=*) + +cdef set_epsilon_closure(dict state_set) +cdef dict epsilon_closure(Machines.Node state) + +@cython.locals(state_set_2=dict, state2=Machines.Node) +cdef add_to_epsilon_closure(dict state_set, Machines.Node state) diff --git a/Cython/Plex/DFA.py b/Cython/Plex/DFA.py index 07419415b..66dc4a379 100644 --- a/Cython/Plex/DFA.py +++ b/Cython/Plex/DFA.py @@ -1,3 +1,4 @@ +# cython: auto_cpdef=True """ Python Lexical Analyser @@ -95,14 +96,11 @@ class StateMap(object): Helper class used by nfa_to_dfa() to map back and forth between sets of states from the old machine and states of the new machine. """ - new_machine = None # Machine - old_to_new_dict = None # {(old_state,...) : new_state} - new_to_old_dict = None # {id(new_state) : old_state_set} def __init__(self, new_machine): - self.new_machine = new_machine - self.old_to_new_dict = {} - self.new_to_old_dict = {} + self.new_machine = new_machine # Machine + self.old_to_new_dict = {} # {(old_state,...) : new_state} + self.new_to_old_dict = {} # {id(new_state) : old_state_set} def old_to_new(self, old_state_set): """ @@ -140,9 +138,7 @@ class StateMap(object): Convert a set of states into a uniquified sorted tuple suitable for use as a dictionary key. """ - lst = list(state_set) - lst.sort() - return tuple(lst) + return tuple(sorted(state_set)) def dump(self, file): from .Transitions import state_set_str diff --git a/Cython/Plex/Machines.pxd b/Cython/Plex/Machines.pxd index 9235106b8..786f6c030 100644 --- a/Cython/Plex/Machines.pxd +++ b/Cython/Plex/Machines.pxd @@ -1,10 +1,24 @@ cimport cython +from .Actions cimport Action +from .Transitions cimport TransitionMap + + +@cython.final +cdef class Machine: + cdef readonly list states + cdef readonly dict initial_states + cdef readonly Py_ssize_t next_state_number + + cpdef new_state(self) + cpdef new_initial_state(self, name) + + @cython.final cdef class Node: - cdef readonly object transitions # TransitionMap - cdef readonly object action # Action - cdef public object epsilon_closure # dict + cdef readonly TransitionMap transitions + cdef readonly Action action + cdef public dict epsilon_closure cdef readonly Py_ssize_t number cdef readonly long action_priority diff --git a/Cython/Plex/Machines.py b/Cython/Plex/Machines.py index e55895b31..e9ae4af7a 100644 --- a/Cython/Plex/Machines.py +++ b/Cython/Plex/Machines.py @@ -24,13 +24,10 @@ LOWEST_PRIORITY = -maxint class Machine(object): """A collection of Nodes representing an NFA or DFA.""" - states = None # [Node] - next_state_number = 1 - initial_states = None # {(name, bol): Node} - def __init__(self): - self.states = [] - self.initial_states = {} + self.states = [] # [Node] + self.initial_states = {} # {(name, bol): Node} + self.next_state_number = 1 def __del__(self): for state in self.states: diff --git a/Cython/Plex/Transitions.pxd b/Cython/Plex/Transitions.pxd new file mode 100644 index 000000000..53dd4d58e --- /dev/null +++ b/Cython/Plex/Transitions.pxd @@ -0,0 +1,22 @@ +cimport cython + +cdef long maxint + +@cython.final +cdef class TransitionMap: + cdef list map + cdef dict special + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add(self, event, new_state) + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add_set(self, event, new_set) + + @cython.locals(i=cython.Py_ssize_t, n=cython.Py_ssize_t, else_set=cython.bint) + cpdef iteritems(self) + + @cython.locals(map=list, lo=cython.Py_ssize_t, mid=cython.Py_ssize_t, hi=cython.Py_ssize_t) + cdef split(self, long code) + + cdef get_special(self, event) diff --git a/Cython/Plex/Transitions.py b/Cython/Plex/Transitions.py index 10ee513e7..c0f74b9e5 100644 --- a/Cython/Plex/Transitions.py +++ b/Cython/Plex/Transitions.py @@ -5,18 +5,12 @@ This version represents state sets directly as dicts for speed. """ from __future__ import absolute_import -import cython - -cython.declare(maxint=cython.long) - try: from sys import maxsize as maxint except ImportError: from sys import maxint -@cython.final -@cython.cclass class TransitionMap(object): """ A TransitionMap maps an input event to a set of states. @@ -45,8 +39,6 @@ class TransitionMap(object): kept separately in a dictionary. """ - cython.declare(map=list, special=dict) - def __init__(self, map=None, special=None): if not map: map = [-maxint, {}, maxint] @@ -55,7 +47,6 @@ class TransitionMap(object): self.map = map # The list of codes and states self.special = special # Mapping for special events - @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t, map=list) def add(self, event, new_state): """ Add transition to |new_state| on |event|. @@ -71,7 +62,6 @@ class TransitionMap(object): else: self.get_special(event)[new_state] = 1 - @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t, map=list) def add_set(self, event, new_set): """ Add transitions to the states in |new_set| on |event|. @@ -93,7 +83,6 @@ class TransitionMap(object): """ return self.special.get('') - @cython.locals(map=list, i=cython.Py_ssize_t, n=cython.Py_ssize_t, else_set=cython.bint) def iteritems(self): """ Return the mapping as an iterable of ((code1, code2), state_set) and @@ -121,8 +110,6 @@ class TransitionMap(object): # ------------------- Private methods -------------------- - @cython.ccall - @cython.locals(map=list, lo=cython.Py_ssize_t, mid=cython.Py_ssize_t, hi=cython.Py_ssize_t, code=cython.long) def split(self, code): """ Search the list for the position of the split point for |code|, @@ -153,7 +140,6 @@ class TransitionMap(object): map[hi:hi] = [code, map[hi - 1].copy()] return hi - @cython.ccall def get_special(self, event): """ Get state set for special event, adding a new entry if necessary. @@ -86,6 +86,7 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan "Cython.Plex.Actions", "Cython.Plex.Machines", "Cython.Plex.Transitions", + "Cython.Plex.DFA", "Cython.Compiler.Scanning", "Cython.Compiler.Visitor", "Cython.Compiler.FlowControl", |