B

�j\=�@s�ddgZddlZddlZGdd�de�ZGdd�de�ZGdd	�d	e�ZGd
d�de�ZGdd�de�Z	Gd
d�de
�ZGdd�d�ZGdd�de
�ZGdd�de�ZGdd�ded�ZdS)�Lexer�LexerStateChange�Nc@seZdZdZdd�ZdS)�LexErrora
    Exception raised if an invalid character is encountered and no default
    error handler function is defined.  The .text attribute of the exception
    contains all remaining untokenized text. The .error_index is the index
    location of the error.
    cCs|f|_||_||_dS)N)�args�text�error_index)�self�messagerr�r
�=/u1/h0/jkinne/public_html/cs420-s2019/code/sly-0.4/sly/lex.py�__init__.szLexError.__init__N)�__name__�
__module__�__qualname__�__doc__rr
r
r
rr'src@seZdZdZdS)�PatternErrorzn
    Exception raised if there's some kind of problem with the specified
    regex patterns in the lexer.
    N)r
rrrr
r
r
rr3src@seZdZdZdS)�LexerBuildErrorzN
    Exception raised if there's some sort of problem building the lexer.
    N)r
rrrr
r
r
rr:src@seZdZdZddd�ZdS)rz9
    Exception raised to force a lexing state change
    NcCs||_||_dS)N)�newstate�tok)rrrr
r
rrDszLexerStateChange.__init__)N)r
rrrrr
r
r
rr@sc@seZdZdZdZdd�ZdS)�Tokenz+
    Representation of a single token.
    )�type�value�lineno�indexc	Cs&d|j�d|j�d|j�d|j�d�	S)NzToken(type=z, value=z	, lineno=z, index=�))rrrr)rr
r
r�__repr__MszToken.__repr__N)r
rrr�	__slots__rr
r
r
rrHsrcs2eZdZed�fdd�	�Zdd�Zdd�Z�ZS)	�TokenStrNcst��||�}||_||_|S)N)�super�__new__�key�remap)�clsrr r!r)�	__class__r
rrQszTokenStr.__new__cCs|jdk	r||j|j|f<dS)N)r!r )rr rr
r
r�__setitem__Ys
zTokenStr.__setitem__cCs |jdk	r|j|j|j|f<dS)N)r!r )rr r
r
r�__delitem__^s
zTokenStr.__delitem__)NN)r
rr�staticmethodrr$r%�
__classcell__r
r
)r#rrPsrc@seZdZdd�ZdS)�_BeforecCs||_||_dS)N)r�pattern)rrr)r
r
rrcsz_Before.__init__N)r
rrrr
r
r
rr(bsr(cs@eZdZdZdd�Z�fdd�Z�fdd�Z�fdd	�Z�ZS)
�
LexerMetaDictzZ
    Special dictionary that prohibits duplicate definitions in lexer specifications.
    cCsi|_g|_i|_dS)N)�before�deleter!)rr
r
rrkszLexerMetaDict.__init__cs�t|t�rt|||j�}t|t�r>|j|j|<t|j||j�}||kr�t|t�s�||}t|t�r�t	|�rr||_nt
d|�d���t��||�dS)NzName z
 redefined)
�
isinstance�strrr!r(rr+r)�property�callable�AttributeErrorrr$)rr rZprior)r#r
rr$ps


zLexerMetaDict.__setitem__cs.|j�|�||kr|��rnt��|�SdS)N)r,�append�isupperrr%)rr )r#r
rr%�szLexerMetaDict.__delitem__csH||kr8|�d�d��r8|dd�dkr8t|||j�St��|�SdS)N�ignore_�����_)�splitr3rr!r�__getitem__)rr )r#r
rr9�s*zLexerMetaDict.__getitem__)	r
rrrrr$r%r9r'r
r
)r#rr*gs
r*cs,eZdZdZedd��Z�fdd�Z�ZS)�	LexerMetaz/
    Metaclass for collecting lexing rules
    cCs"t�}dd�}||d<t|d<|S)Ncs|f|���fdd�}|S)Ncs:d�dd��D��}t|d�r0|d|j|_n||_|S)N�|css|]}d|�d�VqdS)�(rNr
)�.0Zpatr
r
r�	<genexpr>�szELexerMeta.__prepare__.<locals>._.<locals>.decorate.<locals>.<genexpr>r))�join�hasattrr))�funcr))�patternsr
r�decorate�s

z2LexerMeta.__prepare__.<locals>._.<locals>.decorater
)r)ZextrarCr
)rBrr7�s
z LexerMeta.__prepare__.<locals>._r7r+)r*r()�meta�name�bases�dr7r
r
r�__prepare__�s
zLexerMeta.__prepare__cs^|d=|d=dd�|��D�}t��||||�}t|�|_|j|_|j|_|j	|_
|��|S)Nr7r+cSs,i|]$\}}t|t�rt|�n|t|��qSr
)r-rr.)r=r �valr
r
r�
<dictcomp>�sz%LexerMeta.__new__.<locals>.<dictcomp>)�itemsrr�dict�_attributesr!�_remapr+�_beforer,�_delete�_build)rDZclsnamerFZ
attributesZcls_attributesr")r#r
rr�s
zLexerMeta.__new__)r
rrr�classmethodrHrr'r
r
)r#rr:�sr:c@s�eZdZe�Ze�ZdZdZeZ	e�Z
iZe�ZiZ
iZiZdZdZedd��Zedd��Zdd	�Zd
d�Zdd
�Zddd�Zdd�ZdS)r�rNcs�g}x$�jD]}t|t�r|�|j�qWt|�}�x4�j��D�]$\}}|�jksj|�	d�sjt
|d��r<t|�r�t
|d�s�td|�d���||kr�|�
|||f�}||f||<|||<n~t|t��r$|�jk�r$�j|}||k�r|�
|||f�}|�|||f�n|�||f�|||<n|�||f�|||<q@t|t�r@|�	d�s@|dkr@t|�d���q@W�fdd	�|D�}|�_dS)
Nr4r)z	function z doesn't have a regex patternr7>�ignore�literalsz  does not match a name in tokenscs"g|]\}}|�jkr||f�qSr
)rP)r=r r)r"r
r�
<listcomp>sz(Lexer._collect_rules.<locals>.<listcomp>)�	__bases__r-r:�extend�_rulesrLrMrK�_token_names�
startswithr@r0rrrrO�insertr2r.)r"Zrules�baseZexistingr r�nr+r
)r"r�_collect_rules�s4
 




zLexer._collect_rulescCs2dt|�krt|j�d���|jt|j�B|_t|j�|_t|j�|_t|j	�|_	x<|j
��D].\\}}}||j	kr~i|j	|<||j	||<q^Wt�}x |j	��D]}|�
|���q�W|t|j�}|r�d�|�}t|�d���|��g}x�|jD]�\}	}
|	�d��r"|	dd�}	|j�|	�t|
t��r4|
}nt|
��rR|
|j|	<t|
d�}d	|	�d
|�d�}y|j�||j�}
Wn4tk
�r�}ztd|	���|�Wdd}~XYnX|
�d
��r�td|	�d���|�|�q�W|�s�dS|j�d�|�|j�|_t|jt��std��t dd�|j!D���s.td��dS)z�
        Build the lexer object from the collected tokens and regular expressions.
        Validate the rules to make sure they look sane.
        �tokensz) class does not define a tokens attributez, z not included in token(s)r4�Nr)z(?P<�>rzInvalid regex for token rSzRegex for token z matches empty inputr;z!ignore specifier must be a stringcss|]}t|t�VqdS)N)r-r.)r=Zlitr
r
rr>MszLexer._build.<locals>.<genexpr>z%literals must be specified as strings)"�varsrrrZ�setr`�_ignored_tokensrL�_token_funcs�
_remappingrNrK�values�updater?r_rYr[�addr-r.r0�getattr�regex_module�compile�reflags�	Exceptionr�matchr2�
_master_rerT�allrU)r"r rIZnewtokZ
remapped_toksrGZ	undefinedZmissingZpartsZtoknamerr)�partZcpat�er
r
rrQsV





"zLexer._buildcCs,t|t�std��|jr"|�|�||_dS)z)
        Begin a new lexer state
        z!state must be a subclass of LexerN)r-r:�AssertionError�_Lexer__set_stater#)rr"r
r
r�beginPs
zLexer.begincCs.|jdkrg|_|j�t|��|�|�dS)z7
        Push a new lexer state onto the stack
        N)�_Lexer__state_stackr2rrw)rr"r
r
r�
push_stateYs
zLexer.push_statecCs|�|j���dS)z2
        Pop a lexer state from the stack
        N)rwrx�pop)rr
r
r�	pop_statebszLexer.pop_stater6c#s�d������������fdd�}||_|t|��||_�zd�x\y||�krf|d7}wNWntk
r|dSXt�}||_||_��||�}|�r0|��}|�	�|_
|j|_|j�krވ|j�|j
|j�|_|j�k�r||_||_�|j||�}|j}|j}|�sqN|j�k�r(qN|VqN||�k�r`|||_
|j
|_|d7}|VqN||_||_d|_||d�|_
|�
|�}|dk	�r�|V|j}|j}qNWWd||_||_||_XdS)Ncs(|j�|j�|j�|j�|j�|j�dS)N)rerqrTrfrUrg)r")�_ignorere�	_literalsrqrgrfr
r�
_set_stateksz"Lexer.tokenize.<locals>._set_stater6ZERROR)rvrr�
IndexErrorrrrrp�end�grouprZ	lastgroup�get�error)rrrrr~r�mr
)r|rer}rqrgrfr�tokenizehsf	




zLexer.tokenizecCs(td|jd�d|j��|j|j��dS)NzIllegal character rz
 at index )rrr)r�tr
r
rr��szLexer.error)r6r)r
rrrdr`rUrTrn�rerlrZrfrergrPrNrxrvrRr_rQrwryr{r�r�r
r
r
rr�s(<J		
N)�	metaclass)�__all__r��copyrorrrr�objectrr.rr(rLr*rr:rr
r
r
r�<module>"s((