o
    0 i                     @   s2  d Z ddlZddlZddlZddlmZ ddlmZmZ ddl	Z	ddl
mZ ddlmZ ddlmZmZmZmZmZmZmZmZ dd	lmZmZmZmZmZmZmZmZ dd
l m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddlm4Z4m5Z5 ddl6m7Z7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z? ddl@mAZAmBZBmCZCmDZDmEZE eCddZFdgZGdZHdZIe2dZJi dddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:i d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_ZKi d`dadbdcdddedfdgdhdidjdkdldmdnddoddpdqdrdsdtdudLdKdvdwdxdydzd{d|d}d~eI dddddddddddddZLi ddgdddgdg dddgddvdgdddgdddgddxdgddgdg ddddgddgdddgdg ddg ddg dddzgi ddxgddxdzgdddgddgddgddgdddgdddgdddgdddgddgddgdg dâdddgddgddgdddgdgddgddgdd|gdМZMdZNdZOG ddԄ deAZPeFG ddք deDZQG dd؄ deBZRG ddڄ deAZSG dd܄ deDZTG ddބ deDZUG dd deDZVdd ZWdd ZXdd ZYeFG dd deDZZde$fddZ[G dd deEZ\G dd deDZ]dd Z^G dd dZ_eddZ`G dd dZaeadeGZbdd Zcdd ZdeFG dd deDZedd  Zfdd Zgddgfddgfdg dfd	g d
fddgfddgfdg dfdg dfddgfddgfddgfddgfddgfgZhdd Zid;dd Zjd!d" Zkd#d$ Zld;d%emdemdeee;emf  fd&d'Znd<d(d)Zod*d+ Zpd,d- ZqG d.d/ d/ZrG d0d1 d1Zsd2d3 Ztg fd4d5Zud6d7 Zvd8emdemfd9d:ZwdS (=  zCParses and compiles Lark grammars into an internal representation.
    N)
namedtuple)copydeepcopy)literal_eval)suppress)ListTupleUnionCallableDictOptionalSequence	Generator   )bfsloggerclassify_boolis_id_continueis_id_startbfs_all_uniquesmall_factors
OrderedSet)TokenTerminalDef
PatternStr	PatternREPattern)ParseTreeBuilder)ParsingFrontend)	LexerConf
ParserConf)RuleOptionsRuleTerminalNonTerminalSymbolTOKEN_DEFAULT_PRIORITY)classify
dedup_list)GrammarErrorUnexpectedCharactersUnexpectedToken
ParseErrorUnexpectedInput)TreeSlottedTree)TransformerVisitorv_argsTransformer_InPlaceTransformer_NonRecursiveT)inlineZgrammarsz.larkZimsluxZ	__empty__.DOT,COMMA:COLON;	SEMICOLON+PLUS-MINUS*STAR/SLASH\	BACKSLASH|VBAR?ZQMARK!ZBANG@AT#HASH$ZDOLLAR%PERCENT^
CIRCUMFLEX&Z	AMPERSAND_Z
UNDERSCORE<ZLESSTHAN>ZMORETHAN=EQUAL"ZDBLQUOTE'ZQUOTE`Z	BACKQUOTE~TILDE(LPAR)RPAR{LBRACE}RBRACE[LSQB]RSQB
NEWLINEz
CRLFZTABSPACE)	 Z_LPARz\(Z_RPARz\)Z_LBRAz\[Z_RBRAz\]Z_LBRACEz\{Z_RBRACEz\}OPz[+*]|[?](?![a-z_])Z_COLONZ_COMMAZ_ORz\|Z_DOTz\.(?!\.)Z_DOTDOTz\.\.ZRULE_MODIFIERSz(!|![?]?|[?]!?)(?=[_a-z])ZRULEz_?[a-z][_a-z0-9]*ZTERMINALz_?[A-Z][_A-Z0-9]*STRINGz"(\\"|\\\\|[^"\n])*?"i?z/(?!/)(\\/|\\\\|[^/])*?/[%s]*z(\r?\n)+\s*z(\r?\n)+\s*\|z[ \t]+z\s*//[^\n]*|\s*#[^\n]*z\\[ ]*\nz->z%ignorez	%overridez%declarez%extendz%importz[+-]?\d+)REGEXP_NLZ_NL_ORWSCOMMENTrG   Z_TOZ_IGNOREZ	_OVERRIDEZ_DECLAREZ_EXTENDZ_IMPORTNUMBERstartZ_listZ_itemz_list _item)ruletermignoreimportdeclareoverrideextendru   rz   zBrule_modifiers RULE template_params priority _COLON expansions _NLZrule_modifiers priorityz_DOT NUMBERZtemplate_paramsz _LBRACE _template_params _RBRACEZ_template_paramsz_template_params _COMMA RULE
expansionsZ_expansions)aliasz_expansions _OR aliasz_expansions _NL_OR aliasz?aliaszexpansion _TO nonterminal	expansionZ
_expansionz_expansion exprz?expr)atomzatom OPzatom TILDE NUMBERz atom TILDE NUMBER _DOTDOT NUMBERz?atom)z_LPAR expansions _RPARmaybevaluer   )terminalnonterminalliteralrangetemplate_usager   r   z?namez?symbolr   z_LBRA expansions _RBRAr   zSTRING _DOTDOT STRINGr   z*nonterminal _LBRACE _template_args _RBRACEZ_template_argsz_template_args _COMMA valuer{   zTERMINAL _COLON expansions _NLz*TERMINAL _DOT NUMBER _COLON expansions _NLr   z_OVERRIDE rulez_OVERRIDE termr   z_EXTEND rulez_EXTEND termr|   z_IGNORE expansions _NLr~   z_DECLARE _declare_args _NLr}   )z_IMPORT _import_path _NLz._IMPORT _import_path _LPAR name_list _RPAR _NLz!_IMPORT _import_path _TO name _NLZ_import_path
import_libZ
import_relZ_import_argsz_DOT _import_argsnamez_import_args _DOT name
_name_listz_name_list _COMMA namesymbolz_declare_args symbolrt   )	name_listr   Z_declare_argsr      2   c                   @   sp   e Zd ZdefddZdedefddZdeee	ef  de
e	d	d	f fd
dZde	fddZde	fddZd	S )FindRuleSizekeep_all_tokensc                 C   s
   || _ d S Nr   )selfr    r   ]/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/lark/load_grammar.py__init__      
zFindRuleSize.__init__symreturnc                 C   sD   t |tr|jd S t |tr| jp|j S |tu rdS J |)NrV   F)
isinstancer$   r   
startswithr#   r   
filter_out_EMPTY)r   r   r   r   r   _will_not_get_removed   s   

z"FindRuleSize._will_not_get_removedargsNc                 c   sD    |D ]}t |tr|V  qt |tr| |rdndV  qJ d S )Nr   r   )r   intr%   r   )r   r   ar   r   r   _args_as_int   s   

zFindRuleSize._args_as_intc                 C      t | |S r   )sumr   r   r   r   r   r   r         zFindRuleSize.expansionc                 C   r   r   )maxr   r   r   r   r   r      r   zFindRuleSize.expansions)__name__
__module____qualname__boolr   r%   r   r   r	   r   r   r   r   r   r   r   r   r   r      s    (		r   c                   @   s   e Zd Zdd ZdefddZdd Zded	efd
dZdd Z	dd Z
dededefddZdedefddZdefddZdS )EBNF_to_BNFc                 C   s"   g | _ i | _d| _d| _d | _d S )NZanonr   )	new_rulesrules_cacheprefixirule_optionsr   r   r   r   r      s
   
zEBNF_to_BNF.__init__innerc                 C   s$   d| j || jf }|  jd7  _|S )Nz
__%s_%s_%dr   )r   r   )r   r   new_namer   r   r   
_name_rule   s   zEBNF_to_BNF._name_rulec                 C   s*   t |}| j||| jf || j|< |S r   )r$   r   appendr   r   )r   keyr   r   tr   r   r   	_add_rule   s   
zEBNF_to_BNF._add_ruletype_exprc              
   C   s`   z| j | W S  ty/   | |}t|}tdtd|gtd||gg}| ||| Y S w )Nr   r   )r   KeyErrorr   r$   STr   )r   r   r   r   r   treer   r   r   _add_recurse_rule   s   

zEBNF_to_BNF._add_recurse_rulec              
   C   sn   ||||f}z| j | W S  ty6   | d||f }tdtd|g| |g|  g}| ||| Y S w )aN  Generate a rule that repeats target ``a`` times, and repeats atom ``b`` times.

        When called recursively (into target), it repeats atom for x(n) times, where:
            x(0) = 1
            x(n) = a(n) * x(n-1) + b

        Example rule when a=3, b=4:

            new_rule: target target target atom atom atom atom

        zrepeat_a%d_b%dr   r   )r   r   r   r   r   )r   r   btargetr   r   r   r   r   r   r   _add_repeat_rule  s   "zEBNF_to_BNF._add_repeat_rulec           	   	      s    |df}z| j | W S  tyB   | d |f }tdfddt D  fddt|D  }| ||| Y S w )a_  Creates a rule that matches atom 0 to (a*n+b)-1 times.

        When target matches n times atom, and target_opt 0 to n-1 times target_opt,

        First we generate target * i followed by target_opt, for i from 0 to a-1
        These match 0 to n*a - 1 times atom

        Then we generate target * a followed by atom * i, for i from 0 to b-1
        These match n*a to n*a + b-1 times atom

        The created rule will not have any shift/reduce conflicts so that it can be used with lalr

        Example rule when a=3, b=4:

            new_rule: target_opt
                    | target target_opt
                    | target target target_opt

                    | target target target
                    | target target target atom
                    | target target target atom atom
                    | target target target atom atom atom

        optzrepeat_a%d_b%d_optr   c                    s"   g | ]}t d  g| g qS r   r   .0r   )r   
target_optr   r   
<listcomp>4  s    z4EBNF_to_BNF._add_repeat_opt_rule.<locals>.<listcomp>c                    s&   g | ]}t d g  g|  qS r   r   r   )r   r   r   r   r   r   6  s    )r   r   r   r   r   r   )	r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   _add_repeat_opt_rule  s   z EBNF_to_BNF._add_repeat_opt_rulerz   mnmxc                    s   |t k rtd fddt||d D S  }t|tD ]\}}| ||| }q||kr/|S || d }t|t} }	tdg }
|dd D ]\}}| |||	|
 }
| |||	 }	qG|d \}}| |||	|
 }
tdtd|g|
g gS )z\Generates a rule tree that repeats ``rule`` exactly between ``mn`` to ``mx`` times.
        r   c                    s   g | ]
}t d  g| qS r   r   )r   nrz   r   r   r   @      z1EBNF_to_BNF._generate_repeats.<locals>.<listcomp>r   r   N)REPEAT_BREAK_THRESHOLDr   r   r   SMALL_FACTOR_THRESHOLDr   r   )r   rz   r   r   Z	mn_targetr   r   diffZdiff_factorsZdiff_targetZdiff_opt_targetr   r   r   _generate_repeats;  s"   "

zEBNF_to_BNF._generate_repeatsopc                 G   s   |j dkrtdg }td||gS |j dkr| d|S |j dkr1| d|}td|tdg gS |j dkrdt|d	krEt|d
  }}ntt|\}}||k sT|d
k r]td|||f | |||S J |)NrJ   r   r   r>   plusrB   starr^   r   r   'Bad Range for %s (%d..%d isn't allowed))r   r   r   lenr   mapr)   r   )r   rz   r   r   emptyr   r   r   r   r   r   r   Z  s    




zEBNF_to_BNF.exprc                 C   s:   | j o| j j}t||}tdtg| }td||gS )Nr   r   )r   r   r   	transformr   r   )r   rz   r   Z	rule_sizer   r   r   r   r   w  s   zEBNF_to_BNF.maybeN)r   r   r   r   strr   r   r.   r   r   r   r   r   r   r   r   r   r   r   r   r      s    %r   c                   @   sB   e Zd ZedefddZdefddZdd Zdefdd	Zd
S )SimplifyRule_Visitorr   c                 C   s"   |  | jr	 |  | jsd S d S r   )Zexpand_kids_by_datadatar   r   r   r   _flatten  s   zSimplifyRule_Visitor._flattenc                    sj     tjD ](\ }t|tr2|jdkr2d_ fddt|jD _   d S q
d S )Nr   c              	      s2   g | ]  td  fddtjD qS )r   c                    s    g | ]\}} |krn|qS r   r   )r   jother)r   optionr   r   r     s    z=SimplifyRule_Visitor.expansion.<locals>.<listcomp>.<listcomp>)visitr   	enumeratechildren)r   r   r   r   )r   r   r     s
    z2SimplifyRule_Visitor.expansion.<locals>.<listcomp>)r   r   r   r   r.   r   r(   )r   r   childr   r   r   r     s   

zSimplifyRule_Visitor.expansionc                 C   sR   |j \}}|jdkr'g }|j d j D ]}|td||g qd|_||_ d S d S )Nr   r   r   )r   r   r   r   )r   r   rz   Z
alias_namealiasesr   r   r   r   r     s   


zSimplifyRule_Visitor.aliasc                 C   s6   |  | tt|jt|jkrt|j|_d S d S r   )r   r   setr   r(   )r   r   r   r   r   r     s   
zSimplifyRule_Visitor.expansionsN)	r   r   r   staticmethodr.   r   r   r   r   r   r   r   r   r   ~  s    	r   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )RuleTreeToTextc                 C   s   |S r   r   )r   xr   r   r   r     s   zRuleTreeToText.expansionsc                 C   s   |d fS r   r   )r   symbolsr   r   r   r        zRuleTreeToText.expansionc                 C   s.   |\\}}}|d u sJ ||d|f||j fS )Nr@   r   )r   r   r   _aliasr   r   r   r   r     s   
zRuleTreeToText.aliasN)r   r   r   r   r   r   r   r   r   r   r     s    r   c                   @   s$   e Zd ZdZdd Zedd ZdS )PrepareAnonTerminalszfCreate a unique list of anonymous terminals. Attempt to give meaningful names to them when we add themc                 C   s8   || _ dd | j D | _dd |D | _d| _d | _d S )Nc                 S      h | ]}|j qS r   r   r   tdr   r   r   	<setcomp>      z0PrepareAnonTerminals.__init__.<locals>.<setcomp>c                 S   s   i | ]}|j |qS r   )patternr   r   r   r   
<dictcomp>      z1PrepareAnonTerminals.__init__.<locals>.<dictcomp>r   )	terminalsterm_setterm_reverser   r   )r   r  r   r   r   r     s
   
zPrepareAnonTerminals.__init__c                 C   st  |j }|| jv r|j| j| jjkrtd| d }t|tr_z| j| j}W nL ty^   zt	| }W n  tyT   |rRt
|rRt|d rR| | jvrR| }Y nw || jv r\d }Y nw t|trp|| jv ro| j| j}nJ ||d u rd| j }|  jd7  _|| jvr|| jvsJ | j| t||}|| j|< | j| | jr| jjrdnt|t}t||dS )Nz+Conflicting flags for the same terminal: %sr   Fz	__ANON_%dr   r   )r   r  flagsr   r)   r   r   r   r   _TERMINAL_NAMESr   r   upperr  r   r   addr   r  r   r   r   r#   )r   pr   	term_nameZtermdefr   r   r   r   r     sF   
&






zPrepareAnonTerminals.patternN)r   r   r   __doc__r   inline_argsr   r   r   r   r   r     s
    r   c                   @   s(   e Zd ZdZdd Zdd Zdd ZdS )	_ReplaceSymbolszHelper for ApplyTemplatesc                 C   s
   i | _ d S r   )namesr   r   r   r   r     r   z_ReplaceSymbols.__init__c                 C   sH   t |dkrt|d tr|d j| jv r| j|d j S | d|d S )Nr   r   r   )r   r   r%   r   r  __default__)r   cr   r   r   r     s   *z_ReplaceSymbols.valuec                 C   sD   |d j }|| jv r| d| j| g|dd   d S | d|d S )Nr   r   r   )r   r  r  )r   r  r   r   r   r   r     s   

"z_ReplaceSymbols.template_usageN)r   r   r   r  r   r   r   r   r   r   r   r    s
    r  c                   @   s    e Zd ZdZdd Zdd ZdS )ApplyTemplateszIApply the templates, creating new rules that represent the used templatesc                 C   s   || _ t | _t | _d S r   )	rule_defsr  replacerr   created_templates)r   r  r   r   r   r     s   zApplyTemplates.__init__c           	         s   |d j  |dd  }d ddd |D f }|| jvr^| j|  fdd| jD \\}}}}t|t|ks?J |t|}tt||| j	_
| j	| | j|g |t|f t|S )Nr   r   z%s{%s}r8   c                 s   s    | ]}|j V  qd S r   r   )r   r   r   r   r   	<genexpr>  s    z0ApplyTemplates.template_usage.<locals>.<genexpr>c                 3   s     | ]}|d   kr|V  qdS )r   Nr   r   r   r   r   r   r    s    )r   joinr  r	  r  r   r   dictzipr  r  r   r   r$   )	r   r  r   Zresult_nameZ_nparamsr   optionsZresult_treer   r   r   r   	  s   

zApplyTemplates.template_usageN)r   r   r   r  r   r   r   r   r   r   r    s    r  c                    s   t  fdd|D S )Nc                 3   s    | ]}  |V  qd S r   )rfind)r   r  sr   r   r    s    z_rfind.<locals>.<genexpr>)r   )r  choicesr   r  r   _rfind     r!  c              
   C   s   d}t | }|D ]3}||7 }|dkr;zt|}W n ty%   td|  w |dkr/|d7 }n|dvr7|d7 }||7 }q|dddd	}d
| }zt|} W | S  tyb } zt| |d }~ww )Nr   rF   z/Literal ended unexpectedly (bad escaping): `%r`\\ZUuxnftrz\"r[   r\   z\'z	u'''%s''')iternextStopIterationr)   replacer   SyntaxError)r  wr   r   Zn2Zto_evaler   r   r   eval_escaping  s2   


r+  c                 C   s,  t | tsJ | j}t|dd }|dksJ ||d  }tdd |D s*J || jdkr7d|v r7td| jd	krHd|v rHd
|vrHtd|d | }|d |d kr\|d dv s^J |dd }t|}|dkrrtd|  | jdkr|dd}t	||| jdS | jd	krt
||| jdS J d)Nz/"r   r   c                 s   s    | ]}|t v V  qd S r   )	_RE_FLAGS)r   fr   r   r   r  <      z&_literal_to_pattern.<locals>.<genexpr>rs   rl   z*You cannot put newlines in string literalsrt   r   zLYou can only use newlines in regular expressions with the `x` (verbose) flagr   z"/r   z$Empty terminals are not allowed (%s)r#  rF   )rawFz:Invariant failed: literal.type not in ["STRING", "REGEXP"])r   r   r   r!  alltyper)   r+  r'  r   r   )r   vZ
flag_startr  r   r  r   r   r   _literal_to_pattern6  s,    

r3  c                   @      e Zd Zdd Zdd ZdS )PrepareLiteralsc                 C   s   t dt|gS )Nr   )r   r3  )r   r   r   r   r   r   Y     zPrepareLiterals.literalc                 C   s   |j |j   krdksJ  J |jdd }|jdd }tt|tt|  kr1dks4J  J d||f }tdt|gS )Nrs   r   r   z[%s-%s]r   )r1  r   r   r+  r   r   )r   ry   endregexpr   r   r   r   \  s    ,zPrepareLiterals.rangeN)r   r   r   r   r   r   r   r   r   r5  W      r5  r   c                 C   s
   t | dS )Nr   )r   )r8  Z	flags_setr   r   r   _make_joined_patterne  r   r:  c                   @   sf   e Zd Zdd Zdee defddZdee defdd	Zdefd
dZdd Z	dd Z
dd ZdS )TerminalTreeToPatternc                 C   s
   |\}|S r   r   )r   Zpsr
  r   r   r   r   i  s   zTerminalTreeToPattern.patternitemsr   c                 C   sH   |st dS t|dkr|d S ddd |D }t|dd |D S )Nr   r   r   c                 s       | ]}|  V  qd S r   	to_regexpr   r   r   r   r  t  r.  z2TerminalTreeToPattern.expansion.<locals>.<genexpr>c                 S   r   r   r  r   r   r   r   r   u  r   z2TerminalTreeToPattern.expansion.<locals>.<setcomp>)r   r   r  r:  )r   r<  r   r   r   r   r   m  s   zTerminalTreeToPattern.expansionexpsc                 C   sP   t |dkr
|d S |jdd d dddd	 |D  }t|d
d |D S )Nr   r   c                 S   s   | j  | j t| j fS r   )	max_width	min_widthr   r   r   r   r   r   <lambda>}  s    z2TerminalTreeToPattern.expansions.<locals>.<lambda>)r   z(?:%s)rH   c                 s   r=  r   r>  r   r   r   r   r    r.  z3TerminalTreeToPattern.expansions.<locals>.<genexpr>c                 S   r   r   r@  r   r   r   r   r     r   z3TerminalTreeToPattern.expansions.<locals>.<setcomp>)r   sortr  r:  )r   rA  r   r   r   r   r   w  s
   z TerminalTreeToPattern.expansionsc                 C   s   |d d \}}|dkr:t |dkrdt|d  }n'tt|dd  \}}||k r3td|||f d||f }nt |dksBJ td| |f |jS )N   r^      z{%d}r   z{%d,%d}z(?:%s)%s)r   r   r   r)   r   r?  r  )r   r   r   r   r   r   r   r   r   r     s   zTerminalTreeToPattern.exprc                 C   s   |  |dg S )NrJ   )r   )r   r   r   r   r   r     r6  zTerminalTreeToPattern.maybec                 C   s   t d)NzBAliasing not allowed in terminals (You used -> in the wrong place))r)   )r   r   r   r   r   r     r   zTerminalTreeToPattern.aliasc                 C   s   |d S Nr   r   r   r2  r   r   r   r     r   zTerminalTreeToPattern.valueN)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r;  h  s    
r;  c                   @   s   e Zd Zdd ZdS )ValidateSymbolsc                 C   s   |\}t |ttfsJ |S r   )r   r.   r%   rJ  r   r   r   r     s   zValidateSymbols.valueN)r   r   r   r   r   r   r   r   rK    s    rK  c                 C   s   t d| S )z#Deepcopy tree `t` without recursionF)r4   r   r   r   r   r   nr_deepcopy_tree  s   rM  c                	   @   s   e Zd ZU eeeeeef f  ed< eeeeedf ee	f  ed< ee ed< deeeeedf ee	f  deeeeeef f  dee ddfddZ
deee ee ee f fd	d
ZdS )Grammar	term_defs.r  r|   r   Nc                 C   s   || _ || _|| _d S r   )rO  r  r|   )r   r  rO  r|   r   r   r   r     s   
zGrammar.__init__c           #         s  dd  j D }dd  jD }|D ]#\}\}}|d u rqt|d}t|dkr5|d js5td| qt t  fdd|D }	t	|	}
t t
  |
 t|9 t }g }d}|t|k r|| \}}}}|d7 }t|dkrwq^|r|jrtd	d
nd }||_||_||
_|}||}||||f |t|k sd||j7 }t|tdd |D ksJ dt }t }g }|D ]}|\}}}|| ||}t|D ]k\}\}}|r|drtd||f tdd |D }t|rt|pt }||_dd |D }n|}|D ]!}t|tsJ |jr3|r3|jr3t|t s0J d|_!qt"t#|||||}|| qqtt$|t|krt%|dd }|& D ].}t|dkr|d j'rytdd(dd |D  tdd |D t|ksJ q\tt)|}	 t|} dd |D dd |D O t*|fdd\}}!|!D ]	}"t+,d|" qt|| krnqdkrdd |D t*|	 fd d\}	}!|!rt+,d!d"d |!D  |	| j-fS )#Nc                 S   s$   g | ]\}\}}|t ||ffqS r   rM  )r   r   r   r
  r   r   r   r        $ z#Grammar.compile.<locals>.<listcomp>c                 S   s$   g | ]\}}}}||t ||fqS r   rP  )r   r   r
  r   or   r   r   r     rQ  r   r   r   zTerminals cannot be empty (%s)c                    s*   g | ]\}\}}|rt | ||qS r   )r   r   )r   r   	term_treer   )transformerr   r   r     s    Tr   c                 S   s   h | ]\}}}|qS r   r   )r   r   Z_tZ_or   r   r   r     s    z"Grammar.compile.<locals>.<setcomp>zWhoops, name collisionrV   zkRule %s is marked for expansion (it starts with an underscore) and isn't allowed to have aliases (alias=%s)c                 s   s    | ]}|t kV  qd S r   r   r   r   r   r   r   r    r.  z"Grammar.compile.<locals>.<genexpr>c                 S   s   g | ]}|t kr|qS r   rU  rV  r   r   r   r     s    Fc                 S   s   | S r   r   rD  r   r   r   rE  	  s    z!Grammar.compile.<locals>.<lambda>zXRules defined twice: %s

(Might happen due to colliding expansion of optionals: [] or ?)r   c                 s   s    | ]}d | V  qdS )z
  * %sNr   r   r   r   r   r    r.  c                 S   s   h | ]
}|j |j|jfqS r   )r   orderr  )r   rr   r   r   r     r   c                 S   s0   h | ]}|j D ]}t|tr||jkr|qqS r   )r   r   r$   origin)r   rX  r  r   r   r   r     s    
c                 S   s   h | ]}t |qS r   )r$   r   r  r   r   r   r     r  c                    s
   | j  v S r   )rY  )rX  )
used_rulesr   r   rE       
 zUnused rule: %srB   c                 S   s(   h | ]}|j D ]
}t|tr|jqqS r   )r   r   r#   r   )r   rX  r   r   r   r   r   &  s    c                    s    | j v p| j  jv p| j v S r   )r   r|   rL  )r   terminals_to_keep
used_termsr   r   rE  )  s     zUnused terminals: %sc                 S      g | ]}|j qS r   r   r  r   r   r   r   +  r   ).rO  r  list	find_datar   r   r)   r5  r;  r   rK  r  r   r   r!   r   r   r   r   r   r   r   r   r   r   tupleanyr   empty_indicesr   r%   is_termr#   r   r"   r$   r   r'   valuesr   r  r   r   r   debugr|   )#r   ry   r]  rO  r  r   rS  r   r   r  Zanon_tokens_transfZebnf_to_bnfrulesr   r  Z	rule_treer  r   r   resZrule_tree_to_textZsimplify_ruleZcompiled_rulesZrule_contentr   r   rd  Zexp_optionsr   rz   
duplicatesZdupsr  ZunusedrX  r   )r   r]  rT  r[  r^  r   compile  s   



"



 
zGrammar.compile)r   r   r   r   r   r   r.   r   __annotations__r!   r   r   r"   rk  r   r   r   r   rN    s   
  J(rN  PackageResourcezpkg_name pathc                   @   sr   e Zd ZU dZeed< ee ed< ddedee ddfddZd	d
 Zde	dee
f dedee
ef fddZdS )FromPackageLoadera  
    Provides a simple way of creating custom import loaders that load from packages via ``pkgutil.get_data`` instead of using `open`.
    This allows them to be compatible even from within zip files.

    Relative imports are handled, so you can just freely use them.

    pkg_name: The name of the package. You can probably provide `__name__` most of the time
    search_paths: All the path that will be search on absolute imports.
    pkg_namesearch_pathsr   r   Nc                 C   s   || _ || _d S r   )ro  rp  )r   ro  rp  r   r   r   r   A  s   
zFromPackageLoader.__init__c                 C   s   dt | j| j| jf S )Nz
%s(%r, %r))r1  r   ro  rp  r   r   r   r   __repr__E  s   zFromPackageLoader.__repr__	base_pathgrammar_pathc           	      C   s   |d u r| j }nt|tr|j| jkrt |jg}d }|D ]9}tj||}z	t	| j|}W n tyD } z|}W Y d }~qd }~ww t| j||rS|
 f  S df  S td|)Nr   z"Cannot find grammar in given paths)rp  r   rm  ro  IOErrorpathosr  pkgutilget_datadecode)	r   rs  rt  to_tryerrrv  	full_pathtextr*  r   r   r   __call__H  s"   &
zFromPackageLoader.__call__)rq  )r   r   r   r  r   rl  r   r   rr  r	   rm  r   r  r   r   r   r   rn  3  s   
 
,rn  Zlarkc           
   
   C   s  	 d}|   D ]R\}}|d u rq|dD ]C}|j\}t|tr(td||f t|trQz| |j }W n tyB   td|j w |d usIJ ||jd< d}qt|t	sXJ qq|s]nq|   D ]!\}}|r|jD ]}dd |
 D }	t||	v rtd	| qkqbd S )
NTFr   z0Rules aren't allowed inside terminals (%s in %s)z!Terminal used but not defined: %sr   c                 S      g | ]}t |qS r   )idrV  r   r   r   r     r  z+resolve_term_references.<locals>.<listcomp>zNRecursion in terminal '%s' (recursion is only allowed in rules, not terminals))r<  ra  r   r   r$   r)   r#   r   r   r.   iter_subtreesr  )
Z	term_dictchangedr   Z
token_treeexpitemZ
term_valuer{   r   idsr   r   r   resolve_term_referencesd  s@   



r  c                 C   s0   t | tsJ |  rt| | ddS t| S NrV   r  )r   r   isupperr#   r   r$   r  r   r   r   symbol_from_strcase  s   "r  c                   @   r4  )PrepareGrammarc                 C   s   t t||ddS r  )r#   r   r   r   r   r   r   r   r     r"  zPrepareGrammar.terminalc                 C   s
   t |jS r   )r$   r   r  r   r   r   r     r   zPrepareGrammar.nonterminalN)r   r   r   r   r   r   r   r   r   r    r9  r  c                 C   s"   | j dksJ dd | dD S )Nr   c                 S   s&   h | ]}| d d D ]}|jqqS )c                 S   s
   t | tS r   )r   r%   rL  r   r   r   rE    r\  z._find_used_symbols.<locals>.<setcomp>.<lambda>)Zscan_valuesr   )r   r   r   r   r   r   r     s
    z%_find_used_symbols.<locals>.<setcomp>r   )r   ra  r   r   r   r   _find_used_symbols  s   r  c                  C   s   zt jW S  tyP   dd t D } dd t D }dd |D }t|t }dd l	}t
| |g d}t||dg}d|_d	|_t||d t _t j Y S w )
Nc                 S   s   g | ]\}}t |t|qS r   )r   r   )r   r   r   r   r   r   r         z_get_parser.<locals>.<listcomp>c                 S   s,   g | ]\}}| d |t|d dfqS )rJ   )expand1)lstripr!   r   )r   r   r   r   r   r   r     s     c              	   S   sF   g | ]\}}}t |D ]\}}tt|d d | D |d|qqS )c                 S   r  r   )r  rZ  r   r   r   r     r  z*_get_parser.<locals>.<listcomp>.<listcomp>N)r   r"   r$   split)r   rX  xsrR  r   r   r   r   r   r     s
    (r   )rv   rw   rG   ry   basicZlalr)_get_parsercacheAttributeError	TERMINALSr<  RULESr   r   Zcreate_callbackrer   r    Z
lexer_typeZparser_typer   )r  rh  callbackr  Z
lexer_confZparser_confr   r   r   r    s&   
r  zIncorrect type of valueza: 1
zUnclosed parenthesisza: (
zUnmatched closing parenthesis)za: )
za: [)
za: (]
z5Expecting rule or terminal definition (missing colon))za
zA
za->
zA->
za A
z#Illegal name for rules or terminalszAa:
zAlias expects lowercase namez
a: -> "a"
zUnexpected colon)za::
za: b:
za: B:
za: "a":
zMisplaced operator)za: b??za: b(?)za:+
za:?
za:*
za:|*
z;Expecting option ("|") or a new rule or terminal definitionza:a
()
z"Terminal names cannot contain dotszA.B
z%Expecting rule or terminal definitionz"a"
z%import expects a namez%import "a"
z%ignore expects a valuez%ignore %import
c                 C   s*   |j | tdd}|r|S d|jv rdS d S )NT)Zuse_acceptsrs   zExpecting a value)Zmatch_examplesGRAMMAR_ERRORSexpected)parser*  errorr   r   r   _translate_parser_exception  s   
r  c              
   C   s   zt  | d |}W nC ty( } z|| }td|j|j||f d }~w tyN } z|| }tt  j|}|rItd||j|j|f  d }~ww t	 
|S )Nrl   z1Unexpected input at line %d column %d in %s: 

%sz%s, at line %s column %s

%s)r  r  r*   Zget_contextr)   linecolumnr+   r  r  r   )r~  r   ry   r   r*  contextr  r   r   r   _parse_grammar  s"   

r  c                 C   sL   t | tr"tt j| }|r|S d| jp| j}dt| j	|f S t| S )Nz, z*Unexpected token %r. Expected one of: {%s})
r   r+   r  r  r  r  Zacceptsr  r   token)r  Zerror2r  r   r   r   _error_repr  s   
r  c                 C   s:   dd }t d| fg|D ]\}}||r||f  S qd S )Nc              	   s   sX    | \}}|  D ] }t|d}z||}W n	 ty    Y q	w ||f |fV  q	d S )Nr   )r   r   
feed_tokenr,   )noderv  r
  choicer   Znew_pr   r   r   expand  s   
z*_search_interactive_parser.<locals>.expandr   )r   )interactive_parser	predicater  rv  r
  r   r   r   _search_interactive_parser  s   r  r~  c                    s^   g   fdd}t  j| d ||d}t dd }dd | D   D ]}d |d	 _q% S )
Nc                    s\     | t| f t| j dd \}}|D ]}| jt|d q| jtdd dS )Nc                 S   s   d|   v S )Nru   )r   )r
  r   r   r   rE    s    z7find_grammar_errors.<locals>.on_error.<locals>.<lambda>r   ru   rl   T)r   r  r  r  Zas_immutabler  r   )r*  Z
token_pathrV   
token_typeerrorsr   r   on_error  s   z%find_grammar_errors.<locals>.on_errorrl   )r  c                 S   s
   | d j S rI  )r  )r*  r   r   r   rE    r\  z%find_grammar_errors.<locals>.<lambda>c                 S   s   g | ]}|d  qS )r   r   )r   elr   r   r   r     r  z'find_grammar_errors.<locals>.<listcomp>r   )r  r  r'   rf  r  )r~  ry   r  Z_treeZerrors_by_liner*  r   r  r   find_grammar_errors  s   
r  c                    s    fdd}|S )Nc                    sT   |  v r	 |  } n| d dkrd| dd  f } nd| f } d ur(| } | S )Nr   rV   z_%s__%sr   z%s__%sr   r  r   base_mangler   r   r   mangle  s   
z_get_mangle.<locals>.mangler   )r   r   r  r  r   r  r   _get_mangle  s   r  c                 C   sT   |d u r| S t | } |  D ]}t|jD ]\}}t|tr&|||j|< qq| S r   )r   r  r   r   r   r%   Zrenamed)r  r  r   r   r  r   r   r   _mangle_definition_tree  s   
r  c           
   	   C   s   | j r| j \}d|v }|r|drtdd|v }nd}d}|j r+|j \}t|}	nd }	|d ur9dd |j D }|||t|||	|rG|dfS d dfS )	NrJ   rV   z4Inlined rules (_rule) cannot use the ?rule modifier.rK   Fc                 S   r_  r   )r   r  r   r   r   r   0  r   z$_make_rule_tuple.<locals>.<listcomp>)r   Ztemplate_source)r   r   r)   r   r!   )
Zmodifiers_treer   r  Zpriority_treer   mr  r   r
  r   r   r   r   _make_rule_tuple  s&   

r  c                   @   s   e Zd ZdddZdS )
Definitionr   Nc                 C   s    || _ || _t|| _|| _d S r   )re  r   rb  r  r  )r   re  r   r  r  r   r   r   r   7  s   

zDefinition.__init__r   N)r   r   r   r   r   r   r   r   r  6  s    r  c                   @   s\  e Zd ZU eed< eeeef  ed< e	eef ed< e	ee
f ed< ee ed< d.dedeeeeef   dee	eef  ddfd	d
Zdd Zdd Zd/ddddZd/ddZdd Zdd Zdd Zd0dededeeegef  ddfdd Zd!d" Zd1d#eed$f d%ee d&e	eef d'eeegef  ddf
d(d)Zd2d*d+Zdefd,d-ZdS )3GrammarBuilderglobal_keep_all_tokensimport_paths
used_files_definitions_ignore_namesFNr   c                 C   s*   || _ |pg | _|pi | _i | _g | _d S r   )r  r  r  r  r  )r   r  r  r  r   r   r   r   F  s
   


zGrammarBuilder.__init__c           	      G   sr   i }t |ddD ]&\}}|dkrdnt|}||d| < d|  |d| < }| |d| < qt|jdi |)	Nr   ry   r   r   )rz   r   r1  Typer   )r   r   titler)   format)	r   re  msgr  r   r   r   ZpostfixZlowercase_typer   r   r   _grammar_errorN  s   zGrammarBuilder._grammar_errorc                 C   sf   |r|d u r
d}|S t |tstdt|f |S |d u r"t }n	t |ts+td| jr1d|_|S )Nr   zBTerminal require a single int as 'options' (e.g. priority), got %sz1Rules require a RuleOptions instance as 'options'T)r   r   r)   r1  r!   r  r   )r   re  r  r   r   r   _check_optionsW  s   


zGrammarBuilder._check_optionsr   )r   c                C   sh   || j v r|s| |d| n	|r| |d| |dr$| |d| t|||| ||| j |< d S )Nz&{Type} '{name}' defined more than oncez+Cannot override a nonexisting {type} {name}__zDNames starting with double-underscore are reserved (Error at {name}))r  r  r   r  r  )r   r   re  r  r  r  r   r   r   r   _defineg  s   

 zGrammarBuilder._definec                 C   s   || j vr| |d| | j | }||jkr| |d| t||jkr+| |d| |jd u r7| |d| |j}t|trD|jdksFJ |j	
d| d S )Nz6Can't extend {type} {name} as it wasn't defined beforezHCannot extend {type} {name} - one is a terminal, while the other is not.z6Cannot extend {type} with different parameters: {name}z,Can't extend {type} {name} - it is abstract.r   r   )r  r  re  rb  r  r   r   r.   r   r   insert)r   r   re  r  r  r  dbaser   r   r   _extends  s   



zGrammarBuilder._extendc                 C   s   t |tr| j| d S t |tsJ |}|jdkrMt|jdkrM|j\}|jdkrMt|jdkrM|j\}|jdkrM|j\}t |trM| j|j	 d S dt| j }| j| t
d|td| j|< d S )Nr   r   r   r   z__IGNORE_%dT)r  )r   r   r  r   r.   r   r   r   r#   r   r  r&   r  )r   Zexp_or_namer   t2r  r   r   r   r   _ignore  s    


zGrammarBuilder._ignorec                 C   s6  t |jdkr|j\}}n|j\}d }t|tr(t|j}|j}tt||}n"t|jd d }|s=|j\}td| |jd }|j|pG|ji}|j	dkrRd }	nD|dkrnzt
jtjd j}
W n tym   d }
Y nw |}
|
rt|
trt|
jt
j|
jd }	nt
j|
d }	nt
jt
jj}	||	|fS )Nr   r   z&Nothing was imported from grammar `%s`r   <string>__main__r   )r   r   r   r.   rb  r  r  r)   r   r   rw  rv  abspathsysmodules__file__r  rm  ro  r  curdir)r   stmtgrammar_nameZ	path_nodeZarg1dotted_pathr  r   r   rs  Z	base_filer   r   r   _unpack_import  s:   





zGrammarBuilder._unpack_importc                    s   |j dkrt|j \}}}}d}n|jd j}d}t|jdkr't|jd nt}|jd }d} d urCt fd	d
|D } |}t| }|||||fS )Nrz   Fr   r   rH  r   r   Tc                 3   s    | ]} |V  qd S r   r   )r   r
  r  r   r   r    r.  z4GrammarBuilder._unpack_definition.<locals>.<genexpr>)	r   r  r   r   r   r   r&   rb  r  )r   r   r  r   r  r  optsre  r   r  r   _unpack_definition  s   
 

z!GrammarBuilder._unpack_definition<?>grammar_textr  r  c              	   C   s  t ||}i }|jD ]:}|jdkrD| ||\}}}	z|| \}
}||
ks.J dd| ||	 W q
 tyC   ||	f||< Y q
w q
| D ]\}\}}	| |||	| qI|jD ]}|jdv rl| j	| 
||  q[|jdkr|j\}| j	| 
||ddi q[|jdkr|j\}| j| 
||  q[|jdkr|d u r| j|j  q[|jd	kr|jD ]$}t|tsJ |t|t}|d u r|j}n||j}| 	||d  qq[|jdkrq[J |dd | j D }t| d S )Nr}   zInconsistent base_path for %s.r6   )r{   rz   r   Tr   r|   r~   Fc                 S   s   i | ]\}}|j r||jqS r   )re  r   )r   r   r  r   r   r   r     s    z/GrammarBuilder.load_grammar.<locals>.<dictcomp>)r  r   r   r  r  updater   r<  	do_importr  r  r  r  r   r%   r#   r   r  r  )r   r  r  r  r   Zimportsr  r  rs  r   Zimport_base_pathZimport_aliasesrX  r   re  r   rO  r   r   r   load_grammar  sX   


	









zGrammarBuilder.load_grammarc                    s8   fdd}t t||  fddj D _d S )Nc                    sD   z j |  }W n ty   g  Y S w |jrg S t|jt|j S r   )r  r   re  r  r   r   r  )r   r  r   r   r   rule_dependencies  s   z8GrammarBuilder._remove_unused.<locals>.rule_dependenciesc                    s   i | ]\}}| v r||qS r   r   )r   kr2  )_usedr   r   r     r  z1GrammarBuilder._remove_unused.<locals>.<dictcomp>)r   r   r  r<  )r   usedr  r   )r  r   r   _remove_unused  s   	zGrammarBuilder._remove_unusedr  .rs  r   r  c              	   C   sx  |sJ t d|||}tjj| t }| j|d ur|gng  tg }|D ]}z/t|r5|||\}	}
n!tj||}	t|	dd}|	 }
W d    n1 sQw   Y  W n	 t
y`   Y q&w t|
}| j|	||krrtd|| j|	< t| j| j| j}||
|	| |t|| |jD ]}|| jv rtd||f q| jjdi |j  d S t|dd J d|f )	Nr  utf8encodingz)Grammar file was changed during importingz5Cannot import '%s' from '%s': Symbol already defined.FzjCouldn't import grammar %s, but a corresponding file was found at a place where lark doesn't search for itr   )r  r  rw  rv  EXTr  stdlib_loadercallableopenreadru  sha256_digestr  getRuntimeErrorr  r  r  r  r   r  r)   r  )r   r  rs  r   r  r  rt  r{  sourceZjoined_pathr~  r-  hgbr   r   r   r   r    s>   



zGrammarBuilder.do_importc                 C   sv  | j  D ]\}}|j}|j}t|D ]!\}}|| j v r$td||f ||d | v r4td||f q|d u r:q|dD ]H}|jd j}|jdd  }	||vr|| j vra| 	|j
d| | t|	t| j | jkrt| j | jt|	}
}| 	|j
d|
|f || q?t|D ]}|| j vr||vr| 	|j
d|| qqt| j | jstd	t| jt| j   d S )
Nz:Template Parameter conflicts with rule %s (in template %s)z0Duplicate Template Parameter %s (in template %s)r   r   r   z5Template '%s' used but not defined (in {type} {name})z]Wrong number of template arguments used for {name} (expected %s, got %s) (in {type2} {name2})z9{Type} '{name}' used but not defined (in {type2} {name2})z8Terminals %s were marked to ignore but were not defined!)r  r<  r  r   r   r)   ra  r   r   r  re  r   r  r   
issupersetr  )r   r   r  r  r  r   r
  tempr   r   r  actualr   r   r   validate>  sB   


zGrammarBuilder.validatec                 C   s   |    g }g }| j D ]-\}}|j|j|j}}}|jr1t|dks'J ||||ff q|||||f qt	||| j
S rI  )r  r  r<  r  r   r  re  r   r   rN  r  )r   r  rO  r   r  r  r  r  r   r   r   build^  s   zGrammarBuilder.build)FNNr  )r  Nr   )r   N)r   r   r   r   rl  r   r	   r   r
   r   r  r   r   r  r  r  r  r  r  r  r  r  r   r  r  rN  r  r   r   r   r   r  =  s&   
 8	
'*5B
$ r  c              	   C   s   |   D ]d\}}d }t|tr0tj|r0t|dd}| }W d    n1 s*w   Y  n!t|trQt	t
 tj| d}W d    n1 sLw   Y  |d u rVqt|}||krhtd|   dS qdS )Nr  r  zutf-8z"File %r changed, rebuilding ParserFT)r<  r   r   rw  rv  existsr  r  rm  r   ru  rx  ry  rz  r  r   info)Zfile_hashesrv  oldr~  r-  currentr   r   r   verify_used_filesm  s&   


r   c                 C   s$   t d|}|| d t|j S )zXReturns a list of paths to the lark grammars imported by the given grammar (recursively)Fr  )r  r  r`  r  keys)grammarr  builderr   r   r   list_grammar_imports  s   
r  c                 C   s$   t ||}|| | | |jfS r   )r  r  r  r  )r  r  r  r  r  r   r   r   r    s   
r  r  c                 C   s6   t jdkrtj| ddd S t| d S )zGet the sha256 digest of a string

    Supports the `usedforsecurity` argument for Python 3.9+ to allow running on
    a FIPS-enabled system.
    )rH  	   r  F)usedforsecurity)r  version_infohashlibsha256encode	hexdigestr  r   r   r   r    s   
r  r  r   )xr  r  Zos.pathrw  r  collectionsr   r   r   rx  astr   
contextlibr   typingr   r   r	   r
   r   r   r   r   utilsr   r   r   r   r   r   r   r   lexerr   r   r   r   r   Zparse_tree_builderr   Zparser_frontendsr   commonr   r    r  r!   r"   r#   r$   r%   r&   r'   r(   
exceptionsr)   r*   r+   r,   r-   r   r.   r/   r   Zvisitorsr0   r1   r2   r3   r4   r  ZIMPORT_PATHSr  r,  r   r  r  r  r   r   r   r   r   r   r   r  r  r!  r+  r3  r5  r:  r;  rK  rM  rN  rm  rn  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r   r  r  r  r   r   r   r   <module>   sR   (( 
	
 !"#(	
!
"()+,./12579;<=ABCDFP -7!3 


-"

,  2