g!9ddlZddlZddlZddlmZddlmZmZmZm Z m Z m Z m Z ddl mZmZejeZejedZee5ZeZdddn #1swxYwYGddZe eZe e eZGdd e ZGd d ZGd d ZGddZ efde e!de!fdZ"dS)N)Template)AnyCallableDictList NamedTupleOptionalTuple)Encoding Tokenizerzvisualizer-styles.csscBeZdZUeed<eed<eed<dededefdZdS) Annotationstartendlabelc0||_||_||_dSN)rrr)selfrrrs W/var/www/html/ai-engine/env/lib/python3.11/site-packages/tokenizers/tools/visualizer.py__init__zAnnotation.__init__s  N)__name__ __module__ __qualname__int__annotations__strrrrrrsZ JJJ HHH JJJcCrrc<eZdZUeeed<eeed<dS) CharStateKeytoken_ixanno_ixN)rrrr rrrrrr r s2sm c]rr cdeZdZUeeed<dZedZedZ de fdZ dS) CharStatechar_ixc0||_d|_g|_dSr)r%r"tokens)rr%s rrzCharState.__init__'s &* !# rcPt|jdkr |jdndS)Nrlenr'rs rr!zCharState.token_ix-s&!$T[!1!1A!5!5t{1~~4?rc2t|jdkS)zJ BPE tokenizers can output more than one token for a char r)r+s r is_multitokenzCharState.is_multitoken1s 4;!##rreturnc8t|j|jS)N)r!r")r r!r"r+s r partition_keyzCharState.partition_key8s#]L    rN) rrrr rrrpropertyr!r.r r1rrrr$r$$s c]$$$ @@X@$$X$  |      rr$ceZdZdS)AlignedN)rrrrrrr4r4?sDrr4c eZdZdZejdejZ ddede de e e ge ffd Zgdfd ed ede e d e efd Zed ed eeeffdZedeed edefdZed eded ed efdZed ed ed efdZed eded ed eefdZdS)EncodingVisualizera Build an EncodingVisualizer Args: tokenizer (:class:`~tokenizers.Tokenizer`): A tokenizer instance default_to_notebook (:obj:`bool`): Whether to render html output in a notebook by default annotation_converter (:obj:`Callable`, `optional`): An optional (lambda) function that takes an annotation in any format and returns an Annotation object z(.{1})?(unk|oov)(.{1})?)flagsTN tokenizerdefault_to_notebookannotation_converterc|r' ddlm}m}n#t$rt dwxYw||_||_||_dS)NrHTMLdisplayzWe couldn't import IPython utils for html display. Are you running in a notebook? You can also pass `default_to_notebook=False` to get back raw HTML )IPython.core.displayr=r> ImportError Exceptionr8r9annotation_coverter)rr8r9r:r=r>s rrzEncodingVisualizer.__init__Vs|   >>>>>>>>>    ##6 #7  s 'text annotationsr/c\|j}||}|r' ddlm}m}n#t$rt dwxYw|j"tt|j|}|j |}t |||}|r|||dS|S)a Build a visualization of the given text Args: text (:obj:`str`): The text to tokenize annotations (:obj:`List[Annotation]`, `optional`): An optional list of annotations of the text. The can either be an annotation class or anything else if you instantiated the visualizer with a converter function default_to_notebook (:obj:`bool`, `optional`, defaults to `False`): If True, will render the html in a notebook. Otherwise returns an html string. Returns: The HTML string if default_to_notebook is False, otherwise (default) returns None and renders the HTML in the notebook Nrr<zeWe couldn't import IPython utils for html display. Are you running in a notebook?) r9r?r=r>r@rArBlistmapr8encoder6_EncodingVisualizer__make_html) rrCrDr9final_default_to_notebookr=r>encodinghtmls r__call__zEncodingVisualizer.__call__ls2%)$<!  *(; % $  >>>>>>>>>   6   # /s4#;[IIJJK>((..!--dHkJJ $  GDDJJ     Ks2ct|dkriSttd|}t|}td|z }|dkrd}d}d}d}i}t |D]}d|d |d |d ||<||z }|S) a Generates a color palette for all the labels in a given set of annotations Args: annotations (:obj:`Annotation`): A list of annotations Returns: :obj:`dict`: A dictionary mapping labels to colors in HSL format rc|jSr)r)xs rz;EncodingVisualizer.calculate_label_colors..s17r @ zhsl(,z%,%)r*setrGrsorted) rDlabels num_labelsh_stepslhcolorsrs rcalculate_label_colorsz)EncodingVisualizer.calculate_label_colorss {  q IS**K8899[[ S:%&& B;;F   F^^  E0100q00A000F5M KAA rconsecutive_chars_listrKc|d}|j|j|j}d|dS|d}|j}|jdz}|||}g} i} |j| d|jr| d|jd zr| d n| d t j|j|j*| d |j|j| d <n| ddd| d} d} | D]\} }| d| d|dz } d| d| d|dS)a Converts a list of "consecutive chars" into a single HTML element. Chars are consecutive if they fall under the same word, token and annotation. The CharState class is a named tuple with a "partition_key" method that makes it easy to compare if two chars are consecutive. Args: consecutive_chars_list (:obj:`List[CharState]`): A list of CharStates that have been grouped together text (:obj:`str`): The original text being processed encoding (:class:`~tokenizers.Encoding`): The encoding returned from the tokenizer Returns: :obj:`str`: The HTML span for a set of consecutive chars rNz(r-tokenz multi-tokenz odd-tokenz even-tokenz special-tokenstokz non-tokenzclass=" "z data-z="z) r%r'r!appendr.r6unk_token_regexsearchjoinitems)rcrCrKfirststokenlastrr span_text css_classes data_itemscssdatakeyvals rconsecutive_chars_to_htmlz,EncodingVisualizer.consecutive_chars_to_htmls2'q) = _U^4FOfNNN N%b) lQsO   > %   w ' ' '" 2""=111~! 1 "";////""<000!1889XYYe""?333%-_U^%D 6"   { + + +4#((;//444"((** + +HC *S**C*** *DD888d88i8888rct|||}|dg}|dj}g}t|}|dj}|3||} | j} || } |d| d| d|ddD] } | j}||kr|t|||| g}||d|3||} | j} || } |d| d| d|}| |dkr|| |t|||| g} |t|||t|} | S)Nrz&r-)rCrKrl) r6%_EncodingVisualizer__make_char_statesr"rbrrmr|r1HTMLBody)rCrKrD char_statescurrent_consecutive_chars prev_anno_ixspanslabel_colors_dict cur_anno_ixannorcolorcsress r __make_htmlzEncodingVisualizer.__make_htmlsA(;;D(KXX %0^$4!"1~- .EEkRR!!n,  "{+DJE%e,E LL`%``W\``` a a aabb/& 1& 1B*Kl** &@@1!!)A.0D)+LL+++*&{3D JE-e4ELL!h%!h!h_d!h!h!hiii&L!!%>q%A%O%O%Q%QQQ)004444 &@@1!!)A.0D))   8 8)! 9     uoo rcdgt|z}t|D]'\}}t|j|jD]}|||<(|S)a Args: text (:obj:`str`): The raw text we want to align to annotations (:obj:`AnnotationList`): A (possibly empty) list of annotations Returns: A list of length len(text) whose entry at index i is None if there is no annotation on charachter i or k, the index of the annotation that covers index i where k is with respect to the list of annotations N)r* enumeraterangerr)rCrDannotation_mapr"ais r__make_anno_mapz"EncodingVisualizer.__make_anno_map<sf#d))+#K00 , ,JGQ17AE** , ,$+q!! ,rct||}dtt|D}t |jD]T\}}||}|8|\}} t|| D]"} || j|#Ut |D]\} } | || _|S)a For each character in the original text, we emit a tuple representing it's "state": * which token_ix it corresponds to * which word_ix it corresponds to * which annotation_ix it corresponds to Args: text (:obj:`str`): The raw text we want to align to annotations (:obj:`List[Annotation]`): A (possibly empty) list of annotations encoding: (:class:`~tokenizers.Encoding`): The encoding returned from the tokenizer Returns: :obj:`List[CharState]`: A list of CharStates, indicating for each char in the text what it's state is c,g|]}t|Sr)r$).0r%s r z9EncodingVisualizer.__make_char_states..js '['['[w '(:(:'['['[r) r6"_EncodingVisualizer__make_anno_maprr*rr'token_to_charsrmr") rCrKrDrrr!rfoffsetsrrrr%r"s r__make_char_statesz%EncodingVisualizer.__make_char_statesQs.,;;D+NN'['[%PSTXPYPYJZJZ'['['[ (99 ; ;OHe--h77G"$ suc**;;AN)00:::: ). 9 9 3 3 GW+2K ( (r)TN)rrr__doc__recompile IGNORECASErnr boolr rrrrrAnnotationListrM staticmethodrrbrr$r r|rIPartialIntListrr~rrrr6r6Cs  !bj!>bmTTTO %)FJ    " 'xz0A'BC     2').2 +++$+&d^ + # ++++ZNtCH~\8A9 $YA9A9A9A9A9\A9F?#???SV???\?Bc>\("""~"Z^_hZi"""\"""rr6childrenr/c>d|}d|d|dS)a[ Generates the full html with css from a list of html spans Args: children (:obj:`List[str]`): A list of strings, assumed to be html elements css_styles (:obj:`str`, `optional`): Optional alternative implementation of the css Returns: :obj:`str`: An HTML string with style markup rkz?
z4
)rp)r css_styles children_texts rrrwsBGGH%%M       r)# itertoolsosrstringrtypingrrrrrr r tokenizersr r pathdirname__file__rp css_filenameopenfreadrxrrrrr r$r4r6rrrrrrsD IIIIIIIIIIIIIIIIII******** '//( # #w||G%<==  T,1 &&((Cj!hsm$:         6        qqqqqqqqh .1tCySs-BBB