§
    Ý˜ºhr  ã                   ó(  — d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ  ej        e¦  «        Zi ddddœ“ddddddddddddœ
“ddddddddddddœ
“ddddddddddddœ
“dddddddddddddd œ“d!dddddddddddœ
“d"dddddddddddddd#œ“d$dddddddddddœ
“d%d&d'd(d)d*œ“d+dddddddddddœ
“d,d-dd.dd/d0œ“d1d2d3d4d5d6d7dd/d8d9dd:œ“d;dddddddd<dd=œ	“d>d-d?d@dAd.d/dBœ“dCdddddddDdEœ“dFdddd/ddGdHdIddJœ	“dKdddddddddLddœ
“dddddddddddMddNœdddddddddddMddNœdddddddddddœ
dOœ¥ZdPdQdRdSdTd&d'd(d)dUdVœ
dWdd&d'd(d)dXœdYœZdZ„ Z G d[„ d\¦  «        Z G d]„ d^e¦  «        Z  G d_„ d`e¦  «        Z! G da„ dbe¦  «        Z" G dc„ dde¦  «        Z# G de„ dfe¦  «        Z$ G dg„ dhe¦  «        Z%i de “de!“die!“d!e!“d"e!“d+e"“d,e#“d$e#“d;e#“d>e#“dCe#“d1e$“dFe#“dKe#“dje%“dke%“dle “dme i¥Z&dnefdo„Z'dS )pz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
é    )ÚarrayN)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramé   ©Ú
AddedToken)ÚGemmaConverterÚGPT2ConverterÚLlamaConverterÚQwen2ConverterÚT5Converter)Úlogging)ÚtqdmÚgeneralÚ
model_typeÚ_model_name_or_path)ÚarchitectureÚnameÚllamaÚmax_position_embeddingsÚnum_hidden_layersÚintermediate_sizeÚhidden_sizeÚhead_dimÚ
rope_thetaÚnum_attention_headsÚnum_key_value_headsÚrms_norm_epsÚ
vocab_size)
Úcontext_lengthÚblock_countÚfeed_forward_lengthÚembedding_lengthúrope.dimension_countúrope.freq_baseúattention.head_countúattention.head_count_kvú attention.layer_norm_rms_epsilonr$   ÚmistralÚqwen2Úqwen2moeÚnum_expertsÚnum_experts_per_tok)r%   r&   r'   r(   r)   r*   r+   r,   r-   r$   Úexpert_countÚexpert_used_countÚqwen3Ú	qwen3_moe)r%   r&   r'   r(   r)   r*   úattention.key_lengthr+   r,   r-   r$   r3   r4   ÚfalconÚ	tokenizerÚbos_token_idÚeos_token_idÚunk_token_idÚpad_token_id)úggml.bos_token_idúggml.eos_token_idúggml.unknown_token_idúggml.padding_token_idÚphi3ÚbloomÚn_layerÚn_headÚlayer_norm_epsilon)r&   r(   r+   r$   úattention.layer_norm_epsilonÚt5Ún_positionsÚ
num_layersÚd_ffÚd_modelÚd_kvÚ	num_headsÚrelative_attention_num_bucketsÚdecoder_start_token_id)r%   r&   r'   r(   r7   r+   r,   rG   z attention.relative_buckets_countrP   r$   ÚstablelmÚlayer_norm_eps)	r%   r&   r'   r(   r)   r+   r,   rG   r$   Úgpt2Ún_ctxÚn_embdr'   )r&   r%   r(   r'   r+   rG   Ú
starcoder2Únorm_epsilon)r&   r%   r(   r'   r+   r,   rG   ÚmambaÚconv_kernelÚ
state_sizeÚtime_step_rank)	r$   r%   r(   r-   r&   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizeÚnemotronÚnorm_epsÚsliding_window)r%   r&   r'   r(   r)   r*   r7   r+   r,   r-   zattention.sliding_windowr$   )Úgemma2Úgemma3ÚdeciÚtokenizer_typeÚtokensÚscoresÚ
token_typeÚmergesÚadd_prefix_space)
ú
ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr>   r?   r@   rA   zggml.add_space_prefixÚchat_template)ri   rh   r>   r?   r@   rA   )r9   Útokenizer_configc                 ó  — t          |t          ¦  «        s|g}t          |¦  «        dk    r|d         }d }n |d         dk    rt          d¦  «        ‚|\  }}|dv rt	          | d         ¦  «        } nŽ|dv rt          | d         ¦  «        } nt|dv rt          | d         ¦  «        } nZ|dv rBt          d	t          | ¦  «        ¦  «                             ¦   «          	                    ¦   «         } n|d
v rt          | |¦  «        } | S )Né   r   é	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rl   r   é   é   é   é
   é   )é   é   )é   )é   ÚB)rm   )Ú
isinstanceÚlistÚlenÚ
ValueErrorÚintÚfloatÚboolr   ÚtobytesÚdecodeÚ_gguf_parse_value)Ú_valueÚ	data_typeÚarray_data_types      úf/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/transformers/integrations/ggml.pyr�   r�   #  s  € Ý�i¥Ñ&Ô&ð  Ø�Kˆ	Ý
ˆ9�~„~˜ÒÐØ˜a”Lˆ	Øˆˆà�QŒ<˜1ÒÐÝÐoÑpÔpÐpØ%.Ñ"ˆ	�?àÐ.Ð.Ð.Ý�V˜A”Y‘”ˆˆØ	�gÐ	Ð	Ý�v˜a”yÑ!Ô!ˆˆØ	�cÐ	Ð	Ý�f˜Q”i‘”ˆˆØ	�cÐ	Ð	Ý�s�D ™LœLÑ)Ô)×1Ò1Ñ3Ô3×:Ò:Ñ<Ô<ˆˆØ	�cÐ	Ð	Ý" 6¨?Ñ;Ô;ˆØ€Mó    c                   ó   — e Zd Zd„ ZdS )ÚGGUFTokenizerSkeletonc                 ó\  ‡‡— |                      ¦   «         D ]\  }}t          | ||¦  «         Œt          | d¦  «        �sBt          | d¦  «        rt          | d¦  «        st          d¦  «        ‚| j        }| j        Šˆfd„t          |¦  «        D ¦   «         Št                               d¦  «         g }t          ‰                      ¦   «         ¦  «        D ]…\  }}g }t          dt          |¦  «        ¦  «        D ]6}	|d |	…         ||	d …         }}
|
|v r||v r|                     |
||f¦  «         Œ7t          |ˆfd„d	¬
¦  «        }|                     |¦  «         Œ†t          |d„ d	¬
¦  «        }d„ |D ¦   «         }|| _        nVd„ | j        D ¦   «         | _        t          | d¦  «        s0d„ t          t          | j        ¦  «        ¦  «        D ¦   «         | _        t          | d¦  «        sg | _        t          | d¦  «        sd | _        t          | d¦  «        r| j        €| j        | _        d S d S d S )Nrf   rc   rd   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.c                 ó(   •— i | ]\  }}|‰|         “ŒS © r‹   )Ú.0ÚiÚtrd   s      €r…   ú
<dictcomp>z2GGUFTokenizerSkeleton.__init__.<locals>.<dictcomp>G  s#   ø€ Ð@Ð@Ð@¡d a¨�Q˜˜qœ	Ð@Ð@Ð@r†   z:Merges were not in checkpoint, building merges on the fly.rl   c                 ó<   •— ‰| d                  ‰| d                  fS )Nr   rl   r‹   )ÚxÚvocabs    €r…   ú<lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>Q  s   ø€ °U¸1¸Q¼4´[À%ÈÈ!ÌÄ+Ð4N€ r†   T)ÚkeyÚreversec                 ó   — | d         S )Nr   r‹   )Úvals    r…   r“   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>S  s
   € °C¸´F€ r†   c                 ó.   — g | ]}|d          |d         f‘ŒS )r   rl   r‹   )rŒ   r—   s     r…   ú
<listcomp>z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>T  s%   € Ð9Ð9Ð9¨3�s˜1”v˜s 1œvÐ&Ð9Ð9Ð9r†   c                 óR   — g | ]$}t          |                     d ¦  «        ¦  «        ‘Œ%S )ú )ÚtupleÚsplit)rŒ   Úmerges     r…   r™   z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>W  s,   € ÐLÐLÐL°u�5 §¢¨SÑ!1Ô!1Ñ2Ô2ÐLÐLÐLr†   c                 ó   — g | ]}d ‘ŒS ©Nr‹   )rŒ   Ú_s     r…   r™   z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>Y  s   € ÐEÐEÐE¨˜tÐEÐEÐEr†   Úadded_tokensr<   Úunknown_token_id)ÚitemsÚsetattrÚhasattrr{   rc   rd   Ú	enumerateÚloggerÚwarningr   Úrangerz   ÚappendÚsortedÚextendrf   r¢   r<   r£   )ÚselfÚdict_ÚkÚvrc   rf   rž   Úpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrd   r’   s               @@r…   Ú__init__zGGUFTokenizerSkeleton.__init__<  s…  øø€ Ø—K’K‘M”Mð 	 ð 	 ‰DˆAˆqÝ�D˜!˜QÑÔÐÐå�t˜XÑ&Ô&ñ 	FÝ˜4 Ñ*Ô*ð µ'¸$ÀÑ2IÔ2Ið Ý Ørñô ð ð ”[ˆFØ”[ˆFØ@Ð@Ð@Ð@­i¸Ñ.?Ô.?Ð@Ñ@Ô@ˆEå�NŠNÐWÑXÔXÐXØˆFÝ&*¨5¯;ª;©=¬=Ñ&9Ô&9ð %ð %Ñ"��{Ø�Ý" 1¥c¨%¡j¤jÑ1Ô1ð Fð F�EØ',¨V¨e¨V¤}°e¸E¸F¸F´m˜W�GØ &Ð(Ð(¨W¸Ð->Ð->ØŸš g¨w¸Ð%DÑEÔEÐEøÝ˜uÐ*NÐ*NÐ*NÐ*NÐX\Ð]Ñ]Ô]�Ø—’˜eÑ$Ô$Ð$Ð$Ý˜FÐ(:Ð(:ÀDÐIÑIÔIˆFØ9Ð9°&Ð9Ñ9Ô9ˆFØ ˆDŒKˆKàLÐLÀÄÐLÑLÔLˆDŒKÝ˜4 Ñ*Ô*ð FØEÐE­Uµ3°t´{Ñ3CÔ3CÑ-DÔ-DÐEÑEÔE�”å�t˜^Ñ,Ô,ð 	#Ø "ˆDÔå�t˜^Ñ,Ô,ð 	%Ø $ˆDÔõ �4Ð+Ñ,Ô,ð 	6°Ô1BÐ1JØ $Ô 5ˆDÔÐÐð	6ð 	6Ð1JÐ1Jr†   N)Ú__name__Ú
__module__Ú__qualname__r·   r‹   r†   r…   rˆ   rˆ   ;  s#   € € € € € ð'6ð '6ð '6ð '6ð '6r†   rˆ   c                   ó2   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ ZdS )ÚGGUFLlamaConverterc                 ó’   — t          |¦  «        | _        | j        | _        i | _        t	          | j        dd¦  «        dk    | _        d S )Nrb   r   )rˆ   ÚprotoÚoriginal_tokenizerÚadditional_kwargsÚgetattrÚis_llama_3_tokenizer©r®   Útokenizer_dicts     r…   r·   zGGUFLlamaConverter.__init__g  sF   € Ý*¨>Ñ:Ô:ˆŒ
Ø"&¤*ˆÔØ!#ˆÔÝ$+¨D¬JÐ8HÈ'Ñ$RÔ$RÐV]Ò$]ˆÔ!Ð!Ð!r†   c                 óP   — t          t          |j        |j        ¦  «        ¦  «        S r    ©ry   Úziprc   rd   ©r®   r¾   s     r…   r’   zGGUFLlamaConverter.vocabm  ó   € Ý•C˜œ e¤lÑ3Ô3Ñ4Ô4Ð4r†   c                 ó   — |j         S r    ©rf   rÈ   s     r…   rf   zGGUFLlamaConverter.mergesp  ó
   € ØŒ|Ðr†   c           	      ó(  — |                       | j        ¦  «        }|                      | j        ¦  «        }d„ t          |¦  «        D ¦   «         }|j        �|j        |j                 nd }t          |dd ¦  «        �|j        |j                 nd }t          |dd ¦  «        �|j        |j                 nd }t          t          |||dd¬¦  «        ¦  «        }g }	t          | j        d¦  «        sv|�%|	                     t          |dd¬¦  «        ¦  «         |�%|	                     t          |dd¬¦  «        ¦  «         |�%|	                     t          |dd¬¦  «        ¦  «         ntt          j        t          j        | j        j        ¦  «        d	k    ¦  «        d
         }
|
D ]7}|	                     t          | j        j        |         dd¬¦  «        ¦  «         Œ8t#          |	¦  «        d
k    r|                     |	¦  «         t#          | j        j        ¦  «        d
k    r)|                     d„ | j        j        D ¦   «         ¦  «         || j        d<   || j        d<   || j        d<   | j        r*d | j        d<   d| j        d<   d| j        d<   d| j        _        |S )Nc                 ó    — i | ]\  }\  }}||“ŒS r‹   r‹   ©rŒ   r�   ÚwordÚ_scores       r…   r�   z0GGUFLlamaConverter.tokenizer.<locals>.<dictcomp>v  ó#   € ÐNÐNÐNÑ!2 ¡N T¨6�T˜1ÐNÐNÐNr†   r:   r;   T)Ú	unk_tokenÚfuse_unkÚbyte_fallbackre   F©Ú
normalizedÚspecialrn   r   c                 ó2   — g | ]}t          |d d ¬¦  «        ‘ŒS )FrÖ   r   )rŒ   Úadded_tokens     r…   r™   z0GGUFLlamaConverter.tokenizer.<locals>.<listcomp>�  s'   € ÐuÐuÐuÈk•˜K°EÀ5ÐIÑIÔIÐuÐuÐur†   rÓ   Ú	eos_tokenÚ	bos_tokenrg   Úclean_up_tokenization_spacesÚlegacy)r’   r¾   rf   r§   r<   rc   rÁ   r:   r   r	   r¦   r«   r   ÚnpÚwherer   re   rz   Úadd_special_tokensr¢   Ú
add_tokensrÀ   rÂ   r¿   rÞ   )r®   r¾   Úvocab_scoresrf   Ú	bpe_vocabrÓ   rÜ   rÛ   r9   Úspecial_tokensÚspecial_tokens_idxÚidxs               r…   r9   zGGUFLlamaConverter.tokenizers  sÅ  € Ø—z’z $¤*Ñ-Ô-ˆØ—’˜TœZÑ(Ô(ˆØNÐNµiÀÑ6MÔ6MÐNÑNÔNˆ	à8=Ô8JÐ8V�E”L Ô!3Ô4Ð4Ð\`ˆ	Ý8?ÀÀ~ÐW[Ñ8\Ô8\Ð8h�E”L Ô!3Ô4Ð4Ðnrˆ	Ý8?ÀÀ~ÐW[Ñ8\Ô8\Ð8h�E”L Ô!3Ô4Ð4Ðnrˆ	åÝØØØ#ØØ"ðñ ô ñ
ô 
ˆ	ð ˆå�t”z <Ñ0Ô0ð 	jØÐ$Ø×%Ò%¥j°ÀuÐVZÐ&[Ñ&[Ô&[Ñ\Ô\Ð\àÐ$Ø×%Ò%¥j°ÀuÐVZÐ&[Ñ&[Ô&[Ñ\Ô\Ð\àÐ$Ø×%Ò%¥j°ÀuÐVZÐ&[Ñ&[Ô&[Ñ\Ô\Ð\øõ "$¤­"¬(°4´:Ô3HÑ*IÔ*IÈQÒ*NÑ!OÔ!OÐPQÔ!RÐà)ð jð j�Ø×%Ò%¥j°´Ô1BÀ3Ô1GÐTYÐcgÐ&hÑ&hÔ&hÑiÔiÐiÐiåˆ~ÑÔ !Ò#Ð#Ø×(Ò(¨Ñ8Ô8Ð8åˆtŒzÔ&Ñ'Ô'¨1Ò,Ð,Ø× Ò ØuÐuÐ]aÔ]gÔ]tÐuÑuÔuñô ð ð /8ˆÔ˜{Ñ+Ø.7ˆÔ˜{Ñ+Ø.7ˆÔ˜{Ñ+àÔ$ð 	3Ø9=ˆDÔ"Ð#5Ñ6ØEIˆDÔ"Ð#AÑBà/4ˆDÔ" 8Ñ,Ø-2ˆDÔ#Ô*àÐr†   c                 ó  — t          j        ¦   «         t          j        ¦   «         t          j        dd¦  «        g}| j        r|t          j        ddd¬¦  «        gz  }|r|t          j        dd¬¦  «        gz  }t          j        |¦  «        S )Nõ   â–�r›   FT©rg   Útrim_offsetsÚ	use_regexrl   ©ÚcontentÚleft)r   ÚByteFallbackÚFuseÚReplacerÂ   Ú	ByteLevelÚStripÚSequence©r®   Úreplacementrg   Úsequences       r…   ÚdecoderzGGUFLlamaConverter.decoder­  s’   € åÔ!Ñ#Ô#ÝŒM‰OŒOÝÔ˜U CÑ(Ô(ð
ˆð Ô$ð 	iØ�Ô+¸UÐQVÐbfÐgÑgÔgÐhÑhˆHàð 	>Ø�œ°¸!Ð<Ñ<Ô<Ð=Ñ=ˆHÝÔ  Ñ*Ô*Ð*r†   c                 óð  — |                       | j        ¦  «        }|                      | j        ¦  «        }|�||_        d}d}t          | j        d¦  «        r| j        j        }|                      ||¦  «        }|�||_        |                      ||¦  «        |_        |                      ¦   «         }|r||_        | j	        r5t          j        ddd¬¦  «        |_        t          j        g ¦  «        |_        |S )Nré   Trg   Frê   )r9   r¾   Ú
normalizerr¦   r¿   rg   Úpre_tokenizerrù   Úpost_processorrÂ   r   ró   r   rõ   )r®   r9   rû   r÷   rg   rü   rý   s          r…   Ú	convertedzGGUFLlamaConverter.converted»  s  € à—N’N 4¤:Ñ.Ô.ˆ	ð —_’_ T¤ZÑ0Ô0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÝ�4Ô*Ð,>Ñ?Ô?ð 	HØ#Ô6ÔGÐà×*Ò*¨;Ð8HÑIÔIˆØÐ$Ø&3ˆIÔ#à ŸLšL¨Ð6FÑGÔGˆ	ÔØ×,Ò,Ñ.Ô.ˆØð 	6Ø'5ˆIÔ$ð Ô$ð 	<Ý&4Ô&>Ø!&°UÀdð'ñ 'ô 'ˆIÔ#õ
 $/Ô#7¸Ñ#;Ô#;ˆIÔ àÐr†   N)	r¸   r¹   rº   r·   r’   rf   r9   rù   rþ   r‹   r†   r…   r¼   r¼   f  sr   € € € € € ð^ð ^ð ^ð5ð 5ð 5ðð ð ð8ð 8ð 8ðt+ð +ð +ð!ð !ð !ð !ð !r†   r¼   c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFQwen2Converterc                 ó<   — t          |¦  «        | _        i | _        d S r    ©rˆ   r¿   rÀ   rÃ   s     r…   r·   zGGUFQwen2Converter.__init__à  ó    € Ý"7¸Ñ"GÔ"GˆÔØ!#ˆÔÐÐr†   Úreturnc           	      ó:  •— d„ t          | j        j        ¦  «        D ¦   «         }| j        j        }t	          ¦   «                              ||¦  «        }|                     t          ddd¬¦  «        t          ddd¬¦  «        t          ddd¬¦  «        g¦  «         |S )Nc                 ó   — i | ]\  }}||“Œ	S r‹   r‹   ©rŒ   r�   rÐ   s      r…   r�   z0GGUFQwen2Converter.converted.<locals>.<dictcomp>å  ó   € ÐRÐRÐR™W˜Q ��qÐRÐRÐRr†   ú<|endoftext|>FTrÖ   z<|im_start|>z
<|im_end|>)r§   r¿   rc   rf   Úsuperrþ   rá   r   ©r®   r’   rf   r9   Ú	__class__s       €r…   rþ   zGGUFQwen2Converter.convertedä  s£   ø€ ØRÐR­	°$Ô2IÔ2PÑ(QÔ(QÐRÑRÔRˆØÔ(Ô/ˆÝ‘G”G×%Ò% e¨VÑ4Ô4ˆ	à×$Ò$å˜?°uÀdÐKÑKÔKÝ˜>°eÀTÐJÑJÔJÝ˜<°EÀ4ÐHÑHÔHðñ	
ô 	
ð 	
ð Ðr†   ©r¸   r¹   rº   r·   r   rþ   Ú__classcell__©r  s   @r…   r   r   ß  sS   ø€ € € € € ð$ð $ð $ð˜9ð ð ð ð ð ð ð ð ð ð r†   r   c                   ó8   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zdefd„Z	dS )	ÚGGUFPhi3Converterc                 óT   — t          |¦  «        | _        | j        | _        i | _        d S r    ©rˆ   r¾   r¿   rÀ   rÃ   s     r…   r·   zGGUFPhi3Converter.__init__ô  s)   € Ý*¨>Ñ:Ô:ˆŒ
Ø"&¤*ˆÔØ!#ˆÔÐÐr†   c                 óP   — t          t          |j        |j        ¦  «        ¦  «        S r    rÆ   rÈ   s     r…   r’   zGGUFPhi3Converter.vocabù  rÉ   r†   c                 ó   — |j         S r    rË   rÈ   s     r…   rf   zGGUFPhi3Converter.mergesü  rÌ   r†   c                 óÌ  — |                       | j        ¦  «        }|                      | j        ¦  «        }d„ t          |¦  «        D ¦   «         }t	          t          ||¦  «        ¦  «        }|                     t          ddddd¬¦  «        t          ddd¬¦  «        t          dddd¬	¦  «        t          d
ddd¬	¦  «        t          dddd¬	¦  «        t          dddd¬	¦  «        t          dddd¬	¦  «        t          dddd¬	¦  «        t          dddd¬	¦  «        t          dddd¬	¦  «        t          dddd¬	¦  «        t          dddd¬	¦  «        g¦  «         |j        �|j	        |j                 nd | j
        d<   |j        �|j	        |j                 nd | j
        d<   |j        �|j	        |j                 nd | j
        d<   |j        �|j	        |j                 nd | j
        d<   |S )Nc                 ó    — i | ]\  }\  }}||“ŒS r‹   r‹   rÏ   s       r…   r�   z/GGUFPhi3Converter.tokenizer.<locals>.<dictcomp>  rÒ   r†   ú</s>TF)ÚrstripÚlstripr×   rØ   r	  rÖ   z<|assistant|>)r  r×   rØ   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>rÓ   rÛ   rÜ   Ú	pad_token)r’   r¾   rf   r§   r   r	   rá   r   r<   rc   rÀ   r;   r:   r=   )r®   r¾   rã   rf   rä   r9   s         r…   r9   zGGUFPhi3Converter.tokenizerÿ  s&  € Ø—z’z $¤*Ñ-Ô-ˆØ—’˜TœZÑ(Ô(ˆØNÐNµiÀÑ6MÔ6MÐNÑNÔNˆ	å�c )¨VÑ4Ô4Ñ5Ô5ˆ	à×$Ò$å˜6¨$°uÈÐX\Ð]Ñ]Ô]Ý˜?°uÀdÐKÑKÔKÝ˜?°4ÀEÐSWÐXÑXÔXÝÐ-°dÀuÐVZÐ[Ñ[Ô[ÝÐ-°dÀuÐVZÐ[Ñ[Ô[ÝÐ-°dÀuÐVZÐ[Ñ[Ô[ÝÐ-°dÀuÐVZÐ[Ñ[Ô[Ý˜<°ÀÐPTÐUÑUÔUÝ˜9¨T¸eÈTÐRÑRÔRÝÐ-°dÀuÐVZÐ[Ñ[Ô[ÝÐ-°dÀuÐVZÐ[Ñ[Ô[Ý˜:¨d¸uÈdÐSÑSÔSðñ	
ô 	
ð 	
ð$ 16Ô0BÐ0NˆEŒL˜Ô+Ô,Ð,ÐTXð 	Ô˜{Ñ+ð 16Ô0BÐ0NˆEŒL˜Ô+Ô,Ð,ÐTXð 	Ô˜{Ñ+ð 16Ô0BÐ0NˆEŒL˜Ô+Ô,Ð,ÐTXð 	Ô˜{Ñ+ð 16Ô0BÐ0NˆEŒL˜Ô+Ô,Ð,ÐTXð 	Ô˜{Ñ+ð Ðr†   c                 óÖ   — t          j        ¦   «         t          j        ¦   «         t          j        |d¦  «        g}|r|t          j        dd¬¦  «        gz  }t          j        |¦  «        S )Nr›   rl   rí   )r   rð   rñ   rò   rô   rõ   rö   s       r…   rù   zGGUFPhi3Converter.decoder&  se   € åÔ!Ñ#Ô#ÝŒM‰OŒOÝÔ˜[¨#Ñ.Ô.ð
ˆð ð 	>Ø�œ°¸!Ð<Ñ<Ô<Ð=Ñ=ˆHÝÔ  Ñ*Ô*Ð*r†   r  c                 óº   — |                       | j        ¦  «        }d}d}t          | j        d¦  «        r| j        j        }|                      ||¦  «        |_        |S )Nré   Trg   )r9   r¾   r¦   r¿   rg   rù   )r®   r9   r÷   rg   s       r…   rþ   zGGUFPhi3Converter.converted1  s`   € Ø—N’N 4¤:Ñ.Ô.ˆ	àˆØÐÝ�4Ô*Ð,>Ñ?Ô?ð 	HØ#Ô6ÔGÐà ŸLšL¨Ð6FÑGÔGˆ	ÔàÐr†   N)
r¸   r¹   rº   r·   r’   rf   r9   rù   r   rþ   r‹   r†   r…   r  r  ó  s{   € € € € € ð$ð $ð $ð
5ð 5ð 5ðð ð ð%ð %ð %ðN	+ð 	+ð 	+ð
˜9ð 
ð 
ð 
ð 
ð 
ð 
r†   r  c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFGPTConverterc                 ó<   — t          |¦  «        | _        i | _        d S r    r  rÃ   s     r…   r·   zGGUFGPTConverter.__init__?  r  r†   r  c                 óª   •— d„ t          | j        j        ¦  «        D ¦   «         }| j        j        }t	          ¦   «                              ||¦  «        }|S )Nc                 ó   — i | ]\  }}||“Œ	S r‹   r‹   r  s      r…   r�   z.GGUFGPTConverter.converted.<locals>.<dictcomp>D  r  r†   )r§   r¿   rc   rf   r
  rþ   r  s       €r…   rþ   zGGUFGPTConverter.convertedC  sN   ø€ ØRÐR­	°$Ô2IÔ2PÑ(QÔ(QÐRÑRÔRˆØÔ(Ô/ˆÝ‘G”G×%Ò% e¨VÑ4Ô4ˆ	ØÐr†   r  r  s   @r…   r  r  >  sS   ø€ € € € € ð$ð $ð $ð˜9ð ð ð ð ð ð ð ð ð ð r†   r  c                   ó2   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„ZdS )ÚGGUFT5Converterc                 ó°   — dg|d<   t          |¦  «        | _        d„ t          | j        j        ¦  «        D ¦   «         | _        | j        | _        i | _        d S )Nú
dummy textrf   c                 ó   — i | ]\  }}||“Œ	S r‹   r‹   )rŒ   r±   r°   s      r…   r�   z,GGUFT5Converter.__init__.<locals>.<dictcomp>P  s   € ÐGÐGÐG¡$ ! Q˜˜AÐGÐGÐGr†   )rˆ   r¾   r§   rc   Útoken2idr¿   rÀ   rÃ   s     r…   r·   zGGUFT5Converter.__init__K  sW   € à$0 >ˆ�xÑ å*¨>Ñ:Ô:ˆŒ
ØGÐG­)°D´JÔ4EÑ*FÔ*FÐGÑGÔGˆŒØ"&¤*ˆÔØ!#ˆÔÐÐr†   c                 óP   — t          t          |j        |j        ¦  «        ¦  «        S r    rÆ   rÈ   s     r…   r’   zGGUFT5Converter.vocabT  rÉ   r†   c                 óð   — t          | j        dd¦  «        r_g }t          | j        dd¦  «        r|t          j        d¬¦  «        gz  }|t          j        dd¬¦  «        gz  }t          j        |¦  «        S d S )NrÞ   Trg   ré   )Úprependr›   )Úpatternrî   )rÁ   r¿   r   ÚPrependrò   rõ   )r®   r¾   rø   s      r…   rû   zGGUFT5Converter.normalizerW  s…   € Ý�4Ô*¨H°dÑ;Ô;ð 	2ØˆHÝ�tÔ.Ð0BÀDÑIÔIð AØ�[Ô0¸Ð?Ñ?Ô?Ð@Ñ@�Ø�Ô,°SÀ%ÐHÑHÔHÐIÑIˆHÝÔ'¨Ñ1Ô1Ð1Øˆtr†   c                 óT   — t          j        ddgg d¢d| j        d         fg¬¦  «        S )Nú$Ar  )r/  r  z$Br  )ÚsingleÚpairrå   )r   ÚTemplateProcessingr(  )r®   s    r…   rý   zGGUFT5Converter.post_processor`  s@   € ÝÔ,Ø˜&�>Ø-Ð-Ð-à˜œ vÔ.Ð/ðð
ñ 
ô 
ð 	
r†   r  c                 óÊ  — |                       | j        ¦  «        }t          t          || j        j        d¬¦  «        ¦  «        }|                      | j        ¦  «        }|�||_        d}d}t          | j        d¦  «        r| j        j        }|  	                    ||¦  «        }|�||_	        |  
                    ||¦  «        |_
        |                      ¦   «         }|r||_        |S )NF©Úunk_idrÕ   ré   Trg   )r’   r¾   r   r
   r<   rû   r¦   r¿   rg   rü   rù   rý   )r®   rã   r9   rû   r÷   rg   rü   rý   s           r…   rþ   zGGUFT5Converter.convertedi  s÷   € Ø—z’z $¤*Ñ-Ô-ˆÝÝØØ”zÔ.Ø#ðñ ô ñ
ô 
ˆ	ð —_’_ T¤ZÑ0Ô0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÝ�4Ô*Ð,>Ñ?Ô?ð 	HØ#Ô6ÔGÐà×*Ò*¨;Ð8HÑIÔIˆØÐ$Ø&3ˆIÔ#à ŸLšL¨Ð6FÑGÔGˆ	ÔØ×,Ò,Ñ.Ô.ˆØð 	6Ø'5ˆIÔ$àÐr†   N)	r¸   r¹   rº   r·   r’   rû   rý   r   rþ   r‹   r†   r…   r$  r$  J  sk   € € € € € ð$ð $ð $ð5ð 5ð 5ðð ð ð
ð 
ð 
ð˜9ð ð ð ð ð ð r†   r$  c                   ó2   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„ZdS )ÚGGUFGemmaConverterc                 ó`   — dg|d<   t          |¦  «        | _        | j        | _        i | _        d S )Nr&  rf   r  rÃ   s     r…   r·   zGGUFGemmaConverter.__init__Š  s5   € à$0 >ˆ�xÑ å*¨>Ñ:Ô:ˆŒ
Ø"&¤*ˆÔØ!#ˆÔÐÐr†   c                 óx  — t          t          |j        |j        ¦  «        ¦  «        }g }|D ]�\  }}|dk    r|                     d|f¦  «         Œ#d|v rOt          |                     ¦   «         ¦  «        dk    r*dt          |¦  «        z  }|                     ||f¦  «         Œv|                     ||f¦  «         ŒŽ|S )Nz<0x09>ú	r›   r   ré   )ry   rÇ   rc   rd   r«   rz   Ústrip)r®   r¾   Úoriginal_vocabÚupdated_vocabÚtokenÚscoreÚunderscoress          r…   r’   zGGUFGemmaConverter.vocab’  sË   € Ý�c %¤,°´Ñ=Ô=Ñ>Ô>ˆØˆà*ð 	5ð 	5‰LˆE�5Ø˜Ò Ð Ø×$Ò$ d¨E ]Ñ3Ô3Ð3Ð3Ø˜��¥# e§k¢k¡m¤mÑ"4Ô"4¸Ò"9Ð"9Ø#¥c¨%¡j¤jÑ0�Ø×$Ò$ k°5Ð%9Ñ:Ô:Ð:Ð:à×$Ò$ e¨U ^Ñ4Ô4Ð4Ð4àÐr†   c                 ó,   — t          j        dd¦  «        S )Nr›   ré   )r   rò   rÈ   s     r…   rû   zGGUFGemmaConverter.normalizer¡  s   € ÝÔ" 3¨Ñ.Ô.Ð.r†   c                 óÖ   — t          j        dd¦  «        t          j        ¦   «         t          j        ¦   «         g}|r|t          j        dd¬¦  «        gz  }t          j        |¦  «        S )Nré   r›   rl   rí   )r   rò   rð   rñ   rô   rõ   rö   s       r…   rù   zGGUFGemmaConverter.decoder¤  se   € åÔ˜U CÑ(Ô(ÝÔ!Ñ#Ô#ÝŒM‰OŒOð
ˆð ð 	>Ø�œ°¸!Ð<Ñ<Ô<Ð=Ñ=ˆHÝÔ  Ñ*Ô*Ð*r†   r  c                 óš  — |                       | j        ¦  «        }t          t          || j        j        | j        ¬¦  «        ¦  «        }|                      | j        ¦  «        }|�||_        d}d}t          | j        d¦  «        r| j        j	        }|  
                    ||¦  «        |_
        |                      ||¦  «        }|�||_        |S )Nr4  ré   Trg   )r’   r¾   r   r
   r<   Úhandle_byte_fallbackrû   r¦   r¿   rg   rù   rü   )r®   rã   r9   rû   r÷   rg   rü   s          r…   rþ   zGGUFGemmaConverter.converted¯  sÙ   € Ø—z’z $¤*Ñ-Ô-ˆÝÝØØ”zÔ.Ø"Ô7ðñ ô ñ
ô 
ˆ	ð —_’_ T¤ZÑ0Ô0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÝ�4Ô*Ð,>Ñ?Ô?ð 	HØ#Ô6ÔGÐà ŸLšL¨Ð6FÑGÔGˆ	ÔØ×*Ò*¨;Ð8HÑIÔIˆØÐ$Ø&3ˆIÔ#àÐr†   N)	r¸   r¹   rº   r·   r’   rû   rù   r   rþ   r‹   r†   r…   r7  r7  ‰  sk   € € € € € ð$ð $ð $ðð ð ð/ð /ð /ð	+ð 	+ð 	+ð˜9ð ð ð ð ð ð r†   r7  Ú	qwen2_moer_   Úgemma3_textra   Údecilmr  c                 ój   — | }t          |         |¦  «        }|                     ¦   «         }||j        fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )ÚGGUF_TO_FAST_CONVERTERSrþ   rÀ   )r   rÄ   Útokenizer_class_nameÚ	converterÚfast_tokenizers        r…   Úconvert_gguf_tokenizerrM  à  s<   € ð (ÐÝ'Ð(<Ô=¸nÑMÔM€IØ×(Ò(Ñ*Ô*€NØ˜9Ô6Ð6Ð6r†   )(Ú__doc__r   Únumpyrß   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr	   r
   Ú r   Úconvert_slow_tokenizerr   r   r   r   r   Úutilsr   Úutils.loggingr   Ú
get_loggerr¸   r¨   ÚGGUF_CONFIG_MAPPINGÚGGUF_TOKENIZER_MAPPINGr�   rˆ   r¼   r   r  r  r$  r7  rI  rM  r‹   r†   r…   ú<module>rY     s~  ðð ð ð
 Ð Ð Ð Ð Ð à Ð Ð Ð Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ *Ð *Ð *Ð *Ð *Ð *Ð *Ð *à Ð Ð Ð Ð Ð Ø oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oÐ oØ Ð Ð Ð Ð Ð Ø  Ð  Ð  Ð  Ð  Ð  ð 
ˆÔ	˜HÑ	%Ô	%€ðeØØ$Ø%ðð ðeð
 Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ðð ðeð$ Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ðð ð%eð> Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ðð ð?eðV Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø%Ø2ðð ðWeðr Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ðð ðseðJ Ø3Ø*Ø2Ø)Ø $Ø&Ø *Ø 5Ø#8Ø,:Ø"Ø%Ø2ðð ðKeðh Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ðð ðieð@ Ø+Ø+Ø!/Ø!/ð	ð ðAeðL Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ðð ðMeðd Ø Ø)Ø (Ø"Ø(<ðð ðeeðr 	Ø'Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ðð ðseðL Ø3Ø*Ø2Ø)Ø $Ø 5Ø#8Ø(8Ø"ð
ð 
ðMeðb Ø Ø!Ø$Ø4Ø (Ø(<ðð ðceðr Ø*Ø3Ø)Ø2Ø 5Ø#8Ø(6ðð ðseðD Ø"Ø3Ø)Ø,@Ø*Ø(Ø&Ø.Ø-ð
ð 
ðEeðZ Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,6Ø"ðð ð[eðt 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ðð ð" 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ðð ð" 4Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ðð ðseð eð eÐ ðR 'ØØØ'ØØ+Ø+Ø!/Ø!/Ø!3ðð ð )Ø"Ø+Ø+Ø!/Ø!/ðð ðð Ð ð0ð ð ð0(6ð (6ð (6ð (6ð (6ñ (6ô (6ð (6ðVvð vð vð vð v˜ñ vô vð vðrð ð ð ð ˜ñ ô ð ð(Hð Hð Hð Hð H˜ñ Hô Hð HðV	ð 	ð 	ð 	ð 	�}ñ 	ô 	ð 	ð<ð <ð <ð <ð <�kñ <ô <ð <ð~>ð >ð >ð >ð >˜ñ >ô >ð >ðBØÐðàÐðð Ð#ðð Ðð	ð
 Ð#ðð Ððð Ððð Ððð Ð ðð Ððð Ð"ðð 	ˆ/ðð Ððð Ð ðð Ð ðð  Ð%ð!ð" Ðð#ð$ Ð ð%ð Ð ð,7¸Ið 7ð 7ð 7ð 7ð 7ð 7r†   