§
    Ý˜ºh±  ã                   ól   — d dl mZmZmZmZmZ d dlmZ d dlm	Z	 d dl
mZ  G d„ d¦  «        Zdefd„Zd	S )
é    )ÚRegexÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚBPE)ÚLlamaTokenizerFast)Úbytes_to_unicodec                   óJ   ‡ — e Zd ZdZ	 	 	 	 dˆ fd„	Zdefd„Zd„ Zd	efd
„Z	ˆ xZ
S )ÚMistralConverterz'
    A general tiktoken converter.
    Nús(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Fc                 ól   •—  t          ¦   «         j        |Ž  || _        || _        || _        || _        d S )N)ÚsuperÚ__init__ÚvocabÚpatternÚadd_prefix_spaceÚadditional_special_tokens)Úselfr   r   r   r   ÚargsÚkwargsÚ	__class__s          €úi/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/transformers/integrations/mistral.pyr   zMistralConverter.__init__   s=   ø€ ð 	�‰ŒÔ˜$ÐÐØˆŒ
ØˆŒØ 0ˆÔØ)BˆÔ&Ð&Ð&ó    r   c                 óH  ‡
‡‡— |Š
t          ¦   «         Šˆfd„Šg }i }t          ‰
                     ¦   «         ¦  «        D ]À\  }\  }}|| j        vrª|| ‰|¦  «        <   t	          |¦  «        dk    rŒ3g }t          dt	          |¦  «        ¦  «        D ]=}|d |…         ||d …         }	}|‰
v r#|	‰
v r||	z   ‰
v r|                     ||	|f¦  «         Œ>t          |ˆ
fd„d¬¦  «        }|                     |¦  «         Œ»|||<   ŒÁt          |d„ d¬¦  «        }ˆfd„|D ¦   «         }||fS )Nc                 ól   •— d                      ˆfd„|                      d¦  «        D ¦   «         ¦  «        S )NÚ c                 ó:   •— g | ]}‰t          |¦  «                 ‘ŒS © )Úord)Ú.0ÚcharÚbyte_encoders     €r   ú
<listcomp>zcMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string.<locals>.<listcomp>!   s$   ø€ ÐTÐTÐT¸˜L­¨T©¬Ô3ÐTÐTÐTr   zlatin-1)ÚjoinÚdecode)Úbr#   s    €r   Útoken_bytes_to_stringzOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string    s6   ø€ Ø—7’7ÐTÐTÐTÐTÀÇÂÈÑ@SÔ@SÐTÑTÔTÑUÔUÐUr   é   c                 ó<   •— ‰| d                  ‰| d                  fS )Nr   r)   r   )ÚxÚ	bpe_rankss    €r   ú<lambda>zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>/   s   ø€ °Y¸qÀ¼t´_ÀiÐPQÐRSÔPTÄoÐ4V€ r   F)ÚkeyÚreversec                 ó   — | d         S )Né   r   )Úvals    r   r-   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>3   s
   € °°A´€ r   c                 óT   •— g | ]$} ‰|d          ¦  «         ‰|d         ¦  «        f‘Œ%S )r   r)   r   )r!   r2   r(   s     €r   r$   zDMistralConverter.extract_vocab_merges_from_model.<locals>.<listcomp>4   s?   ø€ ÐcÐcÐcÐUXÐ(Ð(¨¨Q¬Ñ0Ô0Ð2GÐ2GÈÈAÌÑ2OÔ2OÐPÐcÐcÐcr   )	r
   Ú	enumerateÚitemsr   ÚlenÚrangeÚappendÚsortedÚextend)r   r   ÚmergesÚidxÚtokenÚrankÚlocalÚindexÚpiece_lÚpiece_rr,   r#   r(   s             @@@r   Úextract_vocab_merges_from_modelz0MistralConverter.extract_vocab_merges_from_model   s—  øøø€ Øˆ	Ý'Ñ)Ô)ˆð	Vð 	Vð 	Vð 	Vð 	Vð ˆØˆÝ"+¨I¯OªOÑ,=Ô,=Ñ">Ô">ð 	#ð 	#ÑˆC‘�%˜Ø˜DÔ:Ð:Ð:Ø69�Ð+Ð+¨EÑ2Ô2Ñ3Ý�u‘:”: ’?�?ØØ�Ý" 1¥c¨%¡j¤jÑ1Ô1ð ?ð ?�EØ',¨V¨e¨V¤}°e¸E¸F¸F´m˜W�GØ )Ð+Ð+°¸9Ð0DÐ0DÈ'ÐT[ÑJ[Ð`iÐIiÐIiØŸš g¨w¸Ð%=Ñ>Ô>Ð>øÝ˜uÐ*VÐ*VÐ*VÐ*VÐ`eÐfÑfÔf�Ø—’˜eÑ$Ô$Ð$Ð$à"��e‘�Ý˜Ð$6Ð$6ÀÐFÑFÔFˆØcÐcÐcÐcÐ\bÐcÑcÔcˆØ�fˆ}Ðr   c                 óÀ   — |                       | j        ¦  «        \  }}t          t          ||d¬¦  «        ¦  «        }t	          |j        d¦  «        rd|j        _        |S )NF)Úfuse_unkÚignore_mergesT)rC   r   r   r   ÚhasattrÚmodelrF   )r   Úvocab_scoresr;   Ú	tokenizers       r   rJ   zMistralConverter.tokenizer7   s\   € Ø#×CÒCÀDÄJÑOÔOÑˆ�fÝ�c ,°ÀÐGÑGÔGÑHÔHˆ	Ý�9”? OÑ4Ô4ð 	1Ø,0ˆIŒOÔ)ØÐr   Úreturnc                 ó|  — |                       ¦   «         }t          j        t          j        t	          | j        ¦  «        dd¬¦  «        t          j        | j        d¬¦  «        g¦  «        |_        t          j        ¦   «         |_
        |                     | j        ¦  «         t          j        d¬¦  «        |_        |S )NÚisolatedF)ÚbehaviorÚinvert)r   Ú	use_regex)Útrim_offsets)rJ   r   ÚSequenceÚSplitr   r   Ú	ByteLevelr   Úpre_tokenizerr   ÚdecoderÚadd_special_tokensr   r   Úpost_processor)r   rJ   s     r   Ú	convertedzMistralConverter.converted>   s¨   € Ø—N’NÑ$Ô$ˆ	Ý"0Ô"9åÔ$¥U¨4¬<Ñ%8Ô%8À:ÐV[Ð\Ñ\Ô\ÝÔ(¸$Ô:OÐ[`ÐaÑaÔaðñ#
ô #
ˆ	Ôõ %Ô.Ñ0Ô0ˆ	ÔØ×$Ò$ TÔ%CÑDÔDÐDå#-Ô#7ÀUÐ#KÑ#KÔ#Kˆ	Ô àÐr   )Nr   FN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚstrrC   rJ   r   rY   Ú__classcell__)r   s   @r   r   r      s£   ø€ € € € € ðð ð ð KØØ"&ðCð Cð Cð Cð Cð Cð°Sð ð ð ð ð6ð ð ð˜9ð ð ð ð ð ð ð ð r   r   Útokenizer_filec                 óp  ‡— ddl m} |                     | ¦  «        }|j        j        j        }d„ |j        j        j        D ¦   «         Šˆfd„‰D ¦   «         }|                     |¦  «         |}t          t          |‰¬¦  «         
                    ¦   «         ¬¦  «        }|                     d‰i¦  «         |S )z1Convert a "tekken" tokenizer to a fast Tokenizer.r   )ÚMistralTokenizerc                 ó@   — g | ]}t          |d ¦  «        r|j        n|‘ŒS )Úvalue)rG   rd   )r!   r=   s     r   r$   z,convert_tekken_tokenizer.<locals>.<listcomp>Y   s=   € ð ð ð àõ ˜u gÑ.Ô.Ð9ˆŒˆ°Eðð ð r   c                 ó<   •— i | ]}|‰                      |¦  «        “ŒS r   )r@   )r!   r=   Úall_specials     €r   ú
<dictcomp>z,convert_tekken_tokenizer.<locals>.<dictcomp>]   s)   ø€ ÐPÐPÐP¸5�u˜k×/Ò/°Ñ6Ô6ÐPÐPÐPr   )r   r   )Útokenizer_objectr   )Ú(mistral_common.tokens.tokenizers.mistralrb   Ú	from_fileÚinstruct_tokenizerrJ   Ú_tekken_token2id_nospecialÚ_all_special_tokensÚupdater	   r   rY   rW   )r`   rb   Úmistral_tokenizerr   Úspecials_tokensrJ   rf   s         @r   Úconvert_tekken_tokenizerrq   N   sï   ø€ ð JÐIÐIÐIÐIÐIð )×2Ò2°>ÑBÔBÐð Ô0Ô:ÔU€Eðð à&Ô9ÔCÔWðñ ô €Kð QÐPÐPÐPÀKÐPÑPÔP€OØ×Ò˜5Ñ!Ô!Ð!Ø€Eõ #Ý)°ÐQ\Ð]Ñ]Ô]×gÒgÑiÔiðñ ô €Ið
 × Ò Ð"=¸{Ð!KÑLÔLÐLàÐr   N)Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   Útransformersr	   Ú#transformers.convert_slow_tokenizerr
   r   r^   rq   r   r   r   ú<module>rv      s»   ðØ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MØ !Ð !Ð !Ð !Ð !Ð !à +Ð +Ð +Ð +Ð +Ð +Ø @Ð @Ð @Ð @Ð @Ð @ðCð Cð Cð Cð Cñ Cô Cð CðL¨Sð ð ð ð ð ð r   