§
    Ý˜ºh[  ã                   óJ   — d dl mZ d dlmZ d dlmZ d dlmZmZ dede	fd„Z
dS )	é    )ÚPath)ÚAny)ÚTikTokenConverter)ÚTIKTOKEN_VOCAB_FILEÚTOKENIZER_FILEÚencodingÚ
output_dirc                 ó@  — t          |¦  «        }|                     d¬¦  «         |dz  t          z  }|t          z  }t	          |                     ¦   «         ¦  «        }t	          |                     ¦   «         ¦  «        }	 ddlm} ddlm	} t          | t          ¦  «        r || ¦  «        }  || j        |¦  «         n# t          $ r t          d¦  «        ‚w xY wt          || j        | j        ¬¦  «                             ¦   «         }|                     |¦  «         d	S )
a»  
    Converts given `tiktoken` encoding to `PretrainedTokenizerFast` and saves the configuration of converted tokenizer
    on disk.

    Args:
        encoding (`str` or `tiktoken.Encoding`):
            Tokenizer from `tiktoken` library. If `encoding` is `str`, the tokenizer will be loaded with
            `tiktoken.get_encoding(encoding)`.
        output_dir (`str`):
            Save path for converted tokenizer configuration file.
    T)Úexist_okÚtiktokenr   )Úget_encoding)Údump_tiktoken_bpezY`tiktoken` is required to save a `tiktoken` file. Install it with `pip install tiktoken`.)Ú
vocab_fileÚpatternÚadditional_special_tokensN)r   Úmkdirr   r   ÚstrÚabsoluter   r   Útiktoken.loadr   Ú
isinstanceÚ_mergeable_ranksÚImportErrorÚ
ValueErrorr   Ú_pat_strÚ_special_tokensÚ	convertedÚsave)	r   r	   Ú	save_fileÚtokenizer_fileÚsave_file_absoluteÚoutput_file_absoluter   r   Ú	tokenizers	            új/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/transformers/integrations/tiktoken.pyÚconvert_tiktoken_to_fastr$      sT  € õ �jÑ!Ô!€JØ×Ò˜dÐÑ#Ô#Ð#à˜ZÑ'Õ*=Ñ=€IØ¥.Ñ0€Nå˜Y×/Ò/Ñ1Ô1Ñ2Ô2ÐÝ˜~×6Ò6Ñ8Ô8Ñ9Ô9Ðð	vØ)Ð)Ð)Ð)Ð)Ð)Ø3Ð3Ð3Ð3Ð3Ð3å�h¥Ñ$Ô$ð 	.Ø#�| HÑ-Ô-ˆHàÐ˜(Ô3Ð5GÑHÔHÐHÐHøÝð vð vð vÝÐtÑuÔuÐuðvøøøõ "Ø%¨xÔ/@Ð\dÔ\tðñ ô ç‚i�k„kð ð ‡N‚NÐ'Ñ(Ô(Ð(Ð(Ð(s   Â =B> Â>CN)Úpathlibr   Útypingr   Ú#transformers.convert_slow_tokenizerr   Ú$transformers.tokenization_utils_fastr   r   r   r$   © ó    r#   ú<module>r+      s   ðØ Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AØ TÐ TÐ TÐ TÐ TÐ TÐ TÐ Tð#) sð #)¸ð #)ð #)ð #)ð #)ð #)ð #)r*   