§
    æ˜ºh£  ã                   óŠ   — d dl mZ d dlZd dlZddlmZ ddlmZ d dlm	Z	 d dl
Z
 ej        e¦  «        Z G d„ de¦  «        ZdS )	é    )ÚDatasetNé   )ÚSentenceTransformer)ÚInputExample)ÚListc            	       ó    — e Zd ZdZ	 	 ddedededefd„Zddedededefd„Z		 	 	 dde
e
e                  dededefd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd
S )ÚParallelSentencesDatasetu�  
    This dataset reader can be used to read-in parallel sentences, i.e., it reads in a file with tab-seperated sentences with the same
    sentence in different languages. For example, the file can look like this (EN	DE	ES):
    hello world     hallo welt  hola mundo
    second sentence zweiter satz    segunda oraciÃ³n

    The sentence in the first column will be mapped to a sentence embedding using the given the embedder. For example,
    embedder is a mono-lingual sentence embedding method for English. The sentences in the other languages will also be
    mapped to this English sentence embedding.

    When getting a sample from the dataset, we get one sentence with the according sentence embedding for this sentence.

    teacher_model can be any class that implement an encode function. The encode function gets a list of sentences and
    returns a list of sentence embeddings
    é   TÚstudent_modelÚteacher_modelÚ
batch_sizeÚuse_embedding_cachec                 ó®   — || _         || _        g | _        g | _        g | _        g | _        g | _        g | _        || _        || _	        i | _
        d| _        dS )a3  
        Parallel sentences dataset reader to train student model given a teacher model

        :param student_model: Student sentence embedding model that should be trained
        :param teacher_model: Teacher model, that provides the sentence embeddings for the first column in the dataset file
        r   N)r   r   ÚdatasetsÚdatasets_iteratorÚdatasets_tokenizedÚdataset_indicesÚcopy_dataset_indicesÚcacher   r   Úembedding_cacheÚnum_sentences)Úselfr   r   r   r   s        ú/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/sentence_transformers/datasets/ParallelSentencesDataset.pyÚ__init__z!ParallelSentencesDataset.__init__   sg   € ð +ˆÔØ*ˆÔØˆŒØ!#ˆÔØ"$ˆÔØ!ˆÔØ$&ˆÔ!ØˆŒ
Ø$ˆŒØ#6ˆÔ Ø!ˆÔØˆÔÐÐó    éd   Né€   ÚfilepathÚweightÚmax_sentencesÚmax_sentence_lengthc                 ó  — t                                d|z   ¦  «         g }|                     d¦  «        rt          j        |dd¬¦  «        nt	          |d¬¦  «        5 }d}|D ]y}|                     ¦   «                              d¦  «        }	|�$|dk    rt          d	„ |	D ¦   «         ¦  «        |k    rŒO|                     |	¦  «         |d
z  }|�|dk    r||k    r nŒzddd¦  «         n# 1 swxY w Y   |  	                    ||||¬¦  «         dS )at  
        Reads in a tab-seperated .txt/.csv/.tsv or .gz file. The different columns contain the different translations of the sentence in the first column

        :param filepath: Filepath to the file
        :param weight: If more than one dataset is loaded with load_data: With which frequency should data be sampled from this dataset?
        :param max_sentences: Max number of lines to be read from filepath
        :param max_sentence_length: Skip the example if one of the sentences is has more characters than max_sentence_length
        :param batch_size: Size for encoding parallel sentences
        :return:
        zLoad z.gzÚrtÚutf8)Úencodingr   ú	Nc                 ó,   — g | ]}t          |¦  «        ‘ŒS © ©Úlen©Ú.0Úsents     r   ú
<listcomp>z6ParallelSentencesDataset.load_data.<locals>.<listcomp>P   s   € Ð=Ð=Ð=¨4�S ™YœYÐ=Ð=Ð=r   é   )r   r    r!   )
ÚloggerÚinfoÚendswithÚgzipÚopenÚstripÚsplitÚmaxÚappendÚadd_dataset)
r   r   r   r    r!   Úparallel_sentencesÚfInÚcountÚlineÚ	sentencess
             r   Ú	load_dataz"ParallelSentencesDataset.load_data8   s”  € õ 	�Š�G˜hÑ&Ñ'Ô'Ð'ØÐà;C×;LÒ;LÈUÑ;SÔ;Sð 
�TŒY�x °Ð7Ñ7Ô7Ð7ÕY]Ø˜vðZ
ñ Z
ô Z
ð 	àØˆEØð ð �Ø ŸJšJ™LœL×.Ò.¨tÑ4Ô4�	à'Ð3Ø+¨aÒ/Ð/ÝÐ=Ð=°9Ð=Ñ=Ô=Ñ>Ô>ÐATÒTÐTàà"×)Ò)¨)Ñ4Ô4Ð4Ø˜‘
�Ø Ð,°ÀÒ1BÐ1BÀuÐP]ÒG]ÐG]Ø�Eøð!	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð" 	×ÒØ v¸]Ð`sð 	ñ 	
ô 	
ð 	
ð 	
ð 	
s   ÁA?C(Ã(C,Ã/C,r:   c                 óœ  ‡	— i Š	|D ]‚}|�$|dk    rt          d„ |D ¦   «         ¦  «        |k    rŒ(|d         }|‰	vrt          ¦   «         ‰	|<   |D ]}‰	|                              |¦  «         Œ|�|dk    rt          ‰	¦  «        |k    r nŒƒt          ‰	¦  «        dk    rd S | xj        t          ˆ	fd„‰	D ¦   «         ¦  «        z  c_        t          | j        ¦  «        }| j                             t          ‰	 	                    ¦   «         ¦  «        ¦  «         | j
                             d¦  «         | j                             |g|z  ¦  «         d S )Nr   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r(   r)   r+   s     r   r.   z8ParallelSentencesDataset.add_dataset.<locals>.<listcomp>h   s   € Ð9Ð9Ð9 t�˜T™œÐ9Ð9Ð9r   c                 ó:   •— g | ]}t          ‰|         ¦  «        ‘ŒS r(   r)   )r,   r-   Úsentences_maps     €r   r.   z8ParallelSentencesDataset.add_dataset.<locals>.<listcomp>y   s&   ø€ Ð"VÐ"VÐ"VÀ¥3 }°TÔ':Ñ#;Ô#;Ð"VÐ"VÐ"Vr   )r7   ÚsetÚaddr*   r   Úsumr   r8   ÚlistÚitemsr   r   Úextend)
r   r:   r   r    r!   r>   Úsource_sentencer-   Ú
dataset_idrC   s
            @r   r9   z$ParallelSentencesDataset.add_dataset\   sƒ  ø€ ð ˆØ+ð 	ð 	ˆIà#Ð/Ø'¨!Ò+Ð+ÝÐ9Ð9¨yÐ9Ñ9Ô9Ñ:Ô:Ð=PÒPÐPàà'¨œlˆOØ mÐ3Ð3Ý14±´�˜oÑ.à!ð 9ð 9�Ø˜oÔ.×2Ò2°4Ñ8Ô8Ð8Ð8àÐ(¨]¸QÒ->Ð->Å3À}ÑCUÔCUÐYfÒCfÐCfØ�øåˆ}ÑÔ Ò"Ð"ØˆFàÐÔ�cÐ"VÐ"VÐ"VÐ"VÈÐ"VÑ"VÔ"VÑWÔWÑWÐÔå˜œÑ'Ô'ˆ
ØŒ×Ò�T -×"5Ò"5Ñ"7Ô"7Ñ8Ô8Ñ9Ô9Ð9ØÔ×%Ò% aÑ(Ô(Ð(ØÔ×#Ò# Z L°6Ñ$9Ñ:Ô:Ð:Ð:Ð:r   c                 óŒ  — g }g }| j         D ]D}|                      |¦  «        \  }}|                     |¦  «         |                     |¦  «         ŒE|                      |¦  «        }t	          ||¦  «        D ]4\  }}|D ],}| j                             t          |g|¬¦  «        ¦  «         Œ-Œ5t          j        | j        ¦  «         d S )N)ÚtextsÚlabel)	r   Ú
next_entryr8   Úget_embeddingsÚzipr   r   ÚrandomÚshuffle)	r   Úsource_sentences_listÚtarget_sentences_listÚdata_idxÚsrc_sentenceÚtrg_sentencesÚsrc_embeddingsÚsrc_embeddingÚtrg_sentences	            r   Úgenerate_dataz&ParallelSentencesDataset.generate_data€   sñ   € Ø "ÐØ "ÐØÔ,ð 	8ð 	8ˆHØ*.¯/ª/¸(Ñ*CÔ*CÑ'ˆL˜-Ø!×(Ò(¨Ñ6Ô6Ð6Ø!×(Ò(¨Ñ7Ô7Ð7Ð7ð ×,Ò,Ð-BÑCÔCˆå,/°Ð@UÑ,VÔ,Vð 	[ð 	[Ñ(ˆM˜=Ø -ð [ð [�Ø”
×!Ò!¥,°l°^È=Ð"YÑ"YÔ"YÑZÔZÐZÐZð[õ 	Œ�t”zÑ"Ô"Ð"Ð"Ð"r   c                 ó  — | j         |         | j        |                  \  }}| j        |xx         dz  cc<   | j        |         t          | j         |         ¦  «        k    r)d| j        |<   t          j        | j         |         ¦  «         ||fS )Nr/   r   )r   r   r*   rR   rS   )r   rV   ÚsourceÚtarget_sentencess       r   rO   z#ParallelSentencesDataset.next_entry‘   s“   € Ø#'¤=°Ô#:¸4Ô;QÐRZÔ;[Ô#\Ñ ˆÐ àÔ˜xÐ(Ð(Ô(¨AÑ-Ð(Ð(Ñ(ØÔ! (Ô+­s°4´=ÀÔ3JÑ/KÔ/KÒKÐKØ/0ˆDÔ" 8Ñ,ÝŒN˜4œ=¨Ô2Ñ3Ô3Ð3àÐ'Ð'Ð'r   c                 ój  ‡ — ‰ j         s#‰ j                             |‰ j        dd¬¦  «        S g }|D ] }|‰ j        vr|                     |¦  «         Œ!t          |¦  «        dk    rC‰ j                             |‰ j        dd¬¦  «        }t          ||¦  «        D ]\  }}|‰ j        |<   Œˆ fd„|D ¦   «         S )NFT)r   Úshow_progress_barÚconvert_to_numpyr   c                 ó*   •— g | ]}‰j         |         ‘ŒS r(   )r   )r,   r-   r   s     €r   r.   z;ParallelSentencesDataset.get_embeddings.<locals>.<listcomp>®   s!   ø€ ÐAÐAÐA¨t�Ô$ TÔ*ÐAÐAÐAr   )r   r   Úencoder   r   r8   r*   rQ   )r   r>   Únew_sentencesr-   Únew_embeddingsÚ	embeddings   `     r   rP   z'ParallelSentencesDataset.get_embeddings›   sø   ø€ ØÔ'ð 	ØÔ%×,Ò,Ø d¤oÈÐaeð -ñ ô ð ð
 ˆØð 	+ð 	+ˆDØ˜4Ô/Ð/Ð/Ø×$Ò$ TÑ*Ô*Ð*øåˆ}ÑÔ Ò!Ð!Ø!Ô/×6Ò6Ø¨$¬/ÈUÐeið 7ñ ô ˆNõ $' }°nÑ#EÔ#Eð 7ð 7‘��iØ-6�Ô$ TÑ*Ð*àAÐAÐAÐA°yÐAÑAÔAÐAr   c                 ó   — | j         S )N)r   )r   s    r   Ú__len__z ParallelSentencesDataset.__len__°   s   € ØÔ!Ð!r   c                 óŒ   — t          | j        ¦  «        dk    r|                      ¦   «          | j                             ¦   «         S )Nr   )r*   r   r\   Úpop)r   Úidxs     r   Ú__getitem__z$ParallelSentencesDataset.__getitem__³   s9   € ÝˆtŒz‰?Œ?˜aÒÐØ×ÒÑ Ô Ð àŒz�~Š~ÑÔÐr   )r
   T)r   Nr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚintÚboolr   Ústrr?   r   r9   r\   rO   rP   ri   rm   r(   r   r   r	   r	      s=  € € € € € ðð ð( Ø$(ðð à*ðð +ðð ð	ð
 "ðð ð ð ð4"
ð "
 #ð "
¨sð "
Èð "
Ðjmð "
ð "
ð "
ð "
ðN Ø!Ø#&ð";ð ";à   c¤œOð";ð ð";ð ð	";ð
 !ð";ð ";ð ";ð ";ðH#ð #ð #ð"(ð (ð (ðBð Bð Bð*"ð "ð "ð ð  ð  ð  ð  r   r	   )Útorch.utils.datar   Úloggingr3   Ú r   Úreadersr   Útypingr   rR   Ú	getLoggerrn   r0   r	   r(   r   r   ú<module>r{      sº   ðØ $Ð $Ð $Ð $Ð $Ð $Ø €€€Ø €€€Ø "Ð "Ð "Ð "Ð "Ð "Ø "Ð "Ð "Ð "Ð "Ð "Ø Ð Ð Ð Ð Ð Ø €€€ð 
ˆÔ	˜8Ñ	$Ô	$€ðj ð j ð j ð j ð j ˜wñ j ô j ð j ð j ð j r   