昺h=UDddlZddlZddlmZmZddlmZmZddlm Z ddl Z ddl Z ddl Z ddlZddl ZddlZddlZddlmZmZmZddlmZmZddlZejeZdeded efd Zdeded efd Zdeded efd Zdeded efd Zdeded efdZded efdZ d:dee!de"de#d eeee$e#ffdZ%ddddefdede#de#de#de#deeegefd eeee$e#ffd Z&d eeee!ee#e$fffd!Z'ddd"efd#ed$ede#de#de#deeegefd eeee!ee#e$fffd%Z(d;d&Z)d'efd(Z*d e!fd)Z+d*Z, dd0e!d5e!d1eee"e!fd2ee!d3ee!d ee!f d6Z0 d>d0e!d7e!d1eee"e!fd2ee!d3ee!d ee!f d8Z1d9Z2dS)?N)Tensordevice)ListCallable)tqdm)DictOptionalUnion)snapshot_downloadhf_hub_downloadabreturnc"t||S) Computes the cosine similarity cos_sim(a[i], b[j]) for all i and j. :return: Matrix with res[i][j] = cos_sim(a[i], b[j]) )cos_simr rs b/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/sentence_transformers/util.pypytorch_cos_simrs 1a==c\t|tjstj|}t|tjstj|}t |jdkr|d}t |jdkr|d}tjj |dd}tjj |dd}tj || ddS)rrpdim) isinstancetorchrtensorlenshape unsqueezenn functional normalizemm transpose)r ra_normb_norms rrrs a & & LOO a & & LOO 17||q KKNN 17||q KKNN X * *1q * 9 9F X * *1q * 9 9F 8FF,,Q22 3 33rct|tjstj|}t|tjstj|}t |jdkr|d}t |jdkr|d}tj||ddS)z Computes the dot-product dot_prod(a[i], b[j]) for all i and j. :return: Matrix with res[i][j] = dot_prod(a[i], b[j]) rr) rrrrr r!r"r&r'rs r dot_scorer+5s a & & LOO a & & LOO 17||q KKNN 17||q KKNN 8Aq{{1a(( ) ))rct|tjstj|}t|tjstj|}||zdS)zx Computes the pairwise dot-product dot_prod(a[i], b[i]) :return: Vector with res[i] = dot_prod(a[i], b[i]) )r)rrrrsumrs rpairwise_dot_scorer/Js^ a & & LOO a & & LOO E;;2;  rct|tjstj|}t|tjstj|}t t |t |S)zq Computes the pairwise cossim cos_sim(a[i], b[i]) :return: Vector with res[i] = cos_sim(a[i], b[i]) )rrrrr/normalize_embeddingsrs rpairwise_cos_simr2Ysh a & & LOO a & & LOO 21557KA7N7N O OOr embeddingscPtjj|ddS)z[ Normalizes the embeddings matrix, so that each sentence embedding has unit length rrr)rr#r$r%)r3s rr1r1hs$ 8  ( (qa ( @ @@rF sentencesshow_progress_bar batch_sizecT||||d}t|g|Ri|S)ab Given a list of sentences / texts, this function performs paraphrase mining. It compares all sentences against all other sentences and returns a list with the pairs that have the highest cosine similarity score. :param model: SentenceTransformer model for embedding computation :param sentences: A list of strings (texts or sentences) :param show_progress_bar: Plotting of a progress bar :param batch_size: Number of texts that are encoded simultaneously by the model :param query_chunk_size: Search for most similar pairs for #query_chunk_size at the same time. Decrease, to lower memory footprint (increases run-time). :param corpus_chunk_size: Compare a sentence simultaneously against #corpus_chunk_size other sentences. Decrease, to lower memory footprint (increases run-time). :param max_pairs: Maximal number of text pairs returned. :param top_k: For each sentence, we retrieve up to top_k other sentences :param score_function: Function for computing scores. By default, cosine similarity. :return: Returns a list of triplets with the format [score, id1, id2] T)r7r8convert_to_tensor)encodeparaphrase_mining_embeddings)modelr6r7r8argskwargsr3s rparaphrase_miningr@osG(%6:aeJ ( DT D D DV D DDriii dquery_chunk_sizecorpus_chunk_size max_pairstop_kscore_functionc |dz }tj}d}d}tdt||D]x} tdt||D]U} ||| | |z|| | |z} t j| t |t| dddd\} } | } | } tt| D]}t| |D]r\}}| |z}| |z}||kr]| |||krK| | ||||f|dz }||kr| }|d}sWzt}g}| s{| \}}}t||g\}}||kr5||f|vr/|||f||||g| {t|dd}|S) a Given a list of sentences / texts, this function performs paraphrase mining. It compares all sentences against all other sentences and returns a list with the pairs that have the highest cosine similarity score. :param embeddings: A tensor with the embeddings :param query_chunk_size: Search for most similar pairs for #query_chunk_size at the same time. Decrease, to lower memory footprint (increases run-time). :param corpus_chunk_size: Compare a sentence simultaneously against #corpus_chunk_size other sentences. Decrease, to lower memory footprint (increases run-time). :param max_pairs: Maximal number of text pairs returned. :param top_k: For each sentence, we retrieve up to top_k other sentences :param score_function: Function for computing scores. By default, cosine similarity. :return: Returns a list of triplets with the format [score, id1, id2] rr-rTFrlargestsortedc|dS)Nrxs rz.paraphrase_mining_embeddings..s !A$rkeyreverse)queue PriorityQueueranger rtopkmincputolist enumerateputgetsetemptyrJaddappend)r3rBrCrDrErFpairs min_score num_addedcorpus_start_idxquery_start_idxscoresscores_top_k_valuesscores_top_k_idx query_itr top_k_idx corpus_itrijentry added_pairs pairs_listscoresorted_isorted_js rr<r<s* QJE   ! !EII!!S__6GHH11$QJ9IJJ 1 1O#^?_?O-OOP+.>AR.RRSF 5:JE3vay>>224PU555 1 !1#6"9"9";";"B"B"D"D /3355<<>> "3v;;// 1 1 -67G 7R-S-S 1 1)Iz')3A(:5AAvv"5i"@"Ki"W"W #6y#A)#LaQR"STTT!Q $ 11$)IIKKE(-aI 1 1 14%%KJkkmm;iikk q!#QF^^( x  Xx$8 $K$K OOXx0 1 1 1   uh9 : : : kkmm; EEEJ rct|i|S)z8This function is deprecated. Use semantic_search instead)semantic_search)r>r?s rinformation_retrievalrvs D +F + ++r query_embeddingscorpus_embeddingsc t|tjtjfrt j|}n)t|t rt j|}t|j dkr| d}t|tjtjfrt j|}n)t|t rt j|}|j |j kr| |j }dtt|D}tdt||D]p}tdt||D]M}|||||z||||z} t j| t|t| dddd\} } | } | } tt| D]} t%| | | | D]c\} }|| z}|| z}t|||krt'j||||fFt'j||||fdOrtt|D]b}tt||D]!}|||\}}||d|||<"t-||dd ||<c|S) a This function performs a cosine similarity search between a list of query embeddings and a list of corpus embeddings. It can be used for Information Retrieval / Semantic Search for corpora up to about 1 Million entries. :param query_embeddings: A 2 dimensional tensor with the query embeddings. :param corpus_embeddings: A 2 dimensional tensor with the corpus embeddings. :param query_chunk_size: Process 100 queries simultaneously. Increasing that value increases the speed, but requires more memory. :param corpus_chunk_size: Scans the corpus 100k entries at a time. Increasing that value increases the speed, but requires more memory. :param top_k: Retrieve top k matching entries. :param score_function: Function for computing scores. By default, cosine similarity. :return: Returns a list with one entry for each query. Each entry is a list of dictionaries with the keys 'corpus_id' and 'score', sorted by decreasing cosine similarity scores. rrcg|]}gSrLrL).0_s r z#semantic_search..sDDD!2DDDrTFrH) corpus_idrqc|dS)NrqrLrMs rrOz!semantic_search..s \]^e\frrP)rnpndarraygenericr from_numpyliststackr r!r"rtorUrVrWrXrYzipheapqheappush heappushpoprJ)rxryrBrCrErFqueries_result_listrerd cos_scorescos_scores_top_k_valuescos_scores_top_k_idxri sub_corpus_idrqrquery_iddoc_itrs rrurus*"RZ$<==9 +,<== $d + +9 ;'788  !""a''+55a88#bj"*%=>>;!,->?? %t , ,;!K(9::#3#:::+../@/GHHDDuS1A-B-B'C'CDDD C(8$9$9;KLL]] %a->)?)?AR S S ] ] ' ?EU3U!UV!"25EHY5Y"YZJ =BJCs:a='9'9::4X]=== 9 #%9'>&A&A&C&C&J&J&L&L ##7#;#;#=#=#D#D#F#F "3z??33 ] ] ,/0DY0OQhirQs,t,t]](M5 0= @I.:H.x899EAA/9E9;M)*=h*G%QZI[\\\\] ] ]4#12233vvS!4X!>??@@ ^ ^G28.s ArrPc t|SrrrMs rrOz%community_detection..s #a&&r)rrrrrr1rWr maxrrUTtyper.anyrVrr`rYrJr]rZr)r3 thresholdmin_community_sizer8r7extracted_communities sort_max_size start_idxrthreshold_maskrow_wise_countlarge_enough_maskrr} top_k_indicescountindices top_k_valuesrl top_val_large top_idx_largeunique_communities extracted_ids cluster_id communitynon_overlapped_communityidxs rcommunity_detectionrks j%, / /.\*--  Yz/@AAAI%j11J/ZAAA 22B77ZIIM aZ*--4FTePe*e*e  I ,B BCjlR    !V + +'94N+//22N!/2D D $((** +,=>N#$56J""$$A)DAA A}#&nm"D"D G Gw%,,WVeV_-C-C-E-EFFFF G)oo0BDoQQOL!3|,,-- e e?2&)333=a=3E3E _c3E3d3d0M=(+i77MCPZOO<[<[(+A ,=s:(O(O 7A!}7I7IMcg7I7h7h4 }(+i77MCPZOO<[<[*00}PY?Y1Z1a1a1c1cddd e##8>N>NX\]]]EEM!*+@!A!A;; I#%  5 5C-''(//444 ' ( (,> > >  % %&> ? ? ?  !9 : : : 28H8HRVWWW rc6eZdZdZfdZdeddffd ZxZS) disabled_tqdmz Class to override `disable` argument in case progress bars are globally disabled. Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324. cDd|d<tj|i|dS)NTr)super__init__)selfr>r?rs rrzdisabled_tqdm.__init__s/ y$)&)))))rattrrNc| t|dS#t$r |dkrYdSwxYw)zBFix for https://github.com/huggingface/huggingface_hub/issues/1603_lockN)r __delattr__r)rrrs rrzdisabled_tqdm.__delattr__sV  GG   % % % % %   w s !&;;)rr __qualname____doc__rrr __classcell__)rs@rrrsn *****rrmodel_name_or_pathtoken cache_folderrevisionc Dtt|d|||S)Nz modules.json)r )boolload_file_path)rr r r s ris_sentence_transformer_modelrs' 1>5,aijjj k kkrfilenamectj||}tj|r|S t |||d||S#t $rYdSwxYw)Nsentence-transformers)rr  library_namer  cache_dir)rrjoinexistsr r)rrr r r  file_paths rrrs /::I w~~i    0"      sA A&%A& directorycFtj||}tj|r|S|||dd||td} t di|}n!#t $rd|d<t di|}YnwxYwtj||S)Nz/**r)repo_idr allow_patternsrr r tqdm_classTlocal_files_onlyrL)rrrrrr r)rrr r r dir_pathdownload_kwargs repo_paths r load_dir_pathr!sw||. ::H w~~h&&+++/!#O9%8888 999.2*+%8888 9 7<< 9 - --s A""B?BcFtjfd}|S)Nc|dd}|r#d|vrtd||d<t|dkrg|ddd|ddR}|g|Ri|S)N repo_namerzfProviding a `repo_name` keyword argument to `save_to_hub` is deprecated, please use `repo_id` instead.r)poploggerwarningr )rr>r?r$funcs rwrapperz+save_to_hub_args_decorator..wrappersJJ{D11  *&00 NNx   !*F9  t99>>/T"1"X/t/d122h//DtD*4***6***r) functoolswraps)r(r)s` rsave_to_hub_args_decoratorr,s8_T + + + + + Nr)Fr5)rN)rrwrF)NNNr)3r*rrrrtypingrrtqdm.autonotebookrrrrnumpyrrSloggingrr r huggingface_hubr r r getLoggerrr&rrr+r/r2r1rr rfloatr@r<rvrurrrrrrrrr!r,rLrrr4s !!!!!!!!""""""   ((((((((((>>>>>>>>  8 $ $v&V4v4&4V4444.**F*v***** & V      P P6 Pf P P P PAVAAAAAUWEE3iE48ENQE $uUCZ !"EEEE:!#9@ CCCCC C  C ff-v56 C $uUCZ !"CCCCL,d4S%U BS=S8T3U.V,,,, #9@ JJJJJ J  J ff-v56 J $tCsEz**+ ,-JJJJZ6& 33 3 3 3 30[`TT $s)_TTTTxD,)-"&" lll E$)$ %l3-lsm l  llll#  E$)$ %3-  sm  c] ># .... E$)$ %.3- . sm . c] ....>r