§
    ž˜ºh¬  ã                   óŒ   — d Z ddlmZ ddlmZ 	 ddlZn# e$ r dZY nw xY w G d„ d¦  «        Zd„ Z	e
dk    r e	¦   «          dS dS )	a˜  
A module for language identification using the TextCat algorithm.
An implementation of the text categorization algorithm
presented in Cavnar, W. B. and J. M. Trenkle,
"N-Gram-Based Text Categorization".

The algorithm takes advantage of Zipf's law and uses
n-gram frequencies to profile languages and text-yet to
be identified-then compares using a distance measure.

Language n-grams are provided by the "An Crubadan"
project. A corpus reader was created separately to read
those files.

For details regarding the algorithm, see:
https://www.let.rug.nl/~vannoord/TextCat/textcat.pdf

For details about An Crubadan, see:
https://borel.slu.edu/crubadan/index.html
é    )Úmaxsize)ÚtrigramsNc                   óF   — e Zd ZdZi ZdZdZi Zd„ Zd„ Z	d„ Z
d„ Zd„ Zd	„ ZdS )
ÚTextCatNú<ú>c                 ó¸   — t           st          d¦  «        ‚ddlm} || _        | j                             ¦   «         D ]}| j                             |¦  «         Œd S )Nz›classify.textcat requires the regex module that supports unicode. Try '$ pip install regex' and see https://pypi.python.org/pypi/regex for further details.r   )Úcrubadan)ÚreÚOSErrorÚnltk.corpusr
   Ú_corpusÚlangsÚ	lang_freq)Úselfr
   Úlangs      ú]/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/nltk/classify/textcat.pyÚ__init__zTextCat.__init__7   s|   € Ýð 	Ýð#ñô ð ð 	)Ð(Ð(Ð(Ð(Ð(àˆŒà”L×&Ò&Ñ(Ô(ð 	)ð 	)ˆDØŒL×"Ò" 4Ñ(Ô(Ð(Ð(ð	)ð 	)ó    c                 ó.   — t          j        dd|¦  «        S )z)Get rid of punctuation except apostrophesz[^\P{P}\']+Ú )r   Úsub©r   Útexts     r   Úremove_punctuationzTextCat.remove_punctuationG   s   € åŒv�n b¨$Ñ/Ô/Ð/r   c                 ó  — ddl m}m} |                      |¦  «        } ||¦  «        } |¦   «         }|D ]L}t	          | j        |z   | j        z   ¦  «        }d„ |D ¦   «         }	|	D ]}
|
|v r||
xx         dz  cc<   Œd||
<   ŒŒM|S )z'Create FreqDist of trigrams within textr   )ÚFreqDistÚword_tokenizec                 ó8   — g | ]}d                       |¦  «        ‘ŒS )r   )Újoin)Ú.0Útris     r   ú
<listcomp>z#TextCat.profile.<locals>.<listcomp>U   s"   € ÐKÐKÐK¨s˜bŸgšg c™lœlÐKÐKÐKr   é   )Únltkr   r   r   r   Ú_START_CHARÚ	_END_CHAR)r   r   r   r   Ú
clean_textÚtokensÚfingerprintÚtÚtoken_trigram_tuplesÚtoken_trigramsÚcur_trigrams              r   ÚprofilezTextCat.profileK   sÚ   € à0Ð0Ð0Ð0Ð0Ð0Ð0Ð0à×,Ò,¨TÑ2Ô2ˆ
Ø�˜zÑ*Ô*ˆà�h‘j”jˆØð 	1ð 	1ˆAÝ#+¨DÔ,<¸qÑ,@À4Ä>Ñ,QÑ#RÔ#RÐ ØKÐKÐ6JÐKÑKÔKˆNà-ð 1ð 1�Ø +Ð-Ð-Ø Ð,Ð,Ô,°Ñ1Ð,Ð,Ñ,Ð,à/0�K Ñ,Ð,ð	1ð Ðr   c                 óJ  — | j                              |¦  «        }d}||v r{t          |                     ¦   «         ¦  «                             |¦  «        }t          |                     ¦   «         ¦  «                             |¦  «        }t          ||z
  ¦  «        }nt          }|S )zgCalculate the "out-of-place" measure between the
        text and language profile for a single trigramr   )r   r   ÚlistÚkeysÚindexÚabsr   )r   r   ÚtrigramÚtext_profileÚlang_fdÚdistÚidx_lang_profileÚidx_texts           r   Ú	calc_distzTextCat.calc_dist_   s—   € ð ”,×(Ò(¨Ñ.Ô.ˆØˆà�gÐÐÝ# G§L¢L¡N¤NÑ3Ô3×9Ò9¸'ÑBÔBÐÝ˜L×-Ò-Ñ/Ô/Ñ0Ô0×6Ò6°wÑ?Ô?ˆHõ Ð'¨(Ñ2Ñ3Ô3ˆDˆDõ
 ˆDàˆr   c                 óÂ   — i }|                       |¦  «        }| j        j                             ¦   «         D ](}d}|D ]}||                      |||¦  «        z  }Œ|||<   Œ)|S )zOCalculate the "out-of-place" measure between
        the text and all languagesr   )r/   r   Ú_all_lang_freqr2   r;   )r   r   Ú	distancesr/   r   Ú	lang_distr5   s          r   Ú
lang_distszTextCat.lang_distst   s   € ð ˆ	Ø—,’,˜tÑ$Ô$ˆà”LÔ/×4Ò4Ñ6Ô6ð 	(ð 	(ˆDð ˆIØ"ð Dð D�Ø˜TŸ^š^¨D°'¸7ÑCÔCÑC�	�	à'ˆI�d‰OˆOàÐr   c                 óv   — |                       |¦  «        | _        t          | j        | j        j        ¬¦  «        S )zYFind the language with the min distance
        to the text and return its ISO 639-3 code)Úkey)r@   Úlast_distancesÚminÚgetr   s     r   Úguess_languagezTextCat.guess_language†   s4   € ð #Ÿošo¨dÑ3Ô3ˆÔå�4Ô&¨DÔ,?Ô,CÐDÑDÔDÐDr   )Ú__name__Ú
__module__Ú__qualname__r   Úfingerprintsr&   r'   rC   r   r   r/   r;   r@   rF   © r   r   r   r   /   sŒ   € € € € € Ø€GØ€LØ€KØ€Ià€Nð)ð )ð )ð 0ð 0ð 0ðð ð ð(ð ð ð*ð ð ð$Eð Eð Eð Eð Er   r   c                  óN  ‡
‡— ddl m}  g d¢}dddddd	d
dddœ	}t          ¦   «         }|D ]û}|                      |¦  «        Št	          ‰¦  «        dz
  }t          t          t          ‰¦  «        ¦  «        }d}t          d|¦  «        D ]@Š
dd                     ˆ
ˆfd„t          d|‰
         ¦  «        D ¦   «         ¦  «        z   }||z  }ŒAt          d|dd…         z   dz   ¦  «         | 
                    |¦  «        }	t          d|	› d||	         › d�¦  «         t          d¦  «         Œüd S )Nr   )Úudhr)	zKurdish-UTF8zAbkhaz-UTF8zFarsi_Persian-UTF8z
Hindi-UTF8zHawaiian-UTF8zRussian-UTF8zVietnamese-UTF8zSerbian_Srpski-UTF8zEsperanto-UTF8zNorthern KurdishÚ	AbkhazianzIranian PersianÚHindiÚHawaiianÚRussianÚ
VietnameseÚSerbianÚ	Esperanto)	ÚkmrÚabkÚpesÚhinÚhawÚrusÚvieÚsrpÚepor$   r   ú c                 ó,   •— g | ]}‰‰         |         ‘ŒS rK   rK   )r!   ÚjÚiÚraw_sentencess     €€r   r#   zdemo.<locals>.<listcomp>¶   s#   ø€ Ð&VÐ&VÐ&V¸q }°QÔ'7¸Ô':Ð&VÐ&VÐ&Vr   zLanguage snippet: éŒ   z...zLanguage detection: z (ú)zŒ############################################################################################################################################)r   rM   r   ÚsentsÚlenr1   ÚmapÚranger    ÚprintrF   )rM   r   ÚfriendlyÚtcÚcur_langÚrowsÚcolsÚsampleÚcur_sentÚguessra   rb   s             @@r   Údemorr   �   s‚  øø€ Ø Ð Ð Ð Ð Ð ð
ð 
ð 
€Eð "ØØ ØØØØØØð
ð 
€Hõ 
‰Œ€Bàð ð ˆàŸ
š
 8Ñ,Ô,ˆÝ�=Ñ!Ô! AÑ%ˆÝ•C�˜]Ñ+Ô+Ñ,Ô,ˆàˆõ �q˜$‘”ð 	ð 	ˆAØ˜SŸXšXÐ&VÐ&VÐ&VÐ&VÐ&VÅEÈ!ÈTÐRSÌWÑDUÔDUÐ&VÑ&VÔ&VÑWÔWÑWˆHØ�hÑˆFˆFõ 	Ð" V¨A¨c¨E¤]Ñ2°UÑ:Ñ;Ô;Ð;Ø×!Ò! &Ñ)Ô)ˆÝÐ@ UÐ@Ð@¨h°u¬oÐ@Ð@Ð@ÑAÔAÐAÝˆiÑÔÐÐð#ð r   Ú__main__)Ú__doc__Úsysr   Ú	nltk.utilr   Úregexr   ÚImportErrorr   rr   rG   rK   r   r   ú<module>ry      sÚ   ððð ð* Ð Ð Ð Ð Ð à Ð Ð Ð Ð Ð ðØÐÐÐÐøØð ð ð Ø	€B€B€Bðøøøð\Eð \Eð \Eð \Eð \Eñ \Eô \Eð \Eð@.ð .ð .ðb ˆzÒÐØ€D�F„F€F€F€Fð Ðs   � •ž