§
    ž˜ºhHy  ã                   óÚ   — d Z ddlmZmZ ddlZddlmZmZmZm	Z	m
Z
mZmZmZ ddlmZ d„ Zej                              e¦   «          d¬¦  «        Z G d	„ d
¦  «        Z G d„ d¦  «        ZdS )zC
Unit tests for nltk.tokenize.
See also nltk/test/tokenize.doctest
é    )ÚListÚTupleN)ÚLegalitySyllableTokenizerÚStanfordSegmenterÚSyllableTokenizerÚTreebankWordTokenizerÚTweetTokenizerÚpunktÚsent_tokenizeÚword_tokenize)ÚCharTokenizerc                  óš   — 	 t          ¦   «         } |                      d¦  «         |                      d¦  «         dS # t          $ r Y dS w xY w)NÚarÚzhTF)r   Údefault_configÚLookupError)Úsegs    úd/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/nltk/test/unit/test_tokenize.pyÚload_stanford_segmenterr      sa   € ðÝÑ!Ô!ˆØ×Ò˜4Ñ Ô Ð Ø×Ò˜4Ñ Ô Ð ØˆtøÝð ð ð Øˆuˆuðøøøs   ‚8< ¼
A
Á	A
z/NLTK was unable to find stanford-segmenter.jar.)Úreasonc                   óÒ  — e Zd Zd„ Zej                             ddg d¢g d¢ffdg d¢g d¢ffd	g d
¢g d
¢ffdg d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffdg d¢g d¢ffg¦  «        dedee	e         e	e         f         fd„¦   «         Z
d„ Zd„ Zd„ Zed„ ¦   «         Zed „ ¦   «         Zd!„ Zd"„ Zd#„ Zd$„ Zd%„ Zd&„ Zd'„ Zd(„ Zd)„ Zd*„ Zd+„ Zd,„ Zej                             d-g d.¢¦  «        dPd0„¦   «         Zd1„ Zej                             d2d3g d4¢fd5g d6¢fd7d7gfd8d8gfd9d:d;gfd<d=d>gfd?g d@¢fdAdBd;gfdCdDd;gfdEg dF¢fdGg dF¢fdHdIdJgfg¦  «        dKedLe	e         fdM„¦   «         ZdQdO„Z d/S )RÚTestTokenizec                 ón   — t          dd¬¦  «        }d}|                     |¦  «        }g d¢}||k    sJ ‚dS )zW
        Test TweetTokenizer using words with special and accented characters.
        T)Ústrip_handlesÚ
reduce_lenuA   @myke: Let's test these words: resumÃ© EspaÃ±a MÃ¼nchen franÃ§ais)
ú:zLet'sÚtestÚtheseÚwordsr   u   resumÃ©u   EspaÃ±au   MÃ¼nchenu	   franÃ§aisN©r	   Útokenize)ÚselfÚ	tokenizerÚs9ÚtokensÚexpecteds        r   Útest_tweet_tokenizerz!TestTokenize.test_tweet_tokenizer(   sX   € õ
 #°À$ÐGÑGÔGˆ	ØPˆØ×#Ò# BÑ'Ô'ˆð
ð 
ð 
ˆð ˜Ò!Ð!Ð!Ð!Ð!Ð!ó    ztest_input, expectedsz#My text 0106404243030 is great text)ÚMyÚtextÚ01064042430Ú30ÚisÚgreatr*   )r)   r*   Ú0106404243030r-   r.   r*   zMy ticket id is 1234543124123)r)   ÚticketÚidr-   Ú12345431241Ú23)r)   r0   r1   r-   Ú1234543124123z<@remy: This is waaaaayyyy too much for you!!!!!! 01064042430)r   ÚThisr-   ÚwaaayyyÚtooÚmuchÚforÚyouú!r;   r;   r+   z*My number is 06-46124080, except it's not.)	r)   Únumberr-   z06-46124080ú,Úexceptúit'sÚnotú.z+My number is 601-984-4813, except it's not.)	r)   r<   r-   z601-984-4813r=   r>   r?   r@   rA   )
r)   r<   r-   z601-984-Ú4813r=   r>   r?   r@   rA   z/My number is (393)  928 -3010, except it's not.)	r)   r<   r-   ú(393)  928 -3010r=   r>   r?   r@   rA   )r)   r<   r-   ú(Ú393ú)Ú928ú-Ú3010r=   r>   r?   r@   rA   z1The product identification number is 48103284512.)ÚTheÚproductÚidentificationr<   r-   Ú
4810328451Ú2rA   )rJ   rK   rL   r<   r-   Ú48103284512rA   z(My favourite substraction is 240 - 1353.)r)   Ú	favouriteÚsubstractionr-   z
240 - 1353rA   )r)   rP   rQ   r-   Ú240rH   Ú1353rA   Ú
test_inputÚ	expectedsc                 ó”   — t          ddg|¦  «        D ]4\  }}t          dd|¬¦  «        }|                     |¦  «        }||k    sJ ‚Œ5dS )a¦  
        Test `match_phone_numbers` in TweetTokenizer.

        Note that TweetTokenizer is also passed the following for these tests:
            * strip_handles=True
            * reduce_len=True

        :param test_input: The input string to tokenize using TweetTokenizer.
        :type test_input: str
        :param expecteds: A 2-tuple of tokenized sentences. The first of the two
            tokenized is the expected output of tokenization with `match_phone_numbers=True`.
            The second of the two tokenized lists is the expected output of tokenization
            with `match_phone_numbers=False`.
        :type expecteds: Tuple[List[str], List[str]]
        TF)r   r   Úmatch_phone_numbersN)Úzipr	   r!   )r"   rT   rU   rW   r&   r#   Ú	predicteds          r   Útest_tweet_tokenizer_expandedz*TestTokenize.test_tweet_tokenizer_expanded>   sx   € õH .1°$¸°À	Ñ-JÔ-Jð 	)ð 	)Ñ)Ð Ý&Ø"ØØ$7ðñ ô ˆIð
 "×*Ò*¨:Ñ6Ô6ˆIØ Ò(Ð(Ð(Ð(Ð(ð	)ð 	)r(   c                 ó`   — t          ¦   «         }|                     d¦  «        }|g d¢k    sJ ‚dS )ú3
        Test SyllableTokenizer tokenizer.
        Újustification)ÚjusÚtiÚfiÚcaÚtionN©r   r!   )r"   r#   r%   s      r   Ú+test_sonority_sequencing_syllable_tokenizerz8TestTokenize.test_sonority_sequencing_syllable_tokenizer  s@   € õ &Ñ'Ô'ˆ	Ø×#Ò# OÑ4Ô4ˆØÐ:Ð:Ð:Ò:Ð:Ð:Ð:Ð:Ð:r(   c                 óh   — t          ¦   «         }ddz  }|                     |¦  «        }||gk    sJ ‚dS )r\   Ú9i'  Nrc   )r"   r#   r*   r%   s       r   Útest_syllable_tokenizer_numbersz,TestTokenize.test_syllable_tokenizer_numbers  sD   € õ &Ñ'Ô'ˆ	Ø�U‰{ˆØ×#Ò# DÑ)Ô)ˆØ˜$˜ÒÐÐÐÐÐr(   c                 ó–   — ddl m} d}t          |                     ¦   «         ¦  «        }|                     |¦  «        }|g d¢k    sJ ‚dS )z;
        Test LegalitySyllableTokenizer tokenizer.
        r   )r   Ú	wonderful)ÚwonÚderÚfulN)Únltk.corpusr   r   r!   )r"   r   Ú	test_wordr#   r%   s        r   Ú*test_legality_principle_syllable_tokenizerz7TestTokenize.test_legality_principle_syllable_tokenizer  sa   € ð 	&Ð%Ð%Ð%Ð%Ð%àˆ	Ý-¨e¯kªk©m¬mÑ<Ô<ˆ	Ø×#Ò# IÑ.Ô.ˆØÐ.Ð.Ð.Ò.Ð.Ð.Ð.Ð.Ð.r(   c                 óÖ   — t          ¦   «         }|                     d¦  «         d}|                     |                     ¦   «         ¦  «        }|                     ¦   «         g d¢k    sJ ‚dS )zN
        Test the Stanford Word Segmenter for Arabic (default config)
        r   un   ÙŠØ¨Ø­Ø« Ø¹Ù„Ù… Ø§Ù„Ø­Ø§Ø³ÙˆØ¨ Ø§Ø³ØªØ®Ø¯Ø§Ù… Ø§Ù„Ø­ÙˆØ³Ø¨Ø© Ø¨Ø¬Ù…ÙŠØ¹ Ø§Ø´ÙƒØ§Ù„Ù‡Ø§ Ù„Ø­Ù„ Ø§Ù„Ù…Ø´ÙƒÙ„Ø§Øª)u   ÙŠØ¨Ø­Ø«u   Ø¹Ù„Ù…u   Ø§Ù„Ø­Ø§Ø³ÙˆØ¨u   Ø§Ø³ØªØ®Ø¯Ø§Ù…u   Ø§Ù„Ø­ÙˆØ³Ø¨Ø©u   Ø¨u   Ø¬Ù…ÙŠØ¹u
   Ø§Ø´ÙƒØ§Ù„u   Ù‡Ø§u   Ù„u   Ø­Ù„u   Ø§Ù„Ù…Ø´ÙƒÙ„Ø§ØªN©r   r   ÚsegmentÚsplit©r"   r   ÚsentÚsegmented_sents       r   Útest_stanford_segmenter_arabicz+TestTokenize.test_stanford_segmenter_arabic'  s}   € õ
  Ñ!Ô!ˆØ×Ò˜4Ñ Ô Ð ØˆØŸš T§Z¢Z¡\¤\Ñ2Ô2ˆØ×#Ò#Ñ%Ô%ð *
ð *
ð *
ò 
ð 
ð 
ð 
ð 
ð 
r(   c                 óÖ   — t          ¦   «         }|                     d¦  «         d}|                     |                     ¦   «         ¦  «        }|                     ¦   «         g d¢k    sJ ‚dS )zO
        Test the Stanford Word Segmenter for Chinese (default config)
        r   u$   è¿™æ˜¯æ–¯å�¦ç¦�ä¸­æ–‡åˆ†è¯�å™¨æµ‹è¯•)u   è¿™u   æ˜¯u	   æ–¯å�¦ç¦�u   ä¸­æ–‡u	   åˆ†è¯�å™¨u   æµ‹è¯•Nrq   rt   s       r   Útest_stanford_segmenter_chinesez,TestTokenize.test_stanford_segmenter_chinese?  s}   € õ
  Ñ!Ô!ˆØ×Ò˜4Ñ Ô Ð Ø5ˆØŸš T§Z¢Z¡\¤\Ñ2Ô2ˆØ×#Ò#Ñ%Ô%ð *
ð *
ð *
ò 
ð 
ð 
ð 
ð 
ð 
r(   c                 ó¬   — t          ¦   «         }d}dg}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚dS )zT
        Test a string that resembles a phone number but contains a newline
        rC   z(393)
928 -3010)rD   rE   rF   z	928 -3010Nr    )r"   r#   Útest1r&   ÚresultÚtest2s         r   Útest_phone_tokenizerz!TestTokenize.test_phone_tokenizerQ  s|   € õ #Ñ$Ô$ˆ	Ø"ˆØ&Ð'ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð #ˆØ1Ð1Ð1ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!Ð!Ð!r(   c                 óÂ  — t          ¦   «         }d}dg}|                     |¦  «        }||k    sJ ‚d}dg}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d	}	g d
¢}|                     |	¦  «        }||k    sJ ‚dS )zX
        Test a string that contains Emoji ZWJ Sequences and skin tone modifier
        u   ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘§â€�ðŸ‘§u   ðŸ‘¨ðŸ�¿uŒ   ðŸ¤” ðŸ™ˆ me asÃ­, seðŸ˜Œ ds ðŸ’•ðŸ‘­ðŸ‘™ hello ðŸ‘©ðŸ�¾â€�ðŸŽ“ emoji hello ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘¦â€�ðŸ‘¦ how are ðŸ˜Š you todayðŸ™…ðŸ�½ðŸ™…ðŸ�½)u   ðŸ¤”u   ðŸ™ˆÚmeu   asÃ­r=   Úseu   ðŸ˜ŒÚdsu   ðŸ’•u   ðŸ‘­u   ðŸ‘™Úhellou   ðŸ‘©ðŸ�¾â€�ðŸŽ“Úemojirƒ   u   ðŸ‘¨â€�ðŸ‘©â€�ðŸ‘¦â€�ðŸ‘¦ÚhowÚareu   ðŸ˜Šr:   Útodayõ   ðŸ™…ðŸ�½rˆ   u   ðŸ‡¦ðŸ‡µðŸ‡µðŸ‡±ðŸ‡ª)u   ðŸ‡¦ðŸ‡µõ   ðŸ‡µðŸ‡±u   ðŸ‡ªu   Hi ðŸ‡¨ðŸ‡¦, ðŸ˜�!!)ÚHiõ   ðŸ‡¨ðŸ‡¦r=   u   ðŸ˜�r;   r;   u   <3 ðŸ‡¨ðŸ‡¦ ðŸ¤� ðŸ‡µðŸ‡± <3)ú<3r‹   u   ðŸ¤�r‰   rŒ   Nr    )
r"   r#   r{   r&   r|   r}   Útest3Útest4Útest5Útest6s
             r   Útest_emoji_tokenizerz!TestTokenize.test_emoji_tokenizerd  sW  € õ #Ñ$Ô$ˆ	ð ,ˆØ/Ð0ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð ˆØ�<ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð _ˆð
ð 
ð 
ˆð2 ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð 'ˆØ3Ð3Ð3ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!à%ˆØ<Ð<Ð<ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!à.ˆØ?Ð?Ð?ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!Ð!Ð!r(   c                 ó<   — d}g d¢}t          |¦  «        |k    sJ ‚dS )zA
        Test padding of asterisk for word tokenization.
        z1This is a, *weird sentence with *asterisks in it.)r5   r-   Úar=   Ú*ÚweirdÚsentenceÚwithr”   Ú	asterisksÚinÚitrA   N©r   ©r"   r*   r&   s      r   Útest_pad_asteriskzTestTokenize.test_pad_asterisk¥  s<   € ð Cˆð
ð 
ð 
ˆõ ˜TÑ"Ô" hÒ.Ð.Ð.Ð.Ð.Ð.r(   c                 ó<   — d}g d¢}t          |¦  «        |k    sJ ‚dS )z@
        Test padding of dotdot* for word tokenization.
        zPWhy did dotdot.. not get tokenized but dotdotdot... did? How about manydots.....)ÚWhyÚdidÚdotdotz..r@   ÚgetÚ	tokenizedÚbutÚ	dotdotdotz...r    ú?ÚHowÚaboutÚmanydotsz.....Nr›   rœ   s      r   Útest_pad_dotdotzTestTokenize.test_pad_dotdot»  s<   € ð bˆð
ð 
ð 
ˆõ$ ˜TÑ"Ô" hÒ.Ð.Ð.Ð.Ð.Ð.r(   c                 ó  — t          d¬¦  «        }d}g d¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d	}g d
¢}|                     |¦  «        }||k    sJ ‚d}g d¢}|                     |¦  «        }||k    sJ ‚d}	g d¢}|                     |	¦  «        }||k    sJ ‚d}
g d¢}|                     |
¦  «        }||k    sJ ‚dS )zW
        Test remove_handle() from casual.py with specially crafted edge cases
        T)r   z-@twitter hello @twi_tter_. hi @12345 @123news)rƒ   rA   Úhiu]   @n`@n~@n(@n)@n-@n=@n+@n\@n|@n[@n]@n{@n}@n;@n:@n'@n"@n/@n?@n.@n,@n<@n>@n @n
@n Ã±@n.Ã¼@n.Ã§@n.)ú`ú~rD   rF   rH   ú=ú+ú\ú|ú[ú]Ú{Ú}ú;r   ú'ú"ú/r¦   rA   r=   ú<ú>õ   Ã±rA   õ   Ã¼rA   õ   Ã§rA   zKa@n j@n z@n A@n L@n Z@n 1@n 4@n 7@n 9@n 0@n _@n !@n @@n #@n $@n %@n &@n *@n)&r“   ú@nÚjrÀ   ÚzrÀ   ÚArÀ   ÚLrÀ   ÚZrÀ   Ú1rÀ   Ú4rÀ   Ú7rÀ   rf   rÀ   Ú0rÀ   Ú_rÀ   r;   rÀ   ú@rÀ   ú#rÀ   ú$rÀ   ú%rÀ   ú&rÀ   r”   rÀ   z@n!a @n#a @n$a @n%a @n&a @n*a)r;   r“   rÌ   r“   rÍ   r“   rÎ   r“   rÏ   r“   r”   r“   zD@n!@n @n#@n @n$@n @n%@n @n&@n @n*@n @n@n @@n @n@@n @n_@n @n7@n @nj@n)r;   rÀ   rÌ   rÀ   rÍ   rÀ   rÎ   rÀ   rÏ   rÀ   r”   rÀ   rÀ   rÀ   rË   rÀ   rÀ   rË   rÀ   z@n_rÀ   z@n7rÀ   z@njrÀ   z^@abcdefghijklmnopqrstuvwxyz @abcdefghijklmno1234 @abcdefghijklmno_ @abcdefghijklmnoendofhandle)ÚpqrstuvwxyzÚ1234rÊ   Úendofhandlez^@abcdefghijklmnop@abcde @abcdefghijklmno@abcde @abcdefghijklmno_@abcde @abcdefghijklmno5@abcde)Úpú@abcdez@abcdefghijklmnorÔ   rÊ   rÔ   Ú5rÔ   Nr    )r"   r#   r{   r&   r|   r}   r�   rŽ   r�   r�   Útest7s              r   Útest_remove_handlezTestTokenize.test_remove_handleÔ  s»  € õ
 #°Ð6Ñ6Ô6ˆ	ð @ˆØ'Ð'Ð'ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð sˆð
ð 
ð 
ˆð> ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð ^ˆð'
ð '
ð '
ˆðP ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð 0ˆØOÐOÐOˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð Wˆð
ð 
ð 
ˆð6 ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð qˆØ>Ð>Ð>ˆØ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!ð qˆð	
ð 	
ð 	
ˆð ×#Ò# EÑ*Ô*ˆØ˜Ò!Ð!Ð!Ð!Ð!Ð!r(   c                 óB  — t          ¦   «         }d}g d¢}t          |                     |¦  «        ¦  «        }||k    sJ ‚d}g d¢}t          |                     |¦  «        ¦  «        }||k    sJ ‚d}g d¢}t          |                     |¦  «        ¦  «        }||k    sJ ‚dS )zC
        Test TreebankWordTokenizer.span_tokenize function
        zNGood muffins cost $3.88
in New (York).  Please (buy) me
two of them.
(Thanks).))r   é   )é   é   )é   é   )é   é   )rß   é   )é   é   )é   é   )é   é    )ræ   é$   )rç   é%   )rè   é&   )é(   é.   )é/   é0   )rí   é3   )rî   é4   )é5   é7   )é8   é;   )é<   é>   )é?   éD   )éE   éF   )rù   éL   )rú   éM   )rû   éN   zmThe DUP is similar to the "religious right" in the United States and takes a hardline stance on social issues)©r   é   ©rÙ   é   ©é   é
   ©é   rÞ   ©rß   é   ©é   é   ©râ   rã   ©rã   rç   ©rè   é*   ©r  é+   ©é,   rë   ©rì   é2   ©rî   é9   ©é:   é@   ©éA   r÷   ©rø   éJ   ©éK   rú   )rû   éU   )éV   é\   )é]   é_   )é`   éf   )ég   ém   zqThe DUP is similar to the "religious right" in the United States and takes a ``hardline'' stance on social issues)rý   rÿ   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )rû   éO   )r)  éW   )r*  éY   )éZ   r%  )éa   éc   )éd   éj   )ék   éq   N)r   ÚlistÚspan_tokenize)r"   r#   r{   r&   r|   r}   r�   s          r   Útest_treebank_span_tokenizerz)TestTokenize.test_treebank_span_tokenizerm  sê   € õ
 *Ñ+Ô+ˆ	ð dˆð
ð 
ð 
ˆõ2 �i×-Ò-¨eÑ4Ô4Ñ5Ô5ˆØ˜Ò!Ð!Ð!Ð!ð @ˆð
ð 
ð 
ˆõ0 �i×-Ò-¨eÑ4Ô4Ñ5Ô5ˆØ˜Ò!Ð!Ð!Ð!ð Fˆð
ð 
ð 
ˆõ4 �i×-Ò-¨eÑ4Ô4Ñ5Ô5ˆØ˜Ò!Ð!Ð!Ð!Ð!Ð!r(   c                 ór   — d}g d¢}t          |¦  «        |k    sJ ‚d}g d¢}t          |¦  «        |k    sJ ‚dS )z-
        Test word_tokenize function
        z0The 'v', I've been fooled but I'll seek revenge.)rJ   r¸   Úvr¸   r=   ÚIz'veÚbeenÚfooledr¤   r8  z'llÚseekÚrevengerA   z'v' 're')r¸   r7  r¸   z'rer¸   Nr›   )r"   r–   r&   s      r   Útest_word_tokenizezTestTokenize.test_word_tokenizeÎ  se   € ð
 Fˆð
ð 
ð 
ˆõ" ˜XÑ&Ô&¨(Ò2Ð2Ð2Ð2àˆØ.Ð.Ð.ˆÝ˜XÑ&Ô&¨(Ò2Ð2Ð2Ð2Ð2Ð2r(   c                 ó„   — dddgfdg d¢fdg d¢fg}|D ]+\  }}d„ t          j        |¦  «        D ¦   «         }||k    sJ ‚Œ,d S )	NÚ12©rÆ   rN   )rN   NÚ123)r@  ©rN   Ú3)rC  NrÑ   )r@  rB  )rC  rÇ   )rÇ   Nc                 ó   — g | ]}|‘ŒS © rE  )Ú.0Úxs     r   ú
<listcomp>z5TestTokenize.test_punkt_pair_iter.<locals>.<listcomp>ó  s   € ÐEÐEÐE 1˜QÐEÐEÐEr(   )r
   Ú
_pair_iter)r"   Ú
test_casesrT   Úexpected_outputÚactual_outputs        r   Útest_punkt_pair_iterz!TestTokenize.test_punkt_pair_iterë  s†   € à�J Ð,Ð-ØÐ9Ð9Ð9Ð:ØÐFÐFÐFÐGð
ˆ
ð ,6ð 	4ð 	4Ñ'ˆJ˜ØEÐE­Ô(8¸Ñ(DÔ(DÐEÑEÔEˆMà  OÒ3Ð3Ð3Ð3Ð3ð	4ð 	4r(   c                 ój   — t          g ¦  «        }t          j        |¦  «        }t          |¦  «         d S ©N)Úiterr
   rI  r3  )r"   rš   Úgens      r   Ú5test_punkt_pair_iter_handles_stop_iteration_exceptionzBTestTokenize.test_punkt_pair_iter_handles_stop_iteration_exception÷  s,   € å�"‰XŒXˆåÔ˜rÑ"Ô"ˆåˆS‰	Œ	ˆ	ˆ	ˆ	r(   c                 ó¨   — t          j        ¦   «         } G d„ d¦  «        } |¦   «         |_        t          |                     d¦  «        ¦  «         d S )Nc                   ó   — e Zd Zd„ ZdS )úkTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exception.<locals>.TestPunktTokenizeWordsMockc                 ó    — t          g ¦  «        S rO  )rP  )r"   Úss     r   r   zyTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exception.<locals>.TestPunktTokenizeWordsMock.word_tokenize  s   € Ý˜B‘x”x�r(   N)Ú__name__Ú
__module__Ú__qualname__r   rE  r(   r   ÚTestPunktTokenizeWordsMockrU    s#   € € € € € ð ð  ð  ð  ð  r(   r[  r   )r
   ÚPunktBaseClassÚ
_lang_varsr3  Ú_tokenize_words)r"   Úobjr[  s      r   Ú:test_punkt_tokenize_words_handles_stop_iteration_exceptionzGTestTokenize.test_punkt_tokenize_words_handles_stop_iteration_exceptionÿ  sl   € ÝÔ"Ñ$Ô$ˆð	 ð 	 ð 	 ð 	 ð 	 ñ 	 ô 	 ð 	 ð 4Ð3Ñ5Ô5ˆŒåˆS× Ò  Ñ(Ô(Ñ)Ô)Ð)Ð)Ð)r(   c                 ó²   —  G d„ dt           j        ¦  «        }t          j         |¦   «         ¬¦  «        }d}g d¢}|                     |¦  «        |k    sJ ‚d S )Nc                   ó   — e Zd ZdZdS )úNTestTokenize.test_punkt_tokenize_custom_lang_vars.<locals>.BengaliLanguageVars)rA   r¦   r;   u   à¥¤N©rX  rY  rZ  Úsent_end_charsrE  r(   r   ÚBengaliLanguageVarsrc    s   € € € € € Ø6ˆNˆNˆNr(   rf  )Ú	lang_varsõc  à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¶à§�à¦°à§€ à¦�à¦® à¦­à§‡à¦™à§�à¦•à¦¾à¦‡à§Ÿà¦¾ à¦¨à¦¾à¦‡à¦¡à§� à¦¸à§‹à¦®à¦¬à¦¾à¦° à¦†à¦‡ à¦†à¦‡ à¦Ÿà¦¿ à¦¦à¦¿à¦²à§�à¦²à¦¿à¦° à¦¹à§€à¦°à¦• à¦œà§Ÿà¦¨à§�à¦¤à§€ à¦‰à¦¦à¦¯à¦¾à¦ªà¦¨à§‡à¦° à¦‰à¦¦à§�à¦¬à§‹à¦§à¦¨ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤ à¦…à¦¨à¦²à¦¾à¦‡à¦¨à§‡à¦° à¦®à¦¾à¦§à§�à¦¯à¦®à§‡ à¦�à¦‡ à¦…à¦¨à§�à¦·à§�à¦ à¦¾à¦¨à§‡ à¦•à§‡à¦¨à§�à¦¦à§�à¦°à§€à§Ÿ à¦®à¦¾à¦¨à¦¬ à¦¸à¦®à§�à¦ªà¦¦ à¦‰à¦¨à§�à¦¨à§Ÿà¦¨à¦®à¦¨à§�à¦¤à§�à¦°à§€ à¦¶à§�à¦°à§€ à¦°à¦®à§‡à¦¶ à¦ªà§‹à¦–à¦°à¦¿à§Ÿà¦¾à¦² â€˜à¦¨à¦¿à¦¶à¦¾à¦™à§�à¦•â€™  à¦‰à¦ªà¦¸à§�à¦¥à¦¿à¦¤ à¦›à¦¿à¦²à§‡à¦¨à¥¤ à¦�à¦‡ à¦‰à¦ªà¦²à¦•à§�à¦·à§�à¦¯à§‡ à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¹à§€à¦°à¦•à¦œà§Ÿà¦¨à§�à¦¤à§€à¦° à¦²à§‹à¦—à§‹ à¦�à¦¬à¦‚ à§¨à§¦à§©à§¦-à¦�à¦° à¦œà¦¨à§�à¦¯ à¦ªà§�à¦°à¦¤à¦¿à¦·à§�à¦ à¦¾à¦¨à§‡à¦° à¦²à¦•à§�à¦·à§�à¦¯ à¦“ à¦ªà¦°à¦¿à¦•à¦²à§�à¦ªà¦¨à¦¾à¦° à¦¨à¦¥à¦¿ à¦ªà§�à¦°à¦•à¦¾à¦¶ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤)u  à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¶à§�à¦°à§€ à¦�à¦® à¦­à§‡à¦™à§�à¦•à¦¾à¦‡à§Ÿà¦¾ à¦¨à¦¾à¦‡à¦¡à§� à¦¸à§‹à¦®à¦¬à¦¾à¦° à¦†à¦‡ à¦†à¦‡ à¦Ÿà¦¿ à¦¦à¦¿à¦²à§�à¦²à¦¿à¦° à¦¹à§€à¦°à¦• à¦œà§Ÿà¦¨à§�à¦¤à§€ à¦‰à¦¦à¦¯à¦¾à¦ªà¦¨à§‡à¦° à¦‰à¦¦à§�à¦¬à§‹à¦§à¦¨ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤u+  à¦…à¦¨à¦²à¦¾à¦‡à¦¨à§‡à¦° à¦®à¦¾à¦§à§�à¦¯à¦®à§‡ à¦�à¦‡ à¦…à¦¨à§�à¦·à§�à¦ à¦¾à¦¨à§‡ à¦•à§‡à¦¨à§�à¦¦à§�à¦°à§€à§Ÿ à¦®à¦¾à¦¨à¦¬ à¦¸à¦®à§�à¦ªà¦¦ à¦‰à¦¨à§�à¦¨à§Ÿà¦¨à¦®à¦¨à§�à¦¤à§�à¦°à§€ à¦¶à§�à¦°à§€ à¦°à¦®à§‡à¦¶ à¦ªà§‹à¦–à¦°à¦¿à§Ÿà¦¾à¦² â€˜à¦¨à¦¿à¦¶à¦¾à¦™à§�à¦•â€™  à¦‰à¦ªà¦¸à§�à¦¥à¦¿à¦¤ à¦›à¦¿à¦²à§‡à¦¨à¥¤u/  à¦�à¦‡ à¦‰à¦ªà¦²à¦•à§�à¦·à§�à¦¯à§‡ à¦‰à¦ªà¦°à¦¾à¦·à§�à¦Ÿà§�à¦°à¦ªà¦¤à¦¿ à¦¹à§€à¦°à¦•à¦œà§Ÿà¦¨à§�à¦¤à§€à¦° à¦²à§‹à¦—à§‹ à¦�à¦¬à¦‚ à§¨à§¦à§©à§¦-à¦�à¦° à¦œà¦¨à§�à¦¯ à¦ªà§�à¦°à¦¤à¦¿à¦·à§�à¦ à¦¾à¦¨à§‡à¦° à¦²à¦•à§�à¦·à§�à¦¯ à¦“ à¦ªà¦°à¦¿à¦•à¦²à§�à¦ªà¦¨à¦¾à¦° à¦¨à¦¥à¦¿ à¦ªà§�à¦°à¦•à¦¾à¦¶ à¦•à¦°à§‡à¦›à§‡à¦¨à¥¤)r
   ÚPunktLanguageVarsÚPunktSentenceTokenizerr!   )r"   rf  r_  Ú	sentencesr&   s        r   Ú$test_punkt_tokenize_custom_lang_varsz1TestTokenize.test_punkt_tokenize_custom_lang_vars
  sŒ   € ð	7ð 	7ð 	7ð 	7ð 	7¥%Ô"9ñ 	7ô 	7ð 	7õ Ô*Ð5HÐ5HÑ5JÔ5JÐKÑKÔKˆð zˆ	ð
ð 
ð 
ˆð �|Š|˜IÑ&Ô&¨(Ò2Ð2Ð2Ð2Ð2Ð2r(   c                 ól   — t          j        ¦   «         }d}dg}|                     |¦  «        |k    sJ ‚d S )Nrh  )r
   rj  r!   )r"   r_  rk  r&   s       r   Ú'test_punkt_tokenize_no_custom_lang_varsz4TestTokenize.test_punkt_tokenize_no_custom_lang_vars  sJ   € ÝÔ*Ñ,Ô,ˆð zˆ	ð rð
ˆð �|Š|˜IÑ&Ô&¨(Ò2Ð2Ð2Ð2Ð2Ð2r(   z%input_text,n_sents,n_splits,lang_vars))z4Subject: Some subject. Attachments: Some attachmentsé   é   )z4Subject: Some subject! Attachments: Some attachmentsro  rp  )z4This is just a normal sentence, just like any other.rp  r   Nc                 ó   — t          j        ¦   «         }|d k    r||_        t          |                     |¦  «        ¦  «        |k    sJ ‚t          t          |                     |¦  «        ¦  «        ¦  «        |k    sJ ‚d S rO  )r
   rj  r]  Úlenr!   r3  Údebug_decisions)r"   Ú
input_textÚn_sentsÚn_splitsrg  r#   s         r   Úpunkt_debug_decisionsz"TestTokenize.punkt_debug_decisions&  s�   € õ$ Ô0Ñ2Ô2ˆ	Ø˜ÒÐØ#,ˆIÔ å�9×%Ò% jÑ1Ô1Ñ2Ô2°gÒ=Ð=Ð=Ð=Ý•4˜	×1Ò1°*Ñ=Ô=Ñ>Ô>Ñ?Ô?À8ÒKÐKÐKÐKÐKÐKr(   c                 óx   —  G d„ dt           j        ¦  «        }|                      ddd |¦   «         ¬¦  «         d S )Nc                   ó   — e Zd ZdZdS )úGTestTokenize.test_punkt_debug_decisions_custom_end.<locals>.ExtLangVars)rA   r¦   r;   ú^Nrd  rE  r(   r   ÚExtLangVarsrz  B  s   € € € € € Ø1ˆNˆNˆNr(   r|  z4Subject: Some subject^ Attachments: Some attachmentsro  rp  )ru  rv  rg  )r
   ri  rw  )r"   r|  s     r   Ú%test_punkt_debug_decisions_custom_endz2TestTokenize.test_punkt_debug_decisions_custom_end?  sj   € ð	2ð 	2ð 	2ð 	2ð 	2�%Ô1ñ 	2ô 	2ð 	2ð 	×"Ò"ØBØØØ!�k‘m”mð	 	#ñ 	
ô 	
ð 	
ð 	
ð 	
r(   zsentences, expectedzthis is a test. . new sentence.)zthis is a test.rA   znew sentence.zThis. . . That)úThis.rA   rA   ÚThatzThis..... ThatzThis... ThatzThis.. . ThatzThis.. .r  zThis. .. Thatr~  z.. ThatzThis. ,. That)r~  z,.r  zThis!!! ThatzThis!!!z
This! ThatzThis!z+1. This is R .
2. This is A .
3. That's all)z1.zThis is R .z2.zThis is A .z3.z
That's allz+1. This is R .	2. This is A .	3. That's allzHello.	TherezHello.ÚThererk  r&   c                 ó0   — t          |¦  «        |k    sJ ‚d S rO  )r   )r"   rk  r&   s      r   Útest_sent_tokenizezTestTokenize.test_sent_tokenizeN  s#   € õ6 ˜YÑ'Ô'¨8Ò3Ð3Ð3Ð3Ð3Ð3r(   Úreturnc                 óÊ   — d}t          ¦   «         }|                     |¦  «        t          |¦  «        k    sJ ‚t          |                     |¦  «        ¦  «        g d¢k    sJ ‚d S )NzHello there))r   rp  )rp  ro  )ro  rþ   )rþ   rÙ   )rÙ   rÚ   )rÚ   é   )r…  r   )r   r  )r  é	   )r†  r  )r  r  )r   r!   r3  r4  )r"   r–   r#   s      r   Útest_string_tokenizerz"TestTokenize.test_string_tokenizerk  s{   € Ø ˆÝ!‘O”Oˆ	Ø×!Ò! (Ñ+Ô+­t°H©~¬~Ò=Ð=Ð=Ð=Ý�I×+Ò+¨HÑ5Ô5Ñ6Ô6ð ;
ð ;
ð ;
ò 
ð 
ð 
ð 
ð 
ð 
r(   rO  ©rƒ  N)!rX  rY  rZ  r'   ÚpytestÚmarkÚparametrizeÚstrr   r   rZ   rd   rg   ro   Úcheck_stanford_segmenterrw   ry   r~   r‘   r�   rª   r×   r5  r=  rM  rR  r`  rl  rn  rw  r}  r‚  r‡  rE  r(   r   r   r   '   se  € € € € € ð"ð "ð "ð, „[×ÒØð 6àNÐNÐNØJÐJÐJððð 0àEÐEÐEØAÐAÐAððð Oðð ð ðð ð ðð ðL =ð
ð 
ð 
ð
ð 
ð 
ððð< >ð
ð 
ð 
ðð ð ððð> Bð
ð 
ð 
ðð ð ðð ðF Dð	ð 	ð 	ðð ð ððð6 ;àPÐPÐPØVÐVÐVðððMn	
ñqô qðd)Øð)Ø*/°°S´	¸4À¼9Ð0DÔ*Eð)ð )ð )ñeqô qðd)ð6;ð ;ð ;ð ð  ð  ð	/ð 	/ð 	/ð ð
ð 
ñ Ôð
ð. ð
ð 
ñ Ôð
ð""ð "ð "ð&?"ð ?"ð ?"ðB/ð /ð /ð,/ð /ð /ð2W"ð W"ð W"ðr_"ð _"ð _"ðB3ð 3ð 3ð:
4ð 
4ð 
4ðð ð ð	*ð 	*ð 	*ð3ð 3ð 3ð"	3ð 	3ð 	3ð „[×ÒØ/ð	
ð 	
ð 	
ñô ð"Lð Lð Lñ#ô ð"Lð
ð 
ð 
ð „[×ÒØð 2Ø9Ð9Ð9ðð Ð:Ð:Ð:Ð;ØÐ 0Ð1Ð2Ø˜nÐ-Ð.Ø˜z¨6Ð2Ð3Ø˜w¨	Ð2Ð3ØÐ5Ð5Ð5Ð6Ø˜i¨Ð0Ð1Ø˜G VÐ,Ð-à?ØNÐNÐNðð
 @ØNÐNÐNðð ˜x¨Ð1Ð2ð+	
ñô ð44¨Cð 4¸4À¼9ð 4ð 4ð 4ñ5ô ð44ð
ð 
ð 
ð 
ð 
ð 
r(   r   c                   ó&   — e Zd Zdd„Zdd„Zdd„ZdS )ÚTestPunktTrainerrƒ  Nc                 óV   — t          j        ¦   «         }|                     d¦  «         d S )NzThis is a test.©r
   ÚPunktTrainerÚtrain©r"   Útrainers     r   Útest_punkt_trainz!TestPunktTrainer.test_punkt_train  s)   € ÝÔ$Ñ&Ô&ˆØ�ŠÐ'Ñ(Ô(Ð(Ð(Ð(r(   c                 óV   — t          j        ¦   «         }|                     d¦  «         d S )Nr~  r‘  r”  s     r   Útest_punkt_train_single_wordz-TestPunktTrainer.test_punkt_train_single_wordƒ  s(   € ÝÔ$Ñ&Ô&ˆØ�Š�gÑÔÐÐÐr(   c                 óV   — t          j        ¦   «         }|                     d¦  «         d S )NzThis is a testr‘  r”  s     r   Útest_punkt_train_no_puncz)TestPunktTrainer.test_punkt_train_no_punc‡  s)   € ÝÔ$Ñ&Ô&ˆØ�ŠÐ&Ñ'Ô'Ð'Ð'Ð'r(   rˆ  )rX  rY  rZ  r–  r˜  rš  rE  r(   r   r�  r�  ~  sP   € € € € € ð)ð )ð )ð )ðð ð ð ð(ð (ð (ð (ð (ð (r(   r�  )Ú__doc__Útypingr   r   r‰  Únltk.tokenizer   r   r   r   r	   r
   r   r   Únltk.tokenize.simpler   r   rŠ  Úskipifr�  r   r�  rE  r(   r   ú<module>r      sM  ððð ð
 Ð Ð Ð Ð Ð Ð Ð à €€€ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð /Ð .Ð .Ð .Ð .Ð .ðð ð ð "œ;×-Ò-ØÐÑ!Ô!Ð!Ø<ð .ñ ô Ð ðT
ð T
ð T
ð T
ð T
ñ T
ô T
ð T
ðn(ð (ð (ð (ð (ñ (ô (ð (ð (ð (r(   