§
    è˜ºhÏÅ  ã                  ó$  — d Z ddlmZ ddlZddlZddlZddlZddlmZm	Z	 ddl
mZ ddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z! ddl"Z"dd	l#m$Z$m%Z%  ej&        e'¦  «        Z( ed
d¬¦  «        Z)ddœdAd„Z*dBd„Z+ G d„ de$e¦  «        Z, G d„ de,¦  «        Z- G d„ d e¦  «        Z. G d!„ d"e¦  «        Z/ G d#„ d$¦  «        Z0 G d%„ d&e¦  «        Z1 G d'„ d(¦  «        Z2 ed)¬*¦  «         G d+„ d,¦  «        ¦   «         Z3dCd.„Z4 G d/„ d0e,¦  «        Z5 G d1„ d2e,¦  «        Z6 G d3„ d4e7e¦  «        Z8 G d5„ d6e,¦  «        Z9 G d7„ d8e,¦  «        Z: G d9„ d:e,¦  «        Z; G d;„ d<e9¦  «        Z< G d=„ d>e9¦  «        Z= G d?„ d@e9¦  «        Z>dS )Daí  **Text Splitters** are classes for splitting text.


**Class hierarchy:**

.. code-block::

    BaseDocumentTransformer --> TextSplitter --> <name>TextSplitter  # Example: CharacterTextSplitter
                                                 RecursiveCharacterTextSplitter -->  <name>TextSplitter

Note: **MarkdownHeaderTextSplitter** and **HTMLHeaderTextSplitter do not derive from TextSplitter.


**Main helpers:**

.. code-block::

    Document, Tokenizer, Language, LineType, HeaderType

é    )ÚannotationsN)ÚABCÚabstractmethod)Ú	dataclass)ÚEnum)ÚBytesIOÚStringIO)ÚAbstractSetÚAnyÚCallableÚ
CollectionÚDictÚIterableÚListÚLiteralÚOptionalÚSequenceÚTupleÚTypeÚ	TypedDictÚTypeVarÚUnionÚcast)ÚBaseDocumentTransformerÚDocumentÚTSÚTextSplitter)Úboundé@B ©Ú
max_lengthÚpipelineÚstrr!   ÚintÚreturnr   c               óä   — 	 dd l }n# t          $ r t          d¦  «        ‚w xY w| dk    r&ddlm}  |¦   «         }|                     d¦  «         n |                     | ddg¬¦  «        }||_        |S )Nr   zCSpacy is not installed, please install it with `pip install spacy`.Úsentencizer)ÚEnglishÚnerÚtagger)Úexclude)ÚspacyÚImportErrorÚspacy.lang.enr(   Úadd_pipeÚloadr!   )r"   r!   r,   r(   r'   s        ú_/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/langchain/text_splitter.pyÚ"_make_spacy_pipeline_for_splittingr2   ;   s«   € ð
ØˆˆˆˆøÝð 
ð 
ð 
ÝØQñ
ô 
ð 	
ð
øøøð �=Ò Ð Ø)Ð)Ð)Ð)Ð)Ð)à�g‘i”iˆØ×Ò˜]Ñ+Ô+Ð+Ð+à—j’j °E¸8Ð3D�jÑEÔEˆØ!+ˆÔØÐs   ‚ ‡!ÚtextÚ	separatorÚkeep_separatorÚboolú	List[str]c                óT  ‡— |r‹|rst          j        d|› d�| ¦  «        Šˆfd„t          dt          ‰¦  «        d¦  «        D ¦   «         }t          ‰¦  «        dz  dk    r|‰dd …         z  }‰d         g|z   }n%t          j        || ¦  «        }nt	          | ¦  «        }d„ |D ¦   «         S )	Nú(ú)c                ó8   •— g | ]}‰|         ‰|d z            z   ‘ŒS )é   © )Ú.0ÚiÚ_splitss     €r1   ú
<listcomp>z*_split_text_with_regex.<locals>.<listcomp>W   s*   ø€ ÐUÐUÐU°a�g˜a”j 7¨1¨q©5¤>Ñ1ÐUÐUÐUó    r<   é   r   éÿÿÿÿc                ó   — g | ]
}|d k    ¯|‘ŒS )Ú r=   ©r>   Úss     r1   rA   z*_split_text_with_regex.<locals>.<listcomp>_   s   € Ð)Ð)Ð)�!  b¢ ˆA   rB   )ÚreÚsplitÚrangeÚlenÚlist)r3   r4   r5   Úsplitsr@   s       @r1   Ú_split_text_with_regexrO   O   sÊ   ø€ ð ð Øð 	/å”hÐ/ 9Ð/Ð/Ð/°Ñ6Ô6ˆGØUÐUÐUÐU½5ÀÅCÈÁLÄLÐRSÑ;TÔ;TÐUÑUÔUˆFÝ�7‰|Œ|˜aÑ 1Ò$Ð$Ø˜' " # #œ,Ñ&�Ø˜a”j�\ FÑ*ˆFˆFå”X˜i¨Ñ.Ô.ˆFˆFå�d‘”ˆØ)Ð)�vÐ)Ñ)Ô)Ð)rB   c                  ó¶   — e Zd ZdZddedddfd8d„Zed9d„¦   «         Z	 d:d;d„Zd<d„Z	d=d"„Z
d>d%„Zed?d)„¦   «         Zed*d e¦   «         d+fd@d5„¦   «         ZdAd7„ZdS )Br   z)Interface for splitting text into chunks.i   éÈ   FTÚ
chunk_sizer$   Úchunk_overlapÚlength_functionúCallable[[str], int]r5   r6   Úadd_start_indexÚstrip_whitespacer%   ÚNonec                ó’   — ||k    rt          d|› d|› d�¦  «        ‚|| _        || _        || _        || _        || _        || _        dS )a  Create a new TextSplitter.

        Args:
            chunk_size: Maximum size of chunks to return
            chunk_overlap: Overlap in characters between chunks
            length_function: Function that measures the length of given chunks
            keep_separator: Whether to keep the separator in the chunks
            add_start_index: If `True`, includes chunk's start index in metadata
            strip_whitespace: If `True`, strips whitespace from the start and end of
                              every document
        zGot a larger chunk overlap (z) than chunk size (z), should be smaller.N)Ú
ValueErrorÚ_chunk_sizeÚ_chunk_overlapÚ_length_functionÚ_keep_separatorÚ_add_start_indexÚ_strip_whitespace)ÚselfrR   rS   rT   r5   rV   rW   s          r1   Ú__init__zTextSplitter.__init__e   s|   € ð( ˜:Ò%Ð%Ýð6¨}ð 6ð 6Øð6ð 6ð 6ñô ð ð &ˆÔØ+ˆÔØ /ˆÔØ-ˆÔØ /ˆÔØ!1ˆÔÐÐrB   r3   r#   r7   c                ó   — dS )z$Split text into multiple components.Nr=   ©ra   r3   s     r1   Ú
split_textzTextSplitter.split_text…   s   € € € rB   NÚtextsÚ	metadatasúOptional[List[dict]]úList[Document]c                ó\  — |pi gt          |¦  «        z  }g }t          |¦  «        D ]„\  }}d}|                      |¦  «        D ]g}t          j        ||         ¦  «        }	| j        r|                     ||dz   ¦  «        }||	d<   t          ||	¬¦  «        }
|                     |
¦  «         ŒhŒ…|S )z&Create documents from a list of texts.rD   r<   Ústart_index©Úpage_contentÚmetadata)	rL   Ú	enumeratere   ÚcopyÚdeepcopyr_   Úfindr   Úappend)ra   rf   rg   Ú
_metadatasÚ	documentsr?   r3   ÚindexÚchunkrn   Únew_docs              r1   Úcreate_documentszTextSplitter.create_documents‰   sÐ   € ð Ð3 2 $­¨U©¬Ñ"3ˆ
Øˆ	Ý  Ñ'Ô'ð 	*ð 	*‰GˆAˆtØˆEØŸš¨Ñ.Ô.ð *ð *�Ýœ=¨°A¬Ñ7Ô7�ØÔ(ð 4Ø ŸIšI e¨U°Q©YÑ7Ô7�EØ.3�H˜]Ñ+Ý"°ÀÐIÑIÔI�Ø× Ò  Ñ)Ô)Ð)Ð)ð*ð ÐrB   ru   úIterable[Document]c                óª   — g g }}|D ]6}|                      |j        ¦  «         |                      |j        ¦  «         Œ7|                      ||¬¦  «        S )zSplit documents.)rg   )rs   rm   rn   ry   )ra   ru   rf   rg   Údocs        r1   Úsplit_documentszTextSplitter.split_documentsš   sa   € à˜rˆyˆØð 	+ð 	+ˆCØ�LŠL˜Ô)Ñ*Ô*Ð*Ø×Ò˜Sœ\Ñ*Ô*Ð*Ð*Ø×$Ò$ U°iÐ$Ñ@Ô@Ð@rB   Údocsr4   úOptional[str]c                óv   — |                      |¦  «        }| j        r|                     ¦   «         }|dk    rd S |S )NrF   )Újoinr`   Ústrip)ra   r~   r4   r3   s       r1   Ú
_join_docszTextSplitter._join_docs¢   s>   € Ø�~Š~˜dÑ#Ô#ˆØÔ!ð 	 Ø—:’:‘<”<ˆDØ�2Š:ˆ:Ø�4àˆKrB   rN   úIterable[str]c                ó¼  — |                       |¦  «        }g }g }d}|D �]�}|                       |¦  «        }||z   t          |¦  «        dk    r|ndz   | j        k    �r|| j        k    r%t                               d|› d| j        › �¦  «         t          |¦  «        dk    rÛ|                      ||¦  «        }	|	�|                     |	¦  «         || j        k    s,||z   t          |¦  «        dk    r|ndz   | j        k    r}|dk    rw||                       |d         ¦  «        t          |¦  «        dk    r|ndz   z  }|dd …         }|| j        k    °K||z   t          |¦  «        dk    r|ndz   | j        k    r|dk    °w|                     |¦  «         ||t          |¦  «        dk    r|ndz   z  }�Œ‘|                      ||¦  «        }	|	�|                     |	¦  «         |S )Nr   zCreated a chunk of size z%, which is longer than the specified r<   )r]   rL   r[   ÚloggerÚwarningrƒ   rs   r\   )
ra   rN   r4   Úseparator_lenr~   Úcurrent_docÚtotalÚdÚ_lenr|   s
             r1   Ú_merge_splitszTextSplitter._merge_splits«   sV  € ð ×-Ò-¨iÑ8Ô8ˆàˆØ!#ˆØˆØð 	Kñ 	KˆAØ×(Ò(¨Ñ+Ô+ˆDà˜‘µ°[Ñ1AÔ1AÀAÒ1EÐ1E  È1ÑMØÔ"ò#ñ #ð ˜4Ô+Ò+Ð+Ý—N’NðQ°5ð Qð QØ>BÔ>NðQð Qñô ð õ �{Ñ#Ô# aÒ'Ð'ØŸ/š/¨+°yÑAÔA�CØ�ØŸš CÑ(Ô(Ð(ð   $Ô"5Ò5Ð5Ø ™½¸[Ñ9IÔ9IÈAÒ9MÐ9M¨¨ÐSTÑUØÔ*ò+ð +à! AšI˜Ià ×!6Ò!6°{À1´~Ñ!FÔ!FÝ-0°Ñ-=Ô-=ÀÒ-AÐ-A˜M˜MÀqñ"ñ ˜ð '2°!°"°"¤o˜ð   $Ô"5Ò5Ð5Ø ™½¸[Ñ9IÔ9IÈAÒ9MÐ9M¨¨ÐSTÑUØÔ*ò+ð +à! AšI˜Ið ×Ò˜qÑ!Ô!Ð!Ø�T­c°+Ñ.>Ô.>ÀÒ.BÐ.B˜]˜]ÈÑJÑJˆE‰EØ�oŠo˜k¨9Ñ5Ô5ˆØˆ?Ø�KŠK˜ÑÔÐØˆrB   Ú	tokenizerr   Úkwargsc                ó¬   ‡— 	 ddl m} t          ‰|¦  «        st          d¦  «        ‚dˆfd„}n# t          $ r t          d	¦  «        ‚w xY w | dd
|i|¤ŽS )z>Text splitter that uses HuggingFace tokenizer to count length.r   )ÚPreTrainedTokenizerBasezATokenizer received was not an instance of PreTrainedTokenizerBaser3   r#   r%   r$   c                óH   •— t          ‰                     | ¦  «        ¦  «        S ©N©rL   Úencode©r3   rŽ   s    €r1   Ú_huggingface_tokenizer_lengthzNTextSplitter.from_huggingface_tokenizer.<locals>._huggingface_tokenizer_lengthà   s   ø€ Ý˜9×+Ò+¨DÑ1Ô1Ñ2Ô2Ð2rB   z`Could not import transformers python package. Please install it with `pip install transformers`.rT   ©r3   r#   r%   r$   r=   )Útransformersr‘   Ú
isinstancerZ   r-   )ÚclsrŽ   r�   r‘   r—   s    `   r1   Úfrom_huggingface_tokenizerz'TextSplitter.from_huggingface_tokenizerÕ   s¸   ø€ ð	Ø<Ð<Ð<Ð<Ð<Ð<å˜iÐ)@ÑAÔAð Ý ØWñô ð ð3ð 3ð 3ð 3ð 3ð 3ð 3øõ ð 	ð 	ð 	ÝðEñô ð ð	øøøð
 ˆsÐKÐKÐ#@ÐKÀFÐKÐKÐKs	   ƒ+/ ¯A	Úgpt2Úallr›   úType[TS]Úencoding_nameÚ
model_nameÚallowed_specialú'Union[Literal['all'], AbstractSet[str]]Údisallowed_specialú&Union[Literal['all'], Collection[str]]r   c                ó  ‡‡‡	— 	 ddl }n# t          $ r t          d¦  «        ‚w xY w|�|                     |¦  «        Š	n|                     |¦  «        Š	dˆˆˆ	fd„}t	          | t
          ¦  «        r||‰‰d	œ}i |¥|¥} | dd
|i|¤ŽS )z9Text splitter that uses tiktoken encoder to count length.r   Nz”Could not import tiktoken python package. This is needed in order to calculate max_tokens_for_prompt. Please install it with `pip install tiktoken`.r3   r#   r%   r$   c                óN   •— t          ‰                     | ‰‰¬¦  «        ¦  «        S ©N)r¢   r¤   r”   )r3   r¢   r¤   Úencs    €€€r1   Ú_tiktoken_encoderz=TextSplitter.from_tiktoken_encoder.<locals>._tiktoken_encoder  s4   ø€ ÝØ—
’
ØØ$3Ø'9ð ñ ô ñô ð rB   )r    r¡   r¢   r¤   rT   r˜   r=   )Útiktokenr-   Úencoding_for_modelÚget_encodingÚ
issubclassÚTokenTextSplitter)
r›   r    r¡   r¢   r¤   r�   r«   rª   Úextra_kwargsr©   s
      ``    @r1   Úfrom_tiktoken_encoderz"TextSplitter.from_tiktoken_encoderê   sú   øøø€ ð	ØˆOˆOˆOˆOøÝð 	ð 	ð 	ÝðAñô ð ð	øøøð Ð!Ø×-Ò-¨jÑ9Ô9ˆCˆCà×'Ò'¨Ñ6Ô6ˆCð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	õ �cÕ,Ñ-Ô-ð 	0à!.Ø(Ø#2Ø&8ð	ð ˆLð 0˜Ð/ ,Ð/ˆFàˆsÐ?Ð?Ð#4Ð?¸Ð?Ð?Ð?s   …
 Š$úSequence[Document]c                óF   — |                       t          |¦  «        ¦  «        S )z2Transform sequence of documents by splitting them.)r}   rM   )ra   ru   r�   s      r1   Útransform_documentsz TextSplitter.transform_documents  s   € ð ×#Ò#¥D¨¡O¤OÑ4Ô4Ð4rB   )rR   r$   rS   r$   rT   rU   r5   r6   rV   r6   rW   r6   r%   rX   ©r3   r#   r%   r7   r“   )rf   r7   rg   rh   r%   ri   )ru   rz   r%   ri   )r~   r7   r4   r#   r%   r   )rN   r„   r4   r#   r%   r7   )rŽ   r   r�   r   r%   r   )r›   rŸ   r    r#   r¡   r   r¢   r£   r¤   r¥   r�   r   r%   r   )ru   r²   r�   r   r%   r²   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__rL   rb   r   re   ry   r}   rƒ   r�   Úclassmethodrœ   Úsetr±   r´   r=   rB   r1   r   r   b   s@  € € € € € Ø3Ð3ð Ø Ø03Ø$Ø %Ø!%ð2ð 2ð 2ð 2ð 2ð@ ð3ð 3ð 3ñ „^ð3ð CGðð ð ð ð ð"Að Að Að Aðð ð ð ð(ð (ð (ð (ðT ðLð Lð Lñ „[ðLð( ð $Ø$(ØCFÀ3Á5Ä5ØEJð)@ð )@ð )@ð )@ñ „[ð)@ðV5ð 5ð 5ð 5ð 5ð 5rB   c                  ó0   ‡ — e Zd ZdZ	 ddˆ fd„Zdd„Zˆ xZS )ÚCharacterTextSplitterz(Splitting text that looks at characters.ú

Fr4   r#   Úis_separator_regexr6   r�   r   r%   rX   c                óV   •—  t          ¦   «         j        di |¤Ž || _        || _        dS )úCreate a new TextSplitter.Nr=   )Úsuperrb   Ú
_separatorÚ_is_separator_regex)ra   r4   r¿   r�   Ú	__class__s       €r1   rb   zCharacterTextSplitter.__init__   s7   ø€ ð 	�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"Ø#ˆŒØ#5ˆÔ Ð Ð rB   r3   r7   c                óÈ   — | j         r| j        nt          j        | j        ¦  «        }t	          ||| j        ¦  «        }| j        rdn| j        }|                      ||¦  «        S )ú&Split incoming text and return chunks.rF   )rÄ   rÃ   rI   ÚescaperO   r^   r�   )ra   r3   r4   rN   rÃ   s        r1   re   z CharacterTextSplitter.split_text(  se   € ð  $Ô7ÐWˆDŒOˆO½R¼YÀtÄÑ=WÔ=Wð 	õ (¨¨i¸Ô9MÑNÔNˆØÔ/ÐD�R�R°T´_ˆ
Ø×!Ò! &¨*Ñ5Ô5Ð5rB   )r¾   F)r4   r#   r¿   r6   r�   r   r%   rX   rµ   ©r¶   r·   r¸   r¹   rb   re   Ú__classcell__©rÅ   s   @r1   r½   r½     sc   ø€ € € € € Ø2Ð2ð CHð6ð 6ð 6ð 6ð 6ð 6ð 6ð6ð 6ð 6ð 6ð 6ð 6ð 6ð 6rB   r½   c                  ó(   — e Zd ZU dZded<   ded<   dS )ÚLineTypezLine type as typed dict.úDict[str, str]rn   r#   ÚcontentN©r¶   r·   r¸   r¹   Ú__annotations__r=   rB   r1   rÍ   rÍ   3  s+   € € € € € € Ø"Ð"àÐÐÑØ€L€L�L€L€LrB   rÍ   c                  ó2   — e Zd ZU dZded<   ded<   ded<   dS )Ú
HeaderTypezHeader type as typed dict.r$   Úlevelr#   ÚnameÚdataNrÐ   r=   rB   r1   rÓ   rÓ   :  s1   € € € € € € Ø$Ð$à€J€J�JØ€I€I�IØ€I€I�I€I€IrB   rÓ   c                  ó0   — e Zd ZdZ	 	 ddd	„Zdd„Zdd„ZdS )ÚMarkdownHeaderTextSplitterz4Splitting markdown files based on specified headers.FTÚheaders_to_split_onúList[Tuple[str, str]]Úreturn_each_liner6   Ústrip_headersc                óR   — || _         t          |d„ d¬¦  «        | _        || _        dS )a  Create a new MarkdownHeaderTextSplitter.

        Args:
            headers_to_split_on: Headers we want to track
            return_each_line: Return each line w/ associated headers
            strip_headers: Strip split headers from the content of the chunk
        c                ó,   — t          | d         ¦  «        S )Nr   )rL   )rJ   s    r1   ú<lambda>z5MarkdownHeaderTextSplitter.__init__.<locals>.<lambda>W  s   € µ3°u¸Q´x±=´=€ rB   T)ÚkeyÚreverseN)rÛ   ÚsortedrÙ   rÜ   )ra   rÙ   rÛ   rÜ   s       r1   rb   z#MarkdownHeaderTextSplitter.__init__E  s@   € ð !1ˆÔõ $*ØÐ%@Ð%@È$ð$
ñ $
ô $
ˆÔ ð +ˆÔÐÐrB   ÚlinesúList[LineType]r%   ri   c                ó4  — g }|D �]}|r8|d         d         |d         k    r |d         dxx         d|d         z   z  cc<   Œ=|r³|d         d         |d         k    r›t          |d         d         ¦  «        t          |d         ¦  «        k     ri|d         d                              d¦  «        d         d         dk    r8| j        s1|d         dxx         d|d         z   z  cc<   |d         |d         d<   Œò|                     |¦  «         �Œ	d„ |D ¦   «         S )	z‚Combine lines with common metadata into chunks
        Args:
            lines: Line of text / associated header metadata
        rD   rn   rÏ   ú  
ú
r   ú#c                óH   — g | ]}t          |d          |d         ¬¦  «        ‘Œ S ©rÏ   rn   rl   ©r   ©r>   rw   s     r1   rA   zHMarkdownHeaderTextSplitter.aggregate_lines_to_chunks.<locals>.<listcomp>�  ó?   € ð 
ð 
ð 
àõ  %¨	Ô"2¸UÀ:Ô=NÐOÑOÔOð
ð 
ð 
rB   )rL   rJ   rÜ   rs   )ra   rã   Úaggregated_chunksÚlines       r1   Úaggregate_lines_to_chunksz4MarkdownHeaderTextSplitter.aggregate_lines_to_chunks\  sn  € ð
 -/Ðàð 	/ñ 	/ˆDà!ð/à% bÔ)¨*Ô5¸¸jÔ9IÒIÐIð
 " "Ô% iÐ0Ð0Ô0°F¸TÀ)¼_Ñ4LÑLÐ0Ð0Ñ0Ð0à!ð/à% bÔ)¨*Ô5¸¸jÔ9IÒIÐIåÐ)¨"Ô-¨jÔ9Ñ:Ô:½SÀÀjÔAQÑ=RÔ=RÒRÐRØ% bÔ)¨)Ô4×:Ò:¸4Ñ@Ô@ÀÔDÀQÔGÈ3ÒNÐNØÔ*ð Oð " "Ô% iÐ0Ð0Ô0°F¸TÀ)¼_Ñ4LÑLÐ0Ð0Ñ0à48¸Ô4DÐ! "Ô% jÑ1Ð1ð "×(Ò(¨Ñ.Ô.Ð.Ñ.ð
ð 
à*ð
ñ 
ô 
ð 	
rB   r3   r#   c                óP  — |                      d¦  «        }g }g }i }g }i }d}d}	|D �]«}
|
                     ¦   «         }|sM|                     d¦  «        r|                     d¦  «        dk    rd}d}	n3|                     d¦  «        rd}d}	n|                     |	¦  «        rd}d}	|r|                     |¦  «         Œ—| j        D �]Œ\  }}|                     |¦  «        �rpt          |¦  «        t          |¦  «        k    s|t          |¦  «                 dk    �r6|�Ã|                     d
¦  «        }|r_|d         d         |k    rM|                     ¦   «         }|d         |v r|                     |d         ¦  «         |r|d         d         |k    °M|||t          |¦  «        d	…                              ¦   «         dœ}|                     |¦  «         |d         ||<   |rQ|                     d                     |¦  «        | 	                    ¦   «         dœ¦  «         | 
                    ¦   «          | j        s|                     |¦  «          nm�ŒŽ|r|                     |¦  «         nS|rQ|                     d                     |¦  «        | 	                    ¦   «         dœ¦  «         | 
                    ¦   «          | 	                    ¦   «         }�Œ­|r+|                     d                     |¦  «        |dœ¦  «         | j        s|                      |¦  «        S d„ |D ¦   «         S )zASplit markdown file
        Args:
            text: Markdown filerç   FrF   z```r<   Tz~~~ú Nrè   rD   rÔ   rÕ   )rÔ   rÕ   rÖ   rÖ   )rÏ   rn   c                óH   — g | ]}t          |d          |d         ¬¦  «        ‘Œ S rê   rë   rì   s     r1   rA   z9MarkdownHeaderTextSplitter.split_text.<locals>.<listcomp>ù  ó?   € ð ð ð àõ  e¨IÔ&6ÀÀzÔARÐSÑSÔSðð ð rB   )rJ   r‚   Ú
startswithÚcountrs   rÙ   rL   Úpopr�   rp   ÚclearrÜ   rÛ   rð   )ra   r3   rã   Úlines_with_metadataÚcurrent_contentÚcurrent_metadataÚheader_stackÚinitial_metadataÚin_code_blockÚopening_fencerï   Ústripped_lineÚseprÕ   Úcurrent_header_levelÚpopped_headerÚheaders                    r1   re   z%MarkdownHeaderTextSplitter.split_text†  sÛ  € ð —
’
˜4Ñ Ô ˆà.0Ðà%'ˆØ+-Ðð *,ˆØ+-ÐàˆØˆàð S	7ñ S	7ˆDØ ŸJšJ™LœLˆMà ð 'à ×+Ò+¨EÑ2Ô2ð *°}×7JÒ7JÈ5Ñ7QÔ7QÐUVÒ7VÐ7VØ$(�MØ$)�M�MØ"×-Ò-¨eÑ4Ô4ð *Ø$(�MØ$)�Møà ×+Ò+¨MÑ:Ô:ð 'Ø$)�MØ$&�Màð Ø×&Ò& }Ñ5Ô5Ð5Øð "Ô5ð <,ñ <,‘	��Tà ×+Ò+¨CÑ0Ô0ñ /õ ˜Ñ&Ô&­#¨c©(¬(Ò2Ð2°mÅCÈÁHÄHÔ6MÐQTÒ6TÑ6Tð Ð'à/2¯yªy¸©~¬~Ð,ð )ð
Là ,¨RÔ 0°Ô 9Ð=QÒ QÐ Qð -9×,<Ò,<Ñ,>Ô,>˜Mð  -¨VÔ4Ð8HÐHÐHØ 0× 4Ò 4°]À6Ô5JÑ KÔ KÐ Kð )ð
Là ,¨RÔ 0°Ô 9Ð=QÒ QÐ Qð &:Ø$(Ø$1µ#°c±(´(°*°*Ô$=×$CÒ$CÑ$EÔ$Eð.ð .˜ð
 %×+Ò+¨FÑ3Ô3Ð3à17¸´Ð(¨Ñ.ð 'ð 0Ø+×2Ò2à+/¯9ª9°_Ñ+EÔ+EØ,<×,AÒ,AÑ,CÔ,Cðð ñô ð ð (×-Ò-Ñ/Ô/Ð/àÔ-ð >Ø'×.Ò.¨}Ñ=Ô=Ð=à�Eùà ð 	,Ø#×*Ò*¨=Ñ9Ô9Ð9Ð9Ø$ð ,Ø'×.Ò.à'+§y¢y°Ñ'AÔ'AØ(8×(=Ò(=Ñ(?Ô(?ðð ñô ð ð $×)Ò)Ñ+Ô+Ð+à/×4Ò4Ñ6Ô6ÐÑàð 	Ø×&Ò&Ø ŸIšI oÑ6Ô6ÐDTÐUÐUñô ð ð Ô$ð 	Ø×1Ò1Ð2EÑFÔFÐFðð à0ðñ ô ð rB   N)FT)rÙ   rÚ   rÛ   r6   rÜ   r6   )rã   rä   r%   ri   ©r3   r#   r%   ri   )r¶   r·   r¸   r¹   rb   rð   re   r=   rB   r1   rØ   rØ   B  sj   € € € € € Ø>Ð>ð
 "'Ø"ð	+ð +ð +ð +ð +ð.(
ð (
ð (
ð (
ðTvð vð vð vð vð vrB   rØ   c                  ó<   — e Zd ZU dZded<   ded<   ded<   ded<   dS )	ÚElementTypezElement type as typed dict.r#   ÚurlÚxpathrÏ   rÎ   rn   NrÐ   r=   rB   r1   r  r  ÿ  s?   € € € € € € Ø%Ð%à€H€H�HØ€J€J�JØ€L€L�LØÐÐÑÐÐrB   r  c                  ó>   — e Zd ZdZ	 ddd„Zdd„Zdd„Zdd„Zdd„ZdS )ÚHTMLHeaderTextSplitterzU
    Splitting HTML files based on specified headers.
    Requires lxml package.
    FrÙ   rÚ   Úreturn_each_elementr6   c                ó<   — || _         t          |¦  «        | _        dS )ay  Create a new HTMLHeaderTextSplitter.

        Args:
            headers_to_split_on: list of tuples of headers we want to track mapped to
                (arbitrary) keys for metadata. Allowed header values: h1, h2, h3, h4,
                h5, h6 e.g. [("h1", "Header 1"), ("h2", "Header 2)].
            return_each_element: Return each element w/ associated headers.
        N)r  râ   rÙ   )ra   rÙ   r  s      r1   rb   zHTMLHeaderTextSplitter.__init__  s#   € ð $7ˆÔ Ý#)Ð*=Ñ#>Ô#>ˆÔ Ð Ð rB   ÚelementsúList[ElementType]r%   ri   c                óÆ   — g }|D ]Q}|r8|d         d         |d         k    r |d         dxx         d|d         z   z  cc<   Œ<|                      |¦  «         ŒRd„ |D ¦   «         S )z¢Combine elements with common metadata into chunks

        Args:
            elements: HTML element content with associated identifying info and metadata
        rD   rn   rÏ   ræ   c                óH   — g | ]}t          |d          |d         ¬¦  «        ‘Œ S rê   rë   rì   s     r1   rA   zGHTMLHeaderTextSplitter.aggregate_elements_to_chunks.<locals>.<listcomp>6  rí   rB   )rs   )ra   r  rî   Úelements       r1   Úaggregate_elements_to_chunksz3HTMLHeaderTextSplitter.aggregate_elements_to_chunks  s¥   € ð 02Ðàð 	2ð 	2ˆGà!ð
2à% bÔ)¨*Ô5¸ÀÔ9LÒLÐLð
 " "Ô% iÐ0Ð0Ô0°F¸WÀYÔ=OÑ4OÑOÐ0Ð0Ñ0Ð0ð "×(Ò(¨Ñ1Ô1Ð1Ð1ð
ð 
à*ð
ñ 
ô 
ð 	
rB   r  r#   c                óx   — t          j        |¦  «        }|                      t          |j        ¦  «        ¦  «        S )zHSplit HTML from web URL

        Args:
            url: web URL
        )ÚrequestsÚgetÚsplit_text_from_filer   rÏ   )ra   r  Úrs      r1   Úsplit_text_from_urlz*HTMLHeaderTextSplitter.split_text_from_url;  s1   € õ ŒL˜ÑÔˆØ×(Ò(­°´Ñ);Ô);Ñ<Ô<Ð<rB   r3   c                óF   — |                       t          |¦  «        ¦  «        S )zJSplit HTML text string

        Args:
            text: HTML text
        )r  r	   rd   s     r1   re   z!HTMLHeaderTextSplitter.split_textD  s   € ð ×(Ò(­°$©¬Ñ8Ô8Ð8rB   Úfiler   c                ó<  ‡‡— 	 ddl m} n"# t          $ r}t          d¦  «        |‚d}~ww xY w|                     ¦   «         }|                     ||¦  «        }t          j        t          ¦  «        j        dz  }|                     |¦  «        }| 	                    |¦  «        } ||¦  «        }	| 
                    t          |	¦  «        ¦  «        }
d„ | j        D ¦   «         Št          | j        ¦  «        Šddi}g }|
                     d	|¦  «        D ]å}|                     d
¦  «        s|                     d¦  «        r¹|                     t!          |d                     d„ |                     d|¦  «        D ¦   «         ¦  «        d                     d„ |                     d|¦  «        D ¦   «         ¦  «        ˆfd„t%          ˆfd„|                     d|¦  «        ¦  «        D ¦   «         ¬¦  «        ¦  «         Œæ| j        s|                      |¦  «        S d„ |D ¦   «         S )zCSplit HTML file

        Args:
            file: HTML file
        r   )Úetreez>Unable to import lxml, please install with `pip install lxml`.Nz7document_transformers/xsl/html_chunks_with_headers.xsltc                ó   — g | ]
}|d          ‘ŒS )r   r=   )r>   r  s     r1   rA   z?HTMLHeaderTextSplitter.split_text_from_file.<locals>.<listcomp>h  s   € ÐJÐJÐJ v˜ œÐJÐJÐJrB   Úhzhttp://www.w3.org/1999/xhtmlz*//*z*[@class='headers']z*[@class='chunk']rF   c                ó   — g | ]	}|j         ‘Œ
S r=   ©r3   ©r>   Únodes     r1   rA   z?HTMLHeaderTextSplitter.split_text_from_file.<locals>.<listcomp>x  ó*   € ð ð ð à$(ð !%¤	ðð ð rB   z*[@class='xpath']c                ó   — g | ]	}|j         ‘Œ
S r=   r!  r"  s     r1   rA   z?HTMLHeaderTextSplitter.split_text_from_file.<locals>.<listcomp>~  r$  rB   c                ó6   •— i | ]}‰|j                  |j        “ŒS r=   )Útagr3   )r>   r#  Úheader_mappings     €r1   ú
<dictcomp>z?HTMLHeaderTextSplitter.split_text_from_file.<locals>.<dictcomp>ƒ  s6   ø€ ð "ð "ð "ð !%ð +¨4¬8Ô4°d´ið"ð "ð "rB   c                ó   •— | j         ‰v S r“   )r'  )ÚxÚheader_filters    €r1   rß   z=HTMLHeaderTextSplitter.split_text_from_file.<locals>.<lambda>ˆ  s   ø€ ¨!¬%°=Ð*@€ rB   z*[@class='headers']/*)r  r	  rÏ   rn   c                óH   — g | ]}t          |d          |d         ¬¦  «        ‘Œ S rê   rë   rì   s     r1   rA   z?HTMLHeaderTextSplitter.split_text_from_file.<locals>.<listcomp>’  rô   rB   )Úlxmlr  r-   Ú
HTMLParserÚparseÚpathlibÚPathÚ__file__ÚparentÚXSLTÚ
fromstringr#   rÙ   ÚdictÚfindallrs   r  r�   Úfilterr  r  )ra   r  r  ÚeÚparserÚtreeÚ	xslt_pathÚ	xslt_treeÚ	transformÚresultÚ
result_domÚns_mapr  r  r,  r(  s                 @@r1   r  z+HTMLHeaderTextSplitter.split_text_from_fileL  sª  øø€ ð	Ø"Ð"Ð"Ð"Ð"Ð"Ð"øÝð 	ð 	ð 	ÝØPñô àðøøøøð	øøøð
 ×!Ò!Ñ#Ô#ˆØ�{Š{˜4 Ñ(Ô(ˆõ
 ŒL�Ñ"Ô"Ô)ØGñHð 	ð —K’K 	Ñ*Ô*ˆ	Ø—J’J˜yÑ)Ô)ˆ	Ø�˜4‘”ˆØ×%Ò%¥c¨&¡k¤kÑ2Ô2ˆ
ð KÐJ°Ô1IÐJÑJÔJˆÝ˜dÔ6Ñ7Ô7ˆð Ð5Ð6ˆð ˆØ!×)Ò)¨&°&Ñ9Ô9ð 	ð 	ˆGØ�ŠÐ4Ñ5Ô5ð ¸¿ºØ#ñ:ô :ð ð —’ÝØ Ø Ÿgšgðð à,3¯OªOÐ<OÐQWÑ,XÔ,Xðñ ô ñô ð !#§¢ðð à,3¯OªOÐ<OÐQWÑ,XÔ,Xðñ ô ñ!ô !ð"ð "ð "ð "õ )/Ø @Ð @Ð @Ð @Ø '§¢Ð0GÈÑ PÔ Pñ)ô )ð	"ñ "ô "ðñ ô ñô ð øð6 Ô'ð 	Ø×4Ò4°XÑ>Ô>Ð>ðð à%ðñ ô ð s   „ ‹
*•%¥*N)F)rÙ   rÚ   r  r6   )r  r  r%   ri   )r  r#   r%   ri   r  )r  r   r%   ri   )	r¶   r·   r¸   r¹   rb   r  r  re   r  r=   rB   r1   r  r    s’   € € € € € ðð ð %*ð?ð ?ð ?ð ?ð ?ð"
ð 
ð 
ð 
ð8=ð =ð =ð =ð9ð 9ð 9ð 9ðIð Ið Ið Ið Ið IrB   r  T)Úfrozenc                  óB   — e Zd ZU dZded<   	 ded<   	 ded<   	 ded<   d	S )
Ú	TokenizerzTokenizer data class.r$   rS   Útokens_per_chunkzCallable[[List[int]], str]ÚdecodezCallable[[str], List[int]]r•   NrÐ   r=   rB   r1   rE  rE  š  sQ   € € € € € € àÐàÐÐÑØ*ØÐÐÑØ,Ø&Ð&Ð&Ñ&Ø=Ø&Ð&Ð&Ñ&Ø=Ð=rB   rE  rŽ   c                óÜ  — g }|                      | ¦  «        }d}t          ||j        z   t          |¦  «        ¦  «        }|||…         }|t          |¦  «        k     r�|                     |                     |¦  «        ¦  «         |t          |¦  «        k    rnT||j        |j        z
  z  }t          ||j        z   t          |¦  «        ¦  «        }|||…         }|t          |¦  «        k     °�|S )z6Split incoming text and return chunks using tokenizer.r   )r•   ÚminrF  rL   rs   rG  rS   )r3   rŽ   rN   Ú	input_idsÚ	start_idxÚcur_idxÚ	chunk_idss          r1   Úsplit_text_on_tokensrN  ¨  së   € à€FØ× Ò  Ñ&Ô&€IØ€IÝ�)˜iÔ8Ñ8½#¸i¹.¼.ÑIÔI€GØ˜) GÐ+Ô,€IØ
•c˜)‘n”nÒ
$Ð
$Ø�Š�i×&Ò& yÑ1Ô1Ñ2Ô2Ð2Ø•c˜)‘n”nÒ$Ð$ØØ�YÔ/°)Ô2IÑIÑIˆ	Ý�i )Ô"<Ñ<½cÀ)¹n¼nÑMÔMˆØ˜i¨Ð/Ô0ˆ	ð •c˜)‘n”nÒ
$Ð
$ð €MrB   c                  óF   ‡ — e Zd ZdZdd e¦   «         dfdˆ fd„Zdd„Zˆ xZS )r¯   z/Splitting text to tokens using model tokenizer.r�   Nrž   r    r#   r¡   r   r¢   r£   r¤   r¥   r�   r   r%   rX   c                ó  •—  t          ¦   «         j        di |¤Ž 	 ddl}n# t          $ r t          d¦  «        ‚w xY w|�|                     |¦  «        }n|                     |¦  «        }|| _        || _        || _        dS )rÁ   r   NzŠCould not import tiktoken python package. This is needed in order to for TokenTextSplitter. Please install it with `pip install tiktoken`.r=   )	rÂ   rb   r«   r-   r¬   r­   Ú
_tokenizerÚ_allowed_specialÚ_disallowed_special)	ra   r    r¡   r¢   r¤   r�   r«   r©   rÅ   s	           €r1   rb   zTokenTextSplitter.__init__¼  s²   ø€ ð 	�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"ð	ØˆOˆOˆOˆOøÝð 	ð 	ð 	ÝðAñô ð ð	øøøð Ð!Ø×-Ò-¨jÑ9Ô9ˆCˆCà×'Ò'¨Ñ6Ô6ˆCØˆŒØ /ˆÔØ#5ˆÔ Ð Ð s   œ! ¡;r3   r7   c                ó€   ‡ — dˆ fd„}t          ‰ j        ‰ j        ‰ j        j        |¬¦  «        }t          ||¬¦  «        S )	NÚ_textr#   r%   ú	List[int]c                óR   •— ‰j                              | ‰j        ‰j        ¬¦  «        S r¨   )rQ  r•   rR  rS  )rU  ra   s    €r1   Ú_encodez-TokenTextSplitter.split_text.<locals>._encodeØ  s1   ø€ Ø”?×)Ò)ØØ $Ô 5Ø#'Ô#;ð *ñ ô ð rB   ©rS   rF  rG  r•   r–   )rU  r#   r%   rV  )rE  r\   r[   rQ  rG  rN  )ra   r3   rX  rŽ   s   `   r1   re   zTokenTextSplitter.split_text×  sc   ø€ ð	ð 	ð 	ð 	ð 	ð 	õ ØÔ-Ø!Ô-Ø”?Ô)Øð	
ñ 
ô 
ˆ	õ $¨¸ÐCÑCÔCÐCrB   )r    r#   r¡   r   r¢   r£   r¤   r¥   r�   r   r%   rX   rµ   )r¶   r·   r¸   r¹   r»   rb   re   rÊ   rË   s   @r1   r¯   r¯   ¹  sx   ø€ € € € € Ø9Ð9ð $Ø$(ØCFÀ3Á5Ä5ØEJð6ð 6ð 6ð 6ð 6ð 6ð 6ð6Dð Dð Dð Dð Dð Dð Dð DrB   r¯   c                  ó\   ‡ — e Zd ZU dZ	 	 	 ddˆ fd„Zdd„Zdd„Zdd„ZdZde	d<   dd„Z
ˆ xZS )Ú%SentenceTransformersTokenTextSplitterz8Splitting text to tokens using sentence model tokenizer.é2   ú'sentence-transformers/all-mpnet-base-v2NrS   r$   r¡   r#   rF  úOptional[int]r�   r   r%   rX   c                ó  •—  t          ¦   «         j        di |¤d|i¤Ž 	 ddlm} n# t          $ r t	          d¦  «        ‚w xY w|| _         || j        ¦  «        | _        | j        j        | _        |                      |¬¦  «         dS )rÁ   rS   r   )ÚSentenceTransformerz·Could not import sentence_transformer python package. This is needed in order to for SentenceTransformersTokenTextSplitter. Please install it with `pip install sentence-transformers`.)rF  Nr=   )	rÂ   rb   Úsentence_transformersr`  r-   r¡   Ú_modelrŽ   Ú_initialize_chunk_configuration)ra   rS   r¡   rF  r�   r`  rÅ   s         €r1   rb   z.SentenceTransformersTokenTextSplitter.__init__ì  sÁ   ø€ ð 	�‰ŒÔÐ?Ð?˜6Ð?Ð?°Ð?Ð?Ð?Ð?ð	ØAÐAÐAÐAÐAÐAÐAøÝð 	ð 	ð 	ÝðNñô ð ð	øøøð %ˆŒØ)Ð)¨$¬/Ñ:Ô:ˆŒØœÔ.ˆŒØ×,Ò,Ð>NÐ,ÑOÔOÐOÐOÐOs	    ' §Ac          	     óê   — t          t          | j        j        ¦  «        | _        |€| j        | _        n|| _        | j        | j        k    r(t          d| j        › d| j        › d| j        › d�¦  «        ‚d S )NzThe token limit of the models 'z' is: z. Argument tokens_per_chunk=z > maximum token limit.)r   r$   rb  Úmax_seq_lengthÚmaximum_tokens_per_chunkrF  rZ   r¡   )ra   rF  s     r1   rc  zESentenceTransformersTokenTextSplitter._initialize_chunk_configuration  s›   € õ )-­S°$´+Ô2LÑ(MÔ(MˆÔ%àÐ#Ø$(Ô$AˆDÔ!Ð!à$4ˆDÔ!àÔ  4Ô#@Ò@Ð@Ýð+°$´/ð +ð +ØÔ5ð+ð +à.2Ô.Cð+ð +ð +ñô ð ð AÐ@rB   r3   r7   c                ó€   ‡ — dˆ fd„}t          ‰ j        ‰ j        ‰ j        j        |¬¦  «        }t          ||¬¦  «        S )	Nr3   r#   r%   rV  c                ó>   •— ‰                      | ¦  «        dd…         S )Nr<   rD   )rX  )r3   ra   s    €r1   Ú%encode_strip_start_and_stop_token_idsz_SentenceTransformersTokenTextSplitter.split_text.<locals>.encode_strip_start_and_stop_token_ids  s   ø€ Ø—<’< Ñ%Ô% a¨ dÔ+Ð+rB   rY  r–   ©r3   r#   r%   rV  )rE  r\   rF  rŽ   rG  rN  )ra   r3   ri  rŽ   s   `   r1   re   z0SentenceTransformersTokenTextSplitter.split_text  sc   ø€ ð	,ð 	,ð 	,ð 	,ð 	,ð 	,õ ØÔ-Ø!Ô2Ø”>Ô(Ø8ð	
ñ 
ô 
ˆ	õ $¨¸ÐCÑCÔCÐCrB   c               óF   — t          |                      |¦  «        ¦  «        S r“   )rL   rX  rd   s     r1   Úcount_tokensz2SentenceTransformersTokenTextSplitter.count_tokens#  s   € Ý�4—<’< Ñ%Ô%Ñ&Ô&Ð&rB   l        Ú _max_length_equal_32_bit_integerrV  c                óJ   — | j                              || j        d¬¦  «        }|S )NÚdo_not_truncate)r!   Ú
truncation)rŽ   r•   rm  )ra   r3   Ú&token_ids_with_start_and_end_token_idss      r1   rX  z-SentenceTransformersTokenTextSplitter._encode(  s4   € Ø15´×1FÒ1FØØÔ<Ø(ð 2Gñ 2
ô 2
Ð.ð
 6Ð5rB   )r\  r]  N)
rS   r$   r¡   r#   rF  r^  r�   r   r%   rX   )rF  r^  r%   rX   rµ   r˜   rj  )r¶   r·   r¸   r¹   rb   rc  re   rl  rm  rÑ   rX  rÊ   rË   s   @r1   r[  r[  é  sÄ   ø€ € € € € € ØBÐBð  ØCØ*.ð	Pð Pð Pð Pð Pð Pð Pð0ð ð ð ð$Dð Dð Dð Dð'ð 'ð 'ð 'ð -2Ð$Ð1Ð1Ð1Ñ1ð6ð 6ð 6ð 6ð 6ð 6ð 6ð 6rB   r[  c                  ób   — e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdS )ÚLanguagez"Enum of the programming languages.ÚcppÚgoÚjavaÚkotlinÚjsÚtsÚphpÚprotoÚpythonÚrstÚrubyÚrustÚscalaÚswiftÚmarkdownÚlatexÚhtmlÚsolÚcsharpÚcobolN)r¶   r·   r¸   r¹   ÚCPPÚGOÚJAVAÚKOTLINÚJSr   ÚPHPÚPROTOÚPYTHONÚRSTÚRUBYÚRUSTÚSCALAÚSWIFTÚMARKDOWNÚLATEXÚHTMLÚSOLÚCSHARPÚCOBOLr=   rB   r1   rs  rs  1  sx   € € € € € Ø,Ð,à
€CØ	€BØ€DØ€FØ	€BØ	€BØ
€CØ€EØ€FØ
€CØ€DØ€DØ€EØ€EØ€HØ€EØ€DØ
€CØ€FØ€E€E€ErB   rs  c                  ól   ‡ — e Zd ZdZ	 	 	 ddˆ fd„Zdd„Zdd„Zedd„¦   «         Ze	dd„¦   «         Z
ˆ xZS )ÚRecursiveCharacterTextSplitterzŠSplitting text by recursively look at characters.

    Recursively tries to split by different characters to find one
    that works.
    NTFÚ
separatorsúOptional[List[str]]r5   r6   r¿   r�   r   r%   rX   c                ób   •—  t          ¦   «         j        dd|i|¤Ž |pg d¢| _        || _        dS )rÁ   r5   )r¾   rç   rò   rF   Nr=   )rÂ   rb   Ú_separatorsrÄ   )ra   r�  r5   r¿   r�   rÅ   s        €r1   rb   z'RecursiveCharacterTextSplitter.__init__Q  sI   ø€ ð 	�‰ŒÔÐAÐA¨ÐA¸&ÐAÐAÐAØ%Ð@Ð)@Ð)@Ð)@ˆÔØ#5ˆÔ Ð Ð rB   r3   r#   r7   c                ó  — g }|d         }g }t          |¦  «        D ]R\  }}| j        r|nt          j        |¦  «        }|dk    r|} n't          j        ||¦  «        r|}||dz   d…         } nŒS| j        r|nt          j        |¦  «        }t          ||| j        ¦  «        }	g }
| j        rdn|}|	D ]¨}|                      |¦  «        | j        k     r|
 	                    |¦  «         Œ6|
r-|  
                    |
|¦  «        }|                     |¦  «         g }
|s| 	                    |¦  «         Œ}|                      ||¦  «        }|                     |¦  «         Œ©|
r+|  
                    |
|¦  «        }|                     |¦  «         |S )rÇ   rD   rF   r<   N)ro   rÄ   rI   rÈ   ÚsearchrO   r^   r]   r[   rs   r�   ÚextendÚ_split_text)ra   r3   r�  Úfinal_chunksr4   Únew_separatorsr?   Ú_srÃ   rN   Ú_good_splitsrH   Úmerged_textÚ
other_infos                 r1   r¤  z*RecursiveCharacterTextSplitter._split_text]  sÔ  € àˆà˜r”Nˆ	ØˆÝ˜zÑ*Ô*ð 	ð 	‰EˆAˆrØ#Ô7ÐJ˜˜½R¼YÀr¹]¼]ˆJØ�RŠxˆxØ�	Ø�ÝŒy˜ TÑ*Ô*ð Ø�	Ø!+¨A°©E¨G¨GÔ!4�Ø�ðð
 #'Ô":ÐT�Y�YÅÄ	È)Ñ@TÔ@Tˆ
Ý'¨¨j¸$Ô:NÑOÔOˆð ˆØÔ/Ð>�R�R°Yˆ
Øð 	4ð 	4ˆAØ×$Ò$ QÑ'Ô'¨$Ô*:Ò:Ð:Ø×#Ò# AÑ&Ô&Ð&Ð&àð &Ø"&×"4Ò"4°\À:Ñ"NÔ"N�KØ ×'Ò'¨Ñ4Ô4Ð4Ø#%�LØ%ð 4Ø ×'Ò'¨Ñ*Ô*Ð*Ð*à!%×!1Ò!1°!°^Ñ!DÔ!D�JØ ×'Ò'¨
Ñ3Ô3Ð3Ð3Øð 	-Ø×,Ò,¨\¸:ÑFÔFˆKØ×Ò Ñ,Ô,Ð,ØÐrB   c                ó8   — |                       || j        ¦  «        S r“   )r¤  r   rd   s     r1   re   z)RecursiveCharacterTextSplitter.split_text…  s   € Ø×Ò  dÔ&6Ñ7Ô7Ð7rB   Úlanguagers  c                óB   — |                       |¦  «        } | d|ddœ|¤ŽS )NT)r�  r¿   r=   )Úget_separators_for_language)r›   r¬  r�   r�  s       r1   Úfrom_languagez,RecursiveCharacterTextSplitter.from_languageˆ  s4   € ð ×4Ò4°XÑ>Ô>ˆ
ØˆsÐL˜j¸TÐLÐLÀVÐLÐLÐLrB   c                óp  — | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j	        k    rg d	¢S | t           j
        k    rg d
¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S | t           j        k    rg d¢S t+          d| › dt-          t           ¦  «        › �¦  «        ‚)N)ú
class z
void z
int z
float z
double ú
if ú
for ú
while ú
switch ú
case r¾   rç   rò   rF   )ú
func ú
var ú
const ú
type r²  r³  rµ  r¶  r¾   rç   rò   rF   )r±  ú
public ú
protected ú	
private ú
static r²  r³  r´  rµ  r¶  r¾   rç   rò   rF   )r±  r»  r¼  r½  z

internal z
companion z
fun ú
val r¸  r²  r³  r´  z
when r¶  ú
else r¾   rç   rò   rF   )ú

function r¹  ú
let r¸  r±  r²  r³  r´  rµ  r¶  ú	
default r¾   rç   rò   rF   )ú
enum ú
interface z
namespace rº  r±  rÁ  r¹  rÂ  r¸  r²  r³  r´  rµ  r¶  rÃ  r¾   rç   rò   rF   )rÁ  r±  r²  ú	
foreach r´  ú
do rµ  r¶  r¾   rç   rò   rF   )
z	
message z	
service rÄ  z
option z
import z
syntax r¾   rç   rò   rF   )r±  ú
def z
	def r¾   rç   rò   rF   )z
=+
z
-+
z
\*+
z

.. *

r¾   rç   rò   rF   )rÈ  r±  r²  z
unless r´  r³  rÇ  z
begin z
rescue r¾   rç   rò   rF   )z
fn r¹  rÂ  r²  r´  r³  z
loop ú
match r¹  r¾   rç   rò   rF   )r±  z
object rÈ  r¿  r¸  r²  r³  r´  rÉ  r¶  r¾   rç   rò   rF   )r·  r±  ú
struct rÄ  r²  r³  r´  rÇ  rµ  r¶  r¾   rç   rò   rF   )	z
#{1,6} z```
z	
\*\*\*+
z
---+
z
___+
r¾   rç   rò   rF   )z
\\chapter{z
\\section{z
\\subsection{z
\\subsubsection{z
\\begin{enumerate}z
\\begin{itemize}z
\\begin{description}z
\\begin{list}z
\\begin{quote}z
\\begin{quotation}z
\\begin{verse}z
\\begin{verbatim}z
\egin{align}z$$ú$rò   rF   )z<bodyz<divz<pz<brz<liz<h1z<h2z<h3z<h4z<h5z<h6z<spanz<tablez<trz<tdz<thz<ulz<olz<headerz<footerz<navz<headz<stylez<scriptz<metaz<titlerF   )rÅ  rÄ  z
implements z

delegate ú
event r±  z

abstract r»  r¼  r½  r¾  z
return r²  z

continue r³  rÆ  r´  rµ  z
break r¶  rÀ  z
try z
throw z	
finally z
catch r¾   rç   rò   rF   )z
pragma z
using z

contract rÅ  z	
library z
constructor rº  rÁ  rÌ  z

modifier z
error rÊ  rÄ  r²  r³  r´  z

do while z

assembly r¾   rç   rò   rF   )z
IDENTIFICATION DIVISION.z
ENVIRONMENT DIVISION.z
DATA DIVISION.z
PROCEDURE DIVISION.z
WORKING-STORAGE SECTION.z
LINKAGE SECTION.z
FILE SECTION.z
INPUT-OUTPUT SECTION.z
OPEN z
CLOSE z
READ z
WRITE z
IF z
ELSE z
MOVE z	
PERFORM z
UNTIL z	
VARYING z
ACCEPT z	
DISPLAY z

STOP RUN.rç   rò   rF   z	Language z& is not supported! Please choose from )rs  rˆ  r‰  rŠ  r‹  rŒ  r   r�  rŽ  r�  r�  r‘  r’  r“  r”  r•  r–  r—  r™  r˜  rš  rZ   rM   ©r¬  s    r1   r®  z:RecursiveCharacterTextSplitter.get_separators_for_language�  s÷  € à•x”|Ò#Ð#ðð ð ð ð( �œÒ$Ð$ðð ð ð ð" �œÒ&Ð&ðð ð ð ð( �œÒ(Ð(ðð ð ð ð2 �œÒ$Ð$ðð ð ð ð( �œÒ$Ð$ðð ð ð ð2 �œÒ%Ð%ðð ð ð ð$ �œÒ'Ð'ðð ð ð ð& �œÒ(Ð(ð
ð 
ð 
ð 
ð �œÒ%Ð%ðð ð ð ð �œÒ&Ð&ðð ð ð ð$ �œÒ&Ð&ðð ð ð ð$ �œÒ'Ð'ðð ð ð ð( �œÒ'Ð'ðð ð ð ð( �Ô*Ò*Ð*ðð ð ð ð& �œÒ'Ð'ðð ð ð ð. �œÒ&Ð&ðð ð ð ð> �œÒ(Ð(ð#ð #ð #ð #ðH �œÒ%Ð%ðð ð ð ð: �œÒ'Ð'ðð ð ð õB ð7˜Hð 7ð 7Ý&*­8¡n¤nð7ð 7ñô ð rB   )NTF)
r�  rž  r5   r6   r¿   r6   r�   r   r%   rX   )r3   r#   r�  r7   r%   r7   rµ   )r¬  rs  r�   r   r%   rœ  )r¬  rs  r%   r7   )r¶   r·   r¸   r¹   rb   r¤  re   rº   r¯  Ústaticmethodr®  rÊ   rË   s   @r1   rœ  rœ  J  sÌ   ø€ € € € € ðð ð +/Ø#Ø#(ð	
6ð 
6ð 
6ð 
6ð 
6ð 
6ð 
6ð&ð &ð &ð &ðP8ð 8ð 8ð 8ð ðMð Mð Mñ „[ðMð ðKð Kð Kñ „\ðKð Kð Kð Kð KrB   rœ  c                  ó0   ‡ — e Zd ZdZ	 ddˆ fd„Zdd„Zˆ xZS )ÚNLTKTextSplitterz"Splitting text using NLTK package.r¾   Úenglishr4   r#   r¬  r�   r   r%   rX   c                ó®   •—  t          ¦   «         j        di |¤Ž 	 ddlm} || _        n# t
          $ r t          d¦  «        ‚w xY w|| _        || _        dS )zInitialize the NLTK splitter.r   )Úsent_tokenizezANLTK is not installed, please install it with `pip install nltk`.Nr=   )rÂ   rb   Únltk.tokenizerÓ  rQ  r-   rÃ   Ú	_language)ra   r4   r¬  r�   rÓ  rÅ   s        €r1   rb   zNLTKTextSplitter.__init__a  s†   ø€ ð 	�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"ð	Ø3Ð3Ð3Ð3Ð3Ð3à+ˆDŒOˆOøÝð 	ð 	ð 	ÝØSñô ð ð	øøøð $ˆŒØ!ˆŒˆˆs	   œ* ªAr3   r7   c                óp   — |                       || j        ¬¦  «        }|                      || j        ¦  «        S )rÇ   rÍ  )rQ  rÕ  r�   rÃ   ©ra   r3   rN   s      r1   re   zNLTKTextSplitter.split_textq  s3   € ð —’ °´�Ñ?Ô?ˆØ×!Ò! &¨$¬/Ñ:Ô:Ð:rB   )r¾   rÑ  )r4   r#   r¬  r#   r�   r   r%   rX   rµ   rÉ   rË   s   @r1   rÐ  rÐ  ^  sb   ø€ € € € € Ø,Ð,ð 8Að"ð "ð "ð "ð "ð "ð "ð ;ð ;ð ;ð ;ð ;ð ;ð ;ð ;rB   rÐ  c                  ó4   ‡ — e Zd ZdZ	 	 	 ddˆ fd„Zdd„Zˆ xZS )ÚSpacyTextSplitteraR  Splitting text using Spacy package.


    Per default, Spacy's `en_core_web_sm` model is used and
    its default max_length is 1000000 (it is the length of maximum character
    this model takes which can be increased for large files). For a faster, but
    potentially less accurate splitting, you can use `pipeline='sentencizer'`.
    r¾   Úen_core_web_smr   r4   r#   r"   r!   r$   r�   r   r%   rX   c                ót   •—  t          ¦   «         j        di |¤Ž t          ||¬¦  «        | _        || _        dS )z#Initialize the spacy text splitter.r    Nr=   )rÂ   rb   r2   rQ  rÃ   )ra   r4   r"   r!   r�   rÅ   s        €r1   rb   zSpacyTextSplitter.__init__‚  sJ   ø€ ð 	�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"Ý<Ø ð
ñ 
ô 
ˆŒð $ˆŒˆˆrB   r3   r7   c                ó€   — d„ |                       |¦  «        j        D ¦   «         }|                      || j        ¦  «        S )rÇ   c              3  ó$   K  — | ]}|j         V — Œd S r“   r!  rG   s     r1   ú	<genexpr>z/SpacyTextSplitter.split_text.<locals>.<genexpr>’  s$   è è € Ð>Ð>˜Q�!”&Ð>Ð>Ð>Ð>Ð>Ð>rB   )rQ  Úsentsr�   rÃ   r×  s      r1   re   zSpacyTextSplitter.split_text�  s<   € à>Ð> $§/¢/°$Ñ"7Ô"7Ô"=Ð>Ñ>Ô>ˆØ×!Ò! &¨$¬/Ñ:Ô:Ð:rB   )r¾   rÚ  r   )
r4   r#   r"   r#   r!   r$   r�   r   r%   rX   rµ   rÉ   rË   s   @r1   rÙ  rÙ  x  sk   ø€ € € € € ðð ð  Ø(Ø#ð	$ð $ð $ð $ð $ð $ð $ð;ð ;ð ;ð ;ð ;ð ;ð ;ð ;rB   rÙ  c                  ó$   ‡ — e Zd ZdZdˆ fd„Zˆ xZS )ÚPythonCodeTextSplitterz/Attempts to split the text along Python syntax.r�   r   r%   rX   c                ó|   •— |                       t          j        ¦  «        } t          ¦   «         j        dd|i|¤Ž dS )z$Initialize a PythonCodeTextSplitter.r�  Nr=   )r®  rs  r�  rÂ   rb   ©ra   r�   r�  rÅ   s      €r1   rb   zPythonCodeTextSplitter.__init__š  s@   ø€ à×5Ò5µh´oÑFÔFˆ
Ø�‰ŒÔÐ9Ð9 JÐ9°&Ð9Ð9Ð9Ð9Ð9rB   ©r�   r   r%   rX   ©r¶   r·   r¸   r¹   rb   rÊ   rË   s   @r1   rá  rá  —  sC   ø€ € € € € Ø9Ð9ð:ð :ð :ð :ð :ð :ð :ð :ð :ð :rB   rá  c                  ó$   ‡ — e Zd ZdZdˆ fd„Zˆ xZS )ÚMarkdownTextSplitterz=Attempts to split the text along Markdown-formatted headings.r�   r   r%   rX   c                ó|   •— |                       t          j        ¦  «        } t          ¦   «         j        dd|i|¤Ž dS )z"Initialize a MarkdownTextSplitter.r�  Nr=   )r®  rs  r•  rÂ   rb   rã  s      €r1   rb   zMarkdownTextSplitter.__init__£  sA   ø€ à×5Ò5µhÔ6GÑHÔHˆ
Ø�‰ŒÔÐ9Ð9 JÐ9°&Ð9Ð9Ð9Ð9Ð9rB   rä  rå  rË   s   @r1   rç  rç     sC   ø€ € € € € ØGÐGð:ð :ð :ð :ð :ð :ð :ð :ð :ð :rB   rç  c                  ó$   ‡ — e Zd ZdZdˆ fd„Zˆ xZS )ÚLatexTextSplitterzAAttempts to split the text along Latex-formatted layout elements.r�   r   r%   rX   c                ó|   •— |                       t          j        ¦  «        } t          ¦   «         j        dd|i|¤Ž dS )zInitialize a LatexTextSplitter.r�  Nr=   )r®  rs  r–  rÂ   rb   rã  s      €r1   rb   zLatexTextSplitter.__init__¬  s@   ø€ à×5Ò5µh´nÑEÔEˆ
Ø�‰ŒÔÐ9Ð9 JÐ9°&Ð9Ð9Ð9Ð9Ð9rB   rä  rå  rË   s   @r1   rê  rê  ©  sC   ø€ € € € € ØKÐKð:ð :ð :ð :ð :ð :ð :ð :ð :ð :rB   rê  )r"   r#   r!   r$   r%   r   )r3   r#   r4   r#   r5   r6   r%   r7   )r3   r#   rŽ   rE  r%   r7   )?r¹   Ú
__future__r   rp   Úloggingr1  rI   Úabcr   r   Údataclassesr   Úenumr   Úior   r	   Útypingr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  Úlangchain_core.documentsr   r   Ú	getLoggerr¶   r†   r   r2   rO   r   r½   rÍ   rÓ   rØ   r  r  rE  rN  r¯   r[  r#   rs  rœ  rÐ  rÙ  rá  rç  rê  r=   rB   r1   ú<module>rõ     s	  ððð ð* #Ð "Ð "Ð "Ð "Ð "à €€€Ø €€€Ø €€€Ø 	€	€	€	Ø #Ð #Ð #Ð #Ð #Ð #Ð #Ð #Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð Ø  Ð  Ð  Ð  Ð  Ð  Ð  Ð  ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð& €€€Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ Fà	ˆÔ	˜8Ñ	$Ô	$€à€WˆT˜Ð(Ñ(Ô(€ð )2ðð ð ð ð ð ð(*ð *ð *ð *ð&x5ð x5ð x5ð x5ð x5Ð*¨Cñ x5ô x5ð x5ðv6ð 6ð 6ð 6ð 6˜Lñ 6ô 6ð 6ð,ð ð ð ð ˆyñ ô ð ðð ð ð ð �ñ ô ð ðzð zð zð zð zñ zô zð zðzð ð ð ð �)ñ ô ð ðMð Mð Mð Mð Mñ Mô Mð Mðd €�$ÐÑÔð
>ð 
>ð 
>ð 
>ð 
>ñ 
>ô 
>ñ Ôð
>ðð ð ð ð"-Dð -Dð -Dð -Dð -D˜ñ -Dô -Dð -Dð`E6ð E6ð E6ð E6ð E6¨Lñ E6ô E6ð E6ðPð ð ð ð ˆs�Dñ ô ð ð2Qð Qð Qð Qð Q \ñ Qô Qð Qðh;ð ;ð ;ð ;ð ;�|ñ ;ô ;ð ;ð4;ð ;ð ;ð ;ð ;˜ñ ;ô ;ð ;ð>:ð :ð :ð :ð :Ð;ñ :ô :ð :ð:ð :ð :ð :ð :Ð9ñ :ô :ð :ð:ð :ð :ð :ð :Ð6ñ :ô :ð :ð :ð :rB   