§
    æ˜ºhŠ4  ã                   óè   — d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZ d dlZd dlmZ d dlmZmZ d dlmZ dZdZd	Zd
ZdZdZdZdZdZ ej        e ¦  «        Z! G d„ dee¦  «        Z"dS )é    N)ÚPath)ÚAnyÚDictÚListÚMappingÚOptionalÚSequenceÚUnion)ÚDocument)Ú	BaseModelÚroot_validator)Ú
BaseLoaderz#{http://www.w3.org/1999/xhtml}tableÚxpathÚidÚsourceÚnameÚ	structureÚtagÚprojectsz#https://api.docugami.com/v1preview1c                   ó„  — e Zd ZU dZeZeed<   	 ej	         
                    d¦  «        Zee         ed<   	 dZ	 dZeed<   	 dZ	 d	Zeed
<   	 dZeed<   	 dZeed<   	 d	Zeed<   	 dZeed<   	 ee         ed<   	 eee                  ed<   	 eeeeef                           ed<   	 dZeed<   	 edeeef         deeef         fd„¦   «         Z	 	 d&dedee         dee          de!e"         fd„Z#dede!e         fd„Z$dede!e         fd„Z%d edefd!„Z&	 	 d&d"ededee         d#ee          de!e"         f
d$„Z'de!e"         fd%„Z(dS )'ÚDocugamiLoaderzdLoad from `Docugami`.

    To use, you should have the ``dgml-utils`` python package installed.
    ÚapiÚDOCUGAMI_API_KEYÚaccess_tokeni   é    Úmin_text_lengthi   FÚinclude_xml_tagsr   Úparent_hierarchy_levelsÚdoc_idÚparent_id_keyÚsub_chunk_tablesTÚwhitespace_normalize_textÚ	docset_idÚdocument_idsÚ
file_pathsÚ(include_project_metadata_in_doc_metadataÚvaluesÚreturnc                 ó\  — |                      d¦  «        r$|                      d¦  «        rt          d¦  «        ‚|                      d¦  «        s$|                      d¦  «        st          d¦  «        ‚|                      d¦  «        r$|                      d¦  «        st          d¦  «        ‚|S )zÁValidate that either local file paths are given, or remote API docset ID.

        Args:
            values: The values to validate.

        Returns:
            The validated values.
        r%   r#   z7Cannot specify both file_paths and remote API docset_idz6Must specify either file_paths or remote API docset_idr   z7Must specify access token if using remote API docset_id)ÚgetÚ
ValueError)Úclsr'   s     úu/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/langchain_community/document_loaders/docugami.pyÚvalidate_local_or_remotez'DocugamiLoader.validate_local_or_remoteN   s±   € ð �:Š:�lÑ#Ô#ð 	X¨¯
ª
°;Ñ(?Ô(?ð 	XÝÐVÑWÔWÐWà�zŠz˜,Ñ'Ô'ð 	W°·
²
¸;Ñ0GÔ0Gð 	WÝÐUÑVÔVÐVà�:Š:�kÑ"Ô"ð 	X¨6¯:ª:°nÑ+EÔ+Eð 	XÝÐVÑWÔWÐWàˆó    NÚcontentÚdocument_nameÚadditional_doc_metadatac           	      óø  ‡ ‡‡— 	 ddl m} n# t          $ r t          d¦  «        ‚w xY w	 ddlm} ddlm} n# t          $ r t          d¦  «        ‚w xY wd|dt          fˆˆˆ fd	„}|                     t          j
        |¦  «        ¦  «        }|                     ¦   «         }	 ||	‰ j        ‰ j        ‰ j        ‰ j        ‰ j        ‰ j        ¬
¦  «        }
i }|
D ]†} ||¦  «        }|j                             t(          ¦  «        }|rX|||<   |j        rL ||j        ¦  «        }|j                             t(          ¦  «        }|r|j        r||j        ‰ j        <   |||<   Œ‡t1          |                     ¦   «         ¦  «        S )z6Parse a single DGML document into a list of Documents.r   ©ÚetreeúPCould not import lxml python package. Please install it with `pip install lxml`.)ÚChunk)Ú
get_chunkszaCould not import from dgml-utils python package. Please install it with `pip install dgml-utils`.Údg_chunkr(   c                 óz  •— t          j        | j                             ¦   «         ¦  «                             ¦   «         }t
          | j        t          |t          ‰t          ‰t          | j        t          | j        i}| j        }‰r‰j        r|                     ‰¦  «         t!          |d ‰j        …         |¬¦  «        S )N)Úpage_contentÚmetadata)ÚhashlibÚmd5ÚtextÚencodeÚ	hexdigestÚ	XPATH_KEYr   ÚID_KEYÚDOCUMENT_NAME_KEYÚDOCUMENT_SOURCE_KEYÚSTRUCTURE_KEYr   ÚTAG_KEYr   r&   Úupdater   Úmax_text_length)r9   Ú
_hashed_idr<   r?   r2   r1   Úselfs       €€€r-   Ú_build_framework_chunkz:DocugamiLoader._parse_dgml.<locals>._build_framework_chunk{   s²   ø€ å œ X¤]×%9Ò%9Ñ%;Ô%;Ñ<Ô<×FÒFÑHÔHˆJå˜8œ>Ý˜
Ý! =Ý# ]Ý˜xÔ1Ý˜œðˆHð ”=ˆDØ&ð =ØÔ@ð =Ø—O’OÐ$;Ñ<Ô<Ð<åØ!Ð"8 DÔ$8Ð"8Ô9Ø!ðñ ô ð r/   )r   rI   r"   r!   r   r   )Úlxmlr5   ÚImportErrorÚdgml_utils.modelsr7   Údgml_utils.segmentationr8   r   ÚparseÚioÚBytesIOÚgetrootr   rI   r"   r!   r   r   r<   r*   rC   Úparentr;   r    Úlistr'   )rK   r0   r1   r2   r5   r7   r8   rL   ÚtreeÚrootÚ	dg_chunksÚframework_chunksr9   Úframework_chunkÚchunk_idÚframework_parent_chunkÚ	parent_ids   ` ``             r-   Ú_parse_dgmlzDocugamiLoader._parse_dgmlc   s   øøø€ ð	Ø"Ð"Ð"Ð"Ð"Ð"Ð"øÝð 	ð 	ð 	Ýð=ñô ð ð	øøøð	Ø/Ð/Ð/Ð/Ð/Ð/Ø:Ð:Ð:Ð:Ð:Ð:Ð:øÝð 	ð 	ð 	ÝðCñô ð ð	øøøð	¨Uð 	µxð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð. �{Š{�2œ: gÑ.Ô.Ñ/Ô/ˆØ�|Š|‰~Œ~ˆà�JØØ Ô0Ø Ô0Ø&*Ô&DØ!Ô2Ø!Ô2Ø$(Ô$@ð
ñ 
ô 
ˆ	ð 13ÐØ!ð 
	Mð 
	MˆHØ4Ð4°XÑ>Ô>ˆOØ&Ô/×3Ò3µFÑ;Ô;ˆHØð MØ-<Ð  Ñ*Ø”?ð MØ-CÐ-CÀHÄOÑ-TÔ-TÐ*Ø 6Ô ?× CÒ CÅFÑ KÔ K�IØ ð MÐ%;Ô%Hð MØGP˜Ô0°Ô1CÑDØ6LÐ(¨Ñ3øåÐ$×+Ò+Ñ-Ô-Ñ.Ô.Ð.s   … Œ&ª7 ·Ac                 ó<  — | j         › d|› d�}g }|rŠt          j        |dd| j        › �i¬¦  «        }|j        rF|                     ¦   «         }|                     |d         ¦  «         |                     dd¦  «        }nt          d	|› d
|j        › d�¦  «        ‚|°Š|S )z1Gets all document details for the given docset IDú	/docsets/z
/documentsÚAuthorizationúBearer )ÚheadersÚ	documentsÚnextNúFailed to download ú
 (status: ú))	r   Úrequestsr*   r   ÚokÚjsonÚextendÚ	ExceptionÚstatus_code)rK   r#   ÚurlÚall_documentsÚresponseÚdatas         r-   Ú_document_details_for_docset_idz.DocugamiLoader._document_details_for_docset_id®   sØ   € à”Ð9Ð9 IÐ9Ð9Ð9ˆØˆàð 	Ý”|ØØ(Ð*G°DÔ4EÐ*GÐ*GÐHðñ ô ˆHð Œ{ð Ø—}’}‘”�Ø×$Ò$ T¨+Ô%6Ñ7Ô7Ð7Ø—h’h˜v tÑ,Ô,��åØP¨#ÐPÐP¸Ô9MÐPÐPÐPñô ð ð ð 	ð Ðr/   c                 ó>  — | j         › d|› �}g }|rŒt          j        d|dd| j        › �ii ¬¦  «        }|j        rF|                     ¦   «         }|                     |d         ¦  «         |                     dd¦  «        }nt          d	|› d
|j	        › d�¦  «        ‚|°Œ|S )z0Gets all project details for the given docset IDz/projects?docset.id=ÚGETrb   rc   ©rd   rs   r   rf   Nrg   rh   ri   )
r   rj   Úrequestr   rk   rl   rm   r*   rn   ro   )rK   r#   rp   Úall_projectsrr   rs   s         r-   Ú_project_details_for_docset_idz-DocugamiLoader._project_details_for_docset_idÃ   sÜ   € à”Ð:Ð:¨yÐ:Ð:ˆØˆàð 	ÝÔ'ØØØ(Ð*G°DÔ4EÐ*GÐ*GÐHØð	ñ ô ˆHð Œ{ð Ø—}’}‘”�Ø×#Ò# D¨Ô$4Ñ5Ô5Ð5Ø—h’h˜v tÑ,Ô,��åØP¨#ÐPÐP¸Ô9MÐPÐPÐPñô ð ð ð 	ð  Ðr/   Úprojectc                 ó
  — |                      t          ¦  «        }| j        › d|› d�}g }i }|r™t          j        d|dd| j        › �ii ¬¦  «        }|j        rF|                     ¦   «         }|                     |d         ¦  «         |                      dd	¦  «        }n(|j	        d
k    r|S t          d|› d|j	        › d�¦  «        ‚|°™|D �]¸}|                      d¦  «        }	|                      d¦  «        }
|                      d¦  «        }|	dk    �ro|
�rl|�ri|t                   }i }t          j        d|
› d�dd| j        › �ii ¬¦  «        }|j        �r	 ddlm} n# t          $ r t          d¦  «        ‚w xY w|                     t          j        |j        ¦  «        ¦  «        }|                     ¦   «         }|j        }|                     d|¬¦  «        }|D ]Š}|                     d|¬¦  «        d         j        }d                     |                     d|¬¦  «        d                              ¦   «         ¦  «                             ¦   «         }|d	| j        …         ||<   Œ‹|||<   �Œ¢t          d|
› d�dz   ¦  «        ‚�Œº|S )z#Gets project metadata for all filesz
/projects/z/artifacts/latestrv   rb   rc   rw   Ú	artifactsrf   Ni”  rg   rh   ri   r   rp   Údocumentzreport-values.xmlz/contentr   r4   r6   z
//pr:Entry)Ú
namespacesz./pr:Headingú z
./pr:Valuez	/content z (status: {response.status_code}))r*   rC   r   rj   rx   r   rk   rl   rm   ro   rn   rM   r5   rN   rQ   rR   rS   r0   rT   Únsmapr   r?   ÚjoinÚitertextÚstripÚmax_metadata_length)rK   r{   Ú
project_idrp   Úall_artifactsÚper_file_metadatarr   rs   ÚartifactÚartifact_nameÚartifact_urlÚartifact_docr   r<   r5   Úartifact_treeÚartifact_rootÚnsÚentriesÚentryÚheadingÚvalues                         r-   Ú_metadata_for_projectz$DocugamiLoader._metadata_for_projectÚ   s!  € à—[’[¥Ñ(Ô(ˆ
à”ÐBÐB ZÐBÐBÐBˆØˆà"$ÐØð 	ÝÔ'ØØØ(Ð*G°DÔ4EÐ*GÐ*GÐHØð	ñ ô ˆHð Œ{ð 
Ø—}’}‘”�Ø×$Ò$ T¨+Ô%6Ñ7Ô7Ð7Ø—h’h˜v tÑ,Ô,��ØÔ%¨Ò,Ð,à(Ð(åØP¨#ÐPÐP¸Ô9MÐPÐPÐPñô ð ð ð 	ð& &ð (	ñ (	ˆHØ$ŸLšL¨Ñ0Ô0ˆMØ#Ÿ<š<¨Ñ.Ô.ˆLØ#Ÿ<š<¨
Ñ3Ô3ˆLàÐ 3Ò3Ñ3¸Ñ3ÈÑ3Ø%¥fÔ-�Ø!#�õ $Ô+ØØ#Ð-Ð-Ð-Ø,Ð.K¸Ô8IÐ.KÐ.KÐLØð	ñ ô �ð ”;ñ ðØ.Ð.Ð.Ð.Ð.Ð.Ð.øÝ&ð ð ð Ý)ðIñô ð ðøøøð
 %*§K¢Kµ´
¸8Ô;KÑ0LÔ0LÑ$MÔ$M�MØ$1×$9Ò$9Ñ$;Ô$;�MØ&Ô,�BØ+×1Ò1°,È2Ð1ÑNÔN�GØ!(ð Nð N˜Ø"'§+¢+¨nÈ +Ñ"LÔ"LÈQÔ"OÔ"T˜Ø #§¢Ø!ŸKšK¨À˜KÑDÔDÀQÔG×PÒPÑRÔRñ!ô !çš%™'œ'ð ð -2Ð2L°DÔ4LÐ2LÔ,M˜ Ñ)Ð)Ø08Ð% fÑ-Ñ-å#ØE¨lÐEÐEÐEØ<ñ=ñô ð ùð
 !Ð s   ÅEÅE6Údocument_idÚadditional_metadatac                 óä   — | j         › d|› d|› d�}t          j        d|dd| j        › �ii ¬¦  «        }|j        r|                      |j        ||¬¦  «        S t          d	|› d
|j        › d�¦  «        ‚)zLoad chunks for a document.ra   z/documents/z/dgmlrv   rb   rc   rw   )r0   r1   r2   rg   rh   ri   )	r   rj   rx   r   rk   r_   r0   rn   ro   )rK   r•   r#   r1   r–   rp   rr   s          r-   Ú_load_chunks_for_documentz(DocugamiLoader._load_chunks_for_document!  sº   € ð ”ÐLÐL IÐLÐL¸+ÐLÐLÐLˆåÔ#ØØØ$Ð&C°Ô0AÐ&CÐ&CÐDØð	
ñ 
ô 
ˆð Œ;ð 		Ø×#Ò#Ø Ô(Ø+Ø(;ð $ñ ô ð õ ØL cÐLÐL°XÔ5IÐLÐLÐLñô ð r/   c                 ó,  ‡ — g }‰ j         �r‰ j        �r‰                      ‰ j        ¦  «        }‰ j        rˆ fd„|D ¦   «         }‰                      ‰ j        ¦  «        }i }|rW‰ j        rP|D ]M}‰                      |¦  «        }|D ]3}||vr||         ||<   Œ||                              ||         ¦  «         Œ4ŒN|D ]_}|t                   }	| 	                    t          ¦  «        }
| 	                    |	¦  «        }|‰                      |	‰ j        |
|¬¦  «        z  }Œ`ny‰ j        rr‰ j        D ]j}t          |¦  «        }t          |d¦  «        5 }|‰                      |                     ¦   «         |j        ¬¦  «        z  }ddd¦  «         n# 1 swxY w Y   Œk|S )zLoad documents.c                 ó<   •— g | ]}|t                    ‰j        v ¯|‘ŒS © )rC   r$   )Ú.0ÚdrK   s     €r-   ú
<listcomp>z'DocugamiLoader.load.<locals>.<listcomp>E  s1   ø€ ð %ð %ð %Ø°Aµf´IÀÔARÐ4RÐ4R�AÐ4RÐ4RÐ4Rr/   )r•   r#   r1   r–   Úrb)r0   r1   N)r   r#   rt   r$   rz   r&   r”   rH   rC   r*   rD   r˜   r%   r   Úopenr_   Úreadr   )rK   ÚchunksÚ_document_detailsÚ_project_detailsÚcombined_project_metadatar{   r<   Úfile_idÚdocr   Údoc_nameÚdoc_metadataÚpathÚfiles   `             r-   ÚloadzDocugamiLoader.load=  sM  ø€ à!#ˆàÔñ '	 ¤ñ '	à $× DÒ DÀTÄ^Ñ TÔ TÐØÔ ð ð%ð %ð %ð %Ø0ð%ñ %ô %Ð!ð  $×BÒBÀ4Ä>ÑRÔRÐØ9;Ð%Øð 	Y DÔ$Qð 	Yð  0ð Yð Y�GØ#×9Ò9¸'ÑBÔB�HØ#+ð Yð Y˜Ø"Ð*CÐCÐCØAIÈ'ÔARÐ5°gÑ>Ð>à5°gÔ>×EÒEÀhÈwÔFWÑXÔXÐXÐXð	Yð )ð 	ð 	�Ø�Vœ�ØŸ7š7Õ#4Ñ5Ô5�Ø8×<Ò<¸VÑDÔD�Ø˜$×8Ò8Ø &Ø"œnØ"*Ø(4ð	 9ñ ô ñ ��ð		ð Œ_ð 	àœð ð �Ý˜D‘z”z�Ý˜$ Ñ%Ô%ð ¨Ø˜d×.Ò.Ø $§	¢	¡¤Ø&*¤ið /ñ ô ñ �Fðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð øð ˆs   Å
2FÆF	ÆF	)NN))Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚDEFAULT_API_ENDPOINTr   ÚstrÚ__annotations__ÚosÚenvironr*   r   r   rI   r   Úintr…   r   Úboolr   r    r!   r"   r	   r
   r   r&   r   r   r   r.   Úbytesr   r   r   r_   rt   rz   r”   r˜   r¬   r›   r/   r-   r   r      sÓ  € € € € € € ðð ð
 $€CˆÐ#Ð#Ñ#Ø+à"$¤*§.¢.Ð1CÑ"DÔ"D€L�(˜3”-ÐDÐDÑDØ/à€OØ,à€O�SÐÐÑØSàÐØ/à"Ð�dÐ"Ð"Ñ"Ø8à#$Ð˜SÐ$Ð$Ñ$ØKà!€M�3Ð!Ð!Ñ!Ø)à"Ð�dÐ"Ð"Ñ"Ø9à&*Ð˜tÐ*Ð*Ñ*ð'ð ˜Œ}ÐÐÑØ,à˜8 Cœ=Ô)Ð)Ð)Ñ)Ø/à˜ %¨¨c¨	Ô"2Ô3Ô4Ð4Ð4Ñ4Ø&à59Ð,¨dÐ9Ð9Ñ9ØVàð¨d°3¸°8¬nð ÀÀcÈ3ÀhÄð ð ð ñ „^ðð. (,Ø59ð	I/ð I/àðI/ð   ”}ðI/ð "*¨'Ô!2ð	I/ð
 
ˆhŒðI/ð I/ð I/ð I/ðV¸ð ÀÀdÄð ð ð ð ð*¸ð ÀÀTÄ
ð ð ð ð ð.E!¨Tð E!°dð E!ð E!ð E!ð E!ðV (,Ø15ðð àðð ðð   ”}ð	ð
 & gÔ.ðð 
ˆhŒðð ð ð ð8-�d˜8”nð -ð -ð -ð -ð -ð -r/   r   )#r=   rR   Úloggingr´   Úpathlibr   Útypingr   r   r   r   r   r	   r
   rj   Úlangchain_core.documentsr   Úlangchain_core.pydantic_v1r   r   Ú)langchain_community.document_loaders.baser   Ú
TABLE_NAMErB   rC   rE   rD   rF   rG   ÚPROJECTS_KEYr±   Ú	getLoggerr­   Úloggerr   r›   r/   r-   ú<module>rÃ      s8  ðØ €€€Ø 	€	€	€	Ø €€€Ø 	€	€	€	Ø Ð Ð Ð Ð Ð Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ Fà €€€Ø -Ð -Ð -Ð -Ð -Ð -Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @à @Ð @Ð @Ð @Ð @Ð @à2€
à€	Ø	€ØÐ ØÐ Ø€Ø
€Ø€à<Ð à	ˆÔ	˜8Ñ	$Ô	$€ðMð Mð Mð Mð M�Z ñ Mô Mð Mð Mð Mr/   