§
    Þ˜ºh"U  ã                   ó.  — d Z ddlZddlmZ ddlmZmZmZ ddlZddl	Zddlm
Z
 ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZ  ej        e¦  «        Ze G d„ de¦  «        ¦   «         Z G d„ de
j        ¦  «        Z	 d%de
j        dej         dej         dej         deej                  de!de!fd„Z" G d„ de
j        ¦  «        Z# G d„ de
j        ¦  «        Z$ G d„ d e¦  «        Z% G d!„ d"e
j        ¦  «        Z& G d#„ d$e
j        ¦  «        Z'dS )&zTPyTorch IdeficsVision model: a copy of CLIPVisionModel using a simpler config objecté    N)Ú	dataclass)ÚCallableÚOptionalÚUnion)Únné   )ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONS)ÚModelOutputÚcan_return_tupleÚloggingé   )ÚIdeficsVisionConfigc                   óÄ   — e Zd ZU dZdZeej                 ed<   dZ	eej                 ed<   dZ
eeej        df                  ed<   dZeeej        df                  ed<   dS )ÚIdeficsVisionModelOutputaÝ  
    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.

    Args:
        image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
            The image embeddings obtained by applying the projection layer to the pooler_output.
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚimage_embedsÚlast_hidden_state.Úhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚFloatTensorÚ__annotations__r   r   Útupler   © ó    új/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/transformers/models/idefics/vision.pyr   r   (   s“   € € € € € € ðð ð* 15€L�(˜5Ô,Ô-Ð4Ð4Ñ4Ø59Ð�x Ô 1Ô2Ð9Ð9Ñ9Ø=A€M�8˜E %Ô"3°SÐ"8Ô9Ô:ÐAÐAÑAØ:>€J�˜˜uÔ0°#Ð5Ô6Ô7Ð>Ð>Ñ>Ð>Ð>r"   r   c                   óz   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdd	ej	        d
e
dej        fd„Zˆ xZS )ÚIdeficsVisionEmbeddingsÚconfigc                 óz  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        ¦  «        ¦  «        | _        t          j        |j        | j        | j        | j        d¬¦  «        | _        | j        | j        z  dz  | _        | j        dz   | _        t          j        | j        | j        ¦  «        | _        |                      dt          j        | j        ¦  «                             d¦  «        d¬¦  «         d S )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasé   r   Úposition_ids)r   éÿÿÿÿ)Ú
persistent)ÚsuperÚ__init__r&   Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	Parameterr   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embeddingÚregister_bufferÚarangeÚexpand©Úselfr&   Ú	__class__s     €r#   r2   z IdeficsVisionEmbeddings.__init__G   s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒå!œ|­E¬K¸¼Ñ,GÔ,GÑHÔHˆÔå!œyØÔ+ØœØœØ”?Øð 
ñ  
ô  
ˆÔð !œO¨t¬Ñ>À1ÑDˆÔØ!Ô-°Ñ1ˆÔÝ"$¤,¨tÔ/AÀ4Ä>Ñ"RÔ"RˆÔØ×Ò˜^­U¬\¸$Ô:LÑ-MÔ-M×-TÒ-TÐU\Ñ-]Ô-]ÐjoÐÑpÔpÐpÐpÐpr"   Ú
embeddingsÚheightÚwidthÚreturnc                 óà  — |j         d         dz
  }|                      | j        ¦  «        }|j         d         dz
  }||k    r||k    r|S |dd…df         }|dd…dd…f         }|j         d         }	|| j        j        z  }
|| j        j        z  }|
dz   |dz   }}
t          j        |¦  «        }|                     dt          |¦  «        t          |¦  «        |	¦  «        }| 	                    dddd¦  «        }|j
        t          j        k    }|r9t                               d¦  «         |                     t          j        ¦  «        }t"          j                             ||
|z  ||z  fd	d
¬¦  «        }|r|                     t          j        ¦  «        }t          |
¦  «        |j         d         k    st          |¦  «        |j         d         k    rJt)          dt          |
¦  «        t          |¦  «        f› d|j         d         |j         d         f› d�¦  «        ‚| 	                    dddd¦  «                             dd|	¦  «        }t          j        |                     d¦  «        |fd¬¦  «        S )a#  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher
        resolution images.

        Source:
        https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174
        r   Nr   r/   gš™™™™™¹?r   r-   zËUpcasting patch_pos_embed to fp32 for interpolation since `upsample_bicubic2d_out_frame` in nn.functional.interpolate is not implemented for 'torch.bfloat16' dtype. This will result in a slight overhead.ÚbicubicF)Úscale_factorÚmodeÚalign_cornerséþÿÿÿzNumber of patches for images (z/) don't match the shape of position embedding (ú)©Údim)Úshaper@   r.   r&   r6   ÚmathÚsqrtÚreshapeÚintÚpermuteÚdtyper   Úbfloat16ÚloggerÚwarning_onceÚtoÚfloatr   Ú
functionalÚinterpolateÚ
ValueErrorÚviewÚcatÚ	unsqueeze)rE   rG   rH   rI   r=   Ú	pos_embedr>   Úclass_pos_embedÚpatch_pos_embedr4   Únum_h_patchesÚnum_w_patchesÚsqrt_num_positionsÚfp32_upcastings                 r#   Úinterpolate_pos_encodingz0IdeficsVisionEmbeddings.interpolate_pos_encoding^   s©  € ð !Ô& qÔ)¨AÑ-ˆØ×+Ò+¨DÔ,=Ñ>Ô>ˆ	Ø!œ¨Ô*¨QÑ.ˆØ˜-Ò'Ð'¨F°eªO¨OØÐØ# A A A q Dœ/ˆØ# A A A q r r EÔ*ˆàÔ$ RÔ(ˆ	Ø $¤+Ô"8Ñ8ˆØ ¤Ô!7Ñ7ˆð (5°sÑ':¸MÈCÑ<O�}ˆÝ!œY }Ñ5Ô5ÐØ)×1Ò1°!µSÐ9KÑ5LÔ5LÍcÐRdÑNeÔNeÐgpÑqÔqˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆØ(Ô.µ%´.Ò@ˆØð 	>Ý×Òðhñô ð ð .×0Ò0µ´Ñ=Ô=ˆOÝœ-×3Ò3ØØ'Ð*<Ñ<¸mÐN`Ñ>`ÐaØØð	 4ñ 
ô 
ˆð ð 	AØ-×0Ò0µ´Ñ@Ô@ˆOÝˆ}ÑÔ Ô!6°rÔ!:Ò:Ð:½cÀ-Ñ>PÔ>PÐTcÔTiÐjlÔTmÒ>mÐ>mÝðhµ°]Ñ1CÔ1CÅSÈÑEWÔEWÐ0Xð hð hØ0?Ô0EÀbÔ0IÈ?ÔK`ÐacÔKdÐ/eðhð hð hñô ð ð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ)ÑTÔTˆÝŒy˜/×3Ò3°AÑ6Ô6¸ÐHÈaÐPÑPÔPÐPr"   FÚpixel_valuesrm   c                 ó<  — |j         \  }}}}|s<|| j        k    s|| j        k    r&t          d|› d|› d| j        › d| j        › d�	¦  «        ‚| j        j        j        }|                      |                     |¬¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }| j	         
                    |dd¦  «        }	t          j        |	|gd¬	¦  «        }
|r|
|                      |
||¦  «        z   }
n|
|                      | j        ¦  «        z   }
|
S )
NzInput image size (Ú*z) doesn't match model (z8). You should try to set `interpolate_pos_encoding=True`)rZ   r-   r   r/   rR   )rT   r5   rb   r<   ÚweightrZ   r^   ÚflattenÚ	transposer9   rC   r   rd   rm   r@   r.   )rE   rn   rm   Ú
batch_sizer;   rH   rI   Útarget_dtypeÚpatch_embedsÚclass_embedsrG   s              r#   ÚforwardzIdeficsVisionEmbeddings.forward�   sc  € Ø2>Ô2DÑ/ˆ
�L &¨%Ø'ð 	Ø˜œÒ(Ð(¨E°T´_Ò,DÐ,DÝ ðu¨ð uð u°%ð uð uØœðuð uØ+/¬?ðuð uð uñô ð ð
 Ô+Ô2Ô8ˆØ×+Ò+¨L¯OªOÀ,¨OÑ,OÔ,OÑPÔPˆà#×+Ò+¨AÑ.Ô.×8Ò8¸¸AÑ>Ô>ˆàÔ+×2Ò2°:¸qÀ"ÑEÔEˆÝ”Y ¨lÐ;ÀÐCÑCÔCˆ
ð $ð 	QØ# d×&CÒ&CÀJÐPVÐX]Ñ&^Ô&^Ñ^ˆJˆJà# d×&=Ò&=¸dÔ>OÑ&PÔ&PÑPˆJàÐr"   ©F)r   r   r   r   r2   r   ÚTensorrX   rm   r   Úboolrx   Ú__classcell__©rF   s   @r#   r%   r%   F   sÂ   ø€ € € € € ðqÐ2ð qð qð qð qð qð qð./Q°5´<ð /QÈð /QÐUXð /QÐ]bÔ]ið /Qð /Qð /Qð /Qðbð  EÔ$5ð ÐQUð ÐbgÔbnð ð ð ð ð ð ð ð r"   r%   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óÀ  — t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |dt           j        ¬¦  «                             |j        ¦  «        }t          j         	                    ||| j
        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )Nr/   rP   )rS   rZ   )ÚpÚtrainingr   r-   )r   Úmatmulrs   r   r`   ÚsoftmaxÚfloat32r^   rZ   r…   rˆ   Ú
contiguous)
r   r€   r�   r‚   rƒ   r„   r…   ÚkwargsÚattn_weightsÚattn_outputs
             r#   Úeager_attention_forwardr�   ª   sÃ   € õ ”<  s§}¢}°R¸Ñ'<Ô'<Ñ=Ô=ÀÑG€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r"   c                   ó¾   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej        deej                 deej                 d	ee	         d
e
ej        eej                 f         f
d„Zˆ xZS )ÚIdeficsVisionAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr&   c                 ó‚  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        |j
        | _        d| _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        t          j        | j        | j        ¦  «        | _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿F)r1   r2   r&   r3   r4   Únum_attention_headsÚ	num_headsÚhead_dimrb   ÚscaleÚattention_dropoutr…   Ú	is_causalr   ÚLinearÚk_projÚv_projÚq_projÚout_projrD   s     €r#   r2   zIdeficsVisionAttention.__init__Ä   s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØÔ/ˆŒØˆŒå”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝ”i ¤°´Ñ?Ô?ˆŒÝœ	 $¤.°$´.ÑAÔAˆŒˆˆr"   NFr   rƒ   Úcausal_attention_maskÚoutput_attentionsrJ   c           
      óÈ  — |j         \  }}}|                      |¦  «        }|                      |¦  «        }	|                      |¦  «        }
|                     ||| j        | j        ¦  «                             dd¦  «        }|	                     ||| j        | j        ¦  «                             dd¦  «        }	|
                     ||| j        | j        ¦  «                             dd¦  «        }
| j        j	        dk    r|�|�||z   }n|�|}n	|du| _
        t          }| j        j	        dk    rD| j        j	        dk    r|rt                               d¦  «         nt          | j        j	                 } || ||	|
|| j
        | j        | j        sdn| j        ¬	¦  «        \  }}|                     |||¦  «                             ¦   «         }|                      |¦  «        }|sd}||fS )
z#Input shape: Batch x Time x Channelr   r-   Úflash_attention_2NÚeagerÚsdpazã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r~   )r™   r„   r…   )rT   r�   r›   rœ   rc   r•   r–   rs   r&   Ú_attn_implementationr™   r�   r\   r]   r   r—   rˆ   r…   rW   rŒ   rž   )rE   r   rƒ   rŸ   r    rt   Ú
seq_lengthr4   ÚqueriesÚkeysÚvaluesÚattention_interfacer�   rŽ   s                 r#   rx   zIdeficsVisionAttention.forwardØ   s  € ð -:Ô,?Ñ)ˆ
�J 	à—+’+˜mÑ,Ô,ˆØ�{Š{˜=Ñ)Ô)ˆØ—’˜]Ñ+Ô+ˆà—,’,˜z¨:°t´~ÀtÄ}ÑUÔU×_Ò_Ð`aÐcdÑeÔeˆØ�yŠy˜ Z°´ÀÄÑOÔO×YÒYÐZ[Ð]^Ñ_Ô_ˆØ—’˜Z¨°T´^ÀTÄ]ÑSÔS×]Ò]Ð^_ÐabÑcÔcˆð Œ;Ô+Ð/BÒBÐBØÐ)Ð.CÐ.OØ!/Ð2GÑ!G��Ø&Ð2Ø!6�øà2¸$Ð>ˆDŒNå(?ÐØŒ;Ô+¨wÒ6Ð6ØŒ{Ô/°6Ò9Ð9Ð>OÐ9Ý×#Ò#ðLñô ð ð õ
 '>¸d¼kÔ>^Ô&_Ð#à$7Ð$7ØØØØØØ”nØ”JØ#œ}Ð>�C�C°$´,ð	%
ñ 	%
ô 	%
Ñ!ˆ�\ð "×)Ò)¨*°jÀ)ÑLÔL×WÒWÑYÔYˆØ—m’m KÑ0Ô0ˆØ ð 	 ØˆLØ˜LÐ(Ð(r"   )NNF)r   r   r   r   r   r2   r   rz   r   r{   r    rx   r|   r}   s   @r#   r’   r’   Á   sÍ   ø€ € € € € ØGÐGðBÐ2ð Bð Bð Bð Bð Bð Bð. 26Ø8<Ø,1ð5)ð 5)à”|ð5)ð ! ¤Ô.ð5)ð  (¨¬Ô5ð	5)ð
 $ Dœ>ð5)ð 
ˆuŒ|˜X e¤lÔ3Ð3Ô	4ð5)ð 5)ð 5)ð 5)ð 5)ð 5)ð 5)ð 5)r"   r’   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚIdeficsVisionMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S ©N)r1   r2   r&   r	   Ú
hidden_actÚactivation_fnr   rš   r3   Úintermediate_sizeÚfc1Úfc2rD   s     €r#   r2   zIdeficsVisionMLP.__init__  sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr"   r   rJ   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r®   )r²   r°   r³   )rE   r   s     r#   rx   zIdeficsVisionMLP.forward  s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr"   )r   r   r   r2   r   rz   rx   r|   r}   s   @r#   r¬   r¬     sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r"   r¬   c                   ó„   ‡ — e Zd Zdefˆ fd„Z	 d
dej        dej        dej        dee         de	ej
                 f
d	„Zˆ xZS )ÚIdeficsVisionEncoderLayerr&   c                 óD  •— t          ¦   «                              ¦   «          |j        | _        t	          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _	        t          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _        d S ©N)Úeps)r1   r2   r3   r4   r’   Ú	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1r¬   ÚmlpÚlayer_norm2rD   s     €r#   r2   z"IdeficsVisionEncoderLayer.__init__"  s   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝ/°Ñ7Ô7ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ# FÑ+Ô+ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÐÐr"   Fr   rƒ   rŸ   r    rJ   c                 óî   — |}|                       |¦  «        }|                      ||||¬¦  «        \  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|f}|r||fz  }|S )aI  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
                `(config.encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        )r   rƒ   rŸ   r    )r½   rº   r¿   r¾   )rE   r   rƒ   rŸ   r    ÚresidualrŽ   Úoutputss           r#   rx   z!IdeficsVisionEncoderLayer.forward*  s¥   € ð" !ˆà×(Ò(¨Ñ7Ô7ˆØ&*§n¢nØ'Ø)Ø"7Ø/ð	 '5ñ '
ô '
Ñ#ˆ�|ð ! =Ñ0ˆà ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆà Ð"ˆàð 	'Ø˜�Ñ&ˆGàˆr"   ry   )r   r   r   r   r2   r   rz   r   r{   r    r   rx   r|   r}   s   @r#   r¶   r¶   !  s¬   ø€ € € € € ðSÐ2ð Sð Sð Sð Sð Sð Sð -2ð&ð &à”|ð&ð œð&ð  %œ|ð	&ð
 $ Dœ>ð&ð 
ˆuÔ Ô	!ð&ð &ð &ð &ð &ð &ð &ð &r"   r¶   c                   óÄ   ‡ — e Zd ZdZdefˆ fd„Ze	 	 	 	 	 ddeej	                 deej	                 dee
         dee
         d	ee
         d
eeef         fd„¦   «         Zˆ xZS )ÚIdeficsVisionEncoderz¿
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`IdeficsVisionEncoderLayer`].

    Args:
        config: IdeficsVisionConfig
    r&   c                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r!   )r¶   )Ú.0Ú_r&   s     €r#   ú
<listcomp>z1IdeficsVisionEncoder.__init__.<locals>.<listcomp>`  s"   ø€ Ð$pÐ$pÐ$pÈ1Õ%>¸vÑ%FÔ%FÐ$pÐ$pÐ$pr"   F)	r1   r2   r&   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚgradient_checkpointingrD   s    `€r#   r2   zIdeficsVisionEncoder.__init__]  sb   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$pÐ$pÐ$pÐ$pÕPUÐV\ÔVnÑPoÔPoÐ$pÑ$pÔ$pÑqÔqˆŒØ&+ˆÔ#Ð#Ð#r"   Nrƒ   rŸ   r    Úoutput_hidden_statesÚreturn_dictrJ   c                 ó@  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|rdnd}|rdnd}|}	t	          | j        ¦  «        D ]2\  }
}|r||	fz   } ||	|||¬¦  «        }|d         }	|r||d         fz   }Œ3|r||	fz   }t          |	||¬¦  «        S )aÕ  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Causal mask for the text model. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        Nr!   )r    r   r   )r   r   r   )r&   r    rÏ   Úuse_return_dictÚ	enumeraterÍ   r   )rE   Úinputs_embedsrƒ   rŸ   r    rÏ   rÐ   Úencoder_statesÚall_attentionsr   ÚidxÚencoder_layerÚlayer_outputss                r#   rx   zIdeficsVisionEncoder.forwardc  s   € ðN 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔB]ˆà3Ð=˜˜¸ˆØ0Ð:˜˜°dˆà%ˆÝ"+¨D¬KÑ"8Ô"8ð 	Fð 	FÑˆC�Ø#ð CØ!/°=Ð2BÑ!B�Ø)˜MØØØ%Ø"3ð	ñ ô ˆMð *¨!Ô,ˆMà ð FØ!/°=ÀÔ3CÐ2EÑ!E�øàð 	?Ø+¨}Ð.>Ñ>ˆNåØ+¸>ÐVdð
ñ 
ô 
ð 	
r"   )NNNNN)r   r   r   r   r   r2   r   r   r   rz   r{   r   r    r   rx   r|   r}   s   @r#   rÄ   rÄ   T  sõ   ø€ € € € € ðð ð,Ð2ð ,ð ,ð ,ð ,ð ,ð ,ð ð 26Ø8<Ø,0Ø/3Ø&*ðD
ð D
ð ! ¤Ô.ðD
ð  (¨¬Ô5ð	D
ð
 $ Dœ>ðD
ð ' tœnðD
ð ˜d”^ðD
ð 
ˆu�oÐ%Ô	&ðD
ð D
ð D
ñ ÔðD
ð D
ð D
ð D
ð D
r"   rÄ   c                   ó¦   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 ddeej                 dee         dee         dee         d	ee         d
e	e
ef         fd„Zˆ xZS )ÚIdeficsVisionTransformerr&   c                 ó4  •— t          ¦   «                              ¦   «          || _        |j        }t	          |¦  «        | _        t          j        ||j        ¬¦  «        | _	        t          |¦  «        | _        t          j        ||j        ¬¦  «        | _        d S r¸   )r1   r2   r&   r3   r%   rG   r   r»   r¼   Úpre_layrnormrÄ   ÚencoderÚpost_layernorm)rE   r&   r4   rF   s      €r#   r2   z!IdeficsVisionTransformer.__init__­  s€   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ&ˆ	å1°&Ñ9Ô9ˆŒÝœL¨¸Ô8MÐNÑNÔNˆÔÝ+¨FÑ3Ô3ˆŒÝ œl¨9¸&Ô:OÐPÑPÔPˆÔÐÐr"   NFrn   r    rÏ   rm   rÐ   rJ   c                 óÂ  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|€t	          d¦  «        ‚|                      ||¬¦  «        }|                      |¦  «        }|                      ||||¬¦  «        }|d         }|dd…ddd…f         }	|                      |	¦  «        }	|s||	f|dd…         z   S t          ||	|j
        |j        ¬¦  «        S )z
        Returns:

        Nz You have to specify pixel_values)rm   )rÔ   r    rÏ   rÐ   r   r   )r   Úpooler_outputr   r   )r&   r    rÏ   rÒ   rb   rG   rÝ   rÞ   rß   r   r   r   )
rE   rn   r    rÏ   rm   rÐ   r   Úencoder_outputsr   Úpooled_outputs
             r#   rx   z IdeficsVisionTransformer.forward¸  s/  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔB]ˆàÐÝÐ?Ñ@Ô@Ð@àŸš¨ÐOg˜ÑhÔhˆØ×)Ò)¨-Ñ8Ô8ˆàŸ,š,Ø'Ø/Ø!5Ø#ð	 'ñ 
ô 
ˆð ,¨AÔ.ÐØ)¨!¨!¨!¨Q°°°¨'Ô2ˆØ×+Ò+¨MÑ:Ô:ˆàð 	LØ% }Ð5¸ÈÈÈÔ8KÑKÐKå)Ø/Ø'Ø)Ô7Ø&Ô1ð	
ñ 
ô 
ð 	
r"   )NNNFN)r   r   r   r   r2   r   r   r   r{   r   r    r   rx   r|   r}   s   @r#   rÛ   rÛ   ¬  sÓ   ø€ € € € € ðQÐ2ð Qð Qð Qð Qð Qð Qð 59Ø,0Ø/3Ø38Ø&*ð+
ð +
à˜uÔ0Ô1ð+
ð $ Dœ>ð+
ð ' tœnð	+
ð
 #+¨4¤.ð+
ð ˜d”^ð+
ð 
ˆuÐ0Ð0Ô	1ð+
ð +
ð +
ð +
ð +
ð +
ð +
ð +
r"   rÛ   )r~   )(r   rU   Údataclassesr   Útypingr   r   r   r   Útorch.utils.checkpointr   Úactivationsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   Úmodeling_utilsr   Úutilsr   r   r   Úconfiguration_ideficsr   Ú
get_loggerr   r\   r   ÚModuler%   rz   r_   r�   r’   r¬   r¶   rÄ   rÛ   r!   r"   r#   ú<module>rï      sè  ðð [Ð Zà €€€Ø !Ð !Ð !Ð !Ð !Ð !Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,à €€€Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ 5Ð 5Ð 5Ð 5Ð 5Ð 5ðð ð ð ð ð ð ð ð ð ð
 7Ð 6Ð 6Ð 6Ð 6Ð 6ð 
ˆÔ	˜HÑ	%Ô	%€ð ð?ð ?ð ?ð ?ð ?˜{ñ ?ô ?ñ „ð?ð:`ð `ð `ð `ð `˜bœiñ `ô `ð `ðV ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ˜Uœ\Ô*ð%ð ð%ð ð%ð %ð %ð %ð.L)ð L)ð L)ð L)ð L)˜RœYñ L)ô L)ð L)ð`ð ð ð ð �r”yñ ô ð ð /ð /ð /ð /ð /Ð :ñ /ô /ð /ðfT
ð T
ð T
ð T
ð T
˜2œ9ñ T
ô T
ð T
ðp7
ð 7
ð 7
ð 7
ð 7
˜rœyñ 7
ô 7
ð 7
ð 7
ð 7
r"   