§
    Ý˜ºh¤1  ã                   óÞ  — d Z ddlmZmZ ddlZddlmZ ddlmZm	Z	 ddl
mZmZmZ  e¦   «         rddlmZ dd	lmZmZmZ  e	j        e¦  «        Z G d
„ d¦  «        Z	 d#dej        dej        dej        deej        eej        ej        f         f         fd„Zeej        ef         Z	 	 	 	 	 d$dej        dee         deeeef                  dee         ddf
d„Zdej        dedej        fd„Z	 	 	 	 d%dej         j!        dej        dej        dej        deej        df         dee"         dee"         d eej                 d!eej                 deej        eej                 f         fd"„Z#dS )&a7  
Partially inspired by torchtune's flex attention implementation

Citation:
@software{torchtune,
  title = {torchtune: PyTorch's finetuning library},
  author = {torchtune maintainers and contributors},
  url = {https//github.com/pytorch/torchtune},
  license = {BSD-3-Clause},
  month = apr,
  year = {2024}
}
é    )ÚOptionalÚUnionN)Úversioné   )Úis_torch_flex_attn_availableÚlogging)Ú_torch_versionÚis_torch_less_or_equalÚis_torchdynamo_compiling)Ú_DEFAULT_SPARSE_BLOCK_SIZE)Ú	BlockMaskÚcreate_block_maskÚflex_attentionc                   ó|   ‡ — e Zd ZdZdZdZdZˆ fd„Zej	         
                    d¬¦  «        d„ ¦   «         Zd„ Zˆ xZS )ÚWrappedFlexAttentionzh
    We are doing a singleton class so that flex attention is compiled once when it's first called.
    NFc                 ól   •— | j         €&t          ¦   «                              | ¦  «        | _         | j         S ©N)Ú	_instanceÚsuperÚ__new__)ÚclsÚargsÚkwargsÚ	__class__s      €úp/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/transformers/integrations/flex_attention.pyr   zWrappedFlexAttention.__new__7   s*   ø€ ØŒ=Ð å!™GœGŸOšO¨CÑ0Ô0ˆCŒMØŒ}Ðó    )Ú	recursivec                 ór  — | j         r|| j        k    r¤|| _        t          d¦  «        r!t          j        t
          d¬¦  «        | _        ndt          j        t          ¦  «        j
        dk    r$|r"t          j        t
          dd¬¦  «        | _        nt          j        t
          ¦  «        | _        d| _         dS dS )	z>
        Initialize or update the singleton instance.
        ú2.5.1F)Údynamicz2.6.0zmax-autotune-no-cudagraphs)r    ÚmodeTN)Ú_is_flex_compiledÚtrainingr
   ÚtorchÚcompiler   Ú_compiled_flex_attentionr   Úparser	   Úbase_version)Úselfr#   s     r   Ú__init__zWrappedFlexAttention.__init__=   s¼   € ð
 Ô%ð 	*¨°T´]Ò)BÐ)BØ$ˆDŒMÝ% gÑ.Ô.ð NÝ05´½nÐV[Ð0\Ñ0\Ô0\�Ô-Ð-õ ”�~Ñ.Ô.Ô;¸wÒFÐFÈ8ÐFÝ05´Ý"¨EÐ8Tð1ñ 1ô 1�Ô-Ð-õ
 16´½nÑ0MÔ0M�Ô-à%)ˆDÔ"Ð"Ð"ð *CÐ)Br   c                 ó   — | j         S r   )r&   )r)   s    r   Ú__call__zWrappedFlexAttention.__call__S   s   € ØÔ,Ð,r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r"   r&   r   r$   ÚcompilerÚdisabler*   r,   Ú__classcell__)r   s   @r   r   r   .   s“   ø€ € € € € ðð ð €IØÐØ#Ððð ð ð ð ð „^×Ò eÐÑ,Ô,ð*ð *ñ -Ô,ð*ð*-ð -ð -ð -ð -ð -ð -r   r   FÚqueryÚkeyÚvalueÚreturnc                 óp   — t          ¦   «         s t          |¦  «        ¦   «         nt          } || ||fi |¤ŽS r   )r   r   r   )r4   r5   r6   r#   r   Úflex_attention_compileds         r   Úcompile_friendly_flex_attentionr:   W   s\   € õ G_ÑF`ÔF`ÐtÐ<Õ2°8Ñ<Ô<Ñ>Ô>Ð>ÕftÐØ"Ð"ØØØðð ð ð	ð ð r   TÚattention_mask_2dÚattention_chunk_sizeÚoffsetsÚ	is_causalr   c                 óv  ‡ ‡‡‡‡‡‡— ‰ j         \  }}|s|}|s|}|t          z  dz   t          z  }t          j        j                             ‰ dd||z
  f¬¦  «        Š ‰ j        }	‰                      ¦   «         Š|�@‰                     ¦   «                              d¦  «         	                    d¦  «        dz
  |z  Šˆ ˆfd„Šˆˆfd„}
ˆ ˆfd„}|s|Šn|€‰n|
Š|�>|d          
                    |	¦  «        Š|d          
                    |	¦  «        Šˆˆˆfd	„}n‰}t          ||d|||	t          d
¦  «         ¬¦  «        S )aG  
    IMPORTANT NOTICE: This function is deprecated in favor of using the mask primitives in `masking_utils.py`,
    and will be removed in a future version without warnings. New code should not use it. It is only kept here
    for BC for now, while models using it are being patched accordingly.

    Create a block (causal) document mask for a batch of sequences, both packed and unpacked.
    Create Block (causal) logic and passing it into :func:`torch.nn.attention.flex_attention.create_block_mask`.
    The resultant BlockMask is a compressed representation of the full (causal) block
    mask. BlockMask is essential for performant computation of flex attention.
    See: https://pytorch.org/blog/flexattention/

    Args:
        attention_mask_2d (torch.Tensor): Attention mask for packed and padded sequences
        of shape (batch_size, total_seq_len). e.g.

        For unpacked sequence:
        [[1, 1, 1, 1, 0, 0, 0],
         [1, 1, 1, 1, 1, 0, 0]]

        For packed sequence:
        [[1, 1, 1, 2, 2, 2, 0],
         [1, 1, 2, 2, 2, 3, 3]]

    Returns:
        BlockMask
    é   r   )r6   ÚpadNéÿÿÿÿc                 ól   •— ||k    }‰	| |f         ‰	| |f         k    }‰| |f         dk    }||z  |z  }|S )zü
        Defines the logic of a block causal mask by combining both a standard causal mask
        and a block diagonal document mask.
        See :func:`~torchtune.modules.attention_utils.create_block_causal_mask`
        for an illustration.
        r   © )
Ú	batch_idxÚhead_idxÚq_idxÚkv_idxÚcausal_maskÚdocument_maskÚpadding_maskÚ
final_maskr;   Údocument_idss
           €€r   Úcausal_mask_modz4make_flex_block_causal_mask.<locals>.causal_mask_mod£   sV   ø€ ð ˜v’oˆØ$ Y°Ð%5Ô6¸,ÀyÐRXÐGXÔ:YÒYˆØ(¨°EÐ)9Ô:¸QÒ>ˆØ  <Ñ/°-Ñ?ˆ
ØÐr   c                 óV   •— ‰| |f         ‰| |f         k    } ‰| |||¦  «        }||z  S )zU
        Combines the chunk mask with the causal mask for chunked attention.
        rD   )rE   rF   rG   rH   Ú
chunk_maskÚcausal_doc_maskrN   Ú
chunk_idxss         €€r   Úchunk_causal_mask_modz:make_flex_block_causal_mask.<locals>.chunk_causal_mask_mod°   sC   ø€ ð   	¨5Ð 0Ô1°ZÀ	È6Ð@QÔ5RÒRˆ
Ø)˜/¨)°X¸uÀfÑMÔMˆØ˜OÑ+Ð+r   c                 óZ   •— ‰| |f         ‰| |f         k    }‰| |f         dk    }||z  }|S )zp
        Utilizes default attention mask to enable encoder and encoder-decoder
        attention masks.
        r   rD   )	rE   rF   rG   rH   rJ   rK   rL   r;   rM   s	          €€r   Údefault_mask_modz5make_flex_block_causal_mask.<locals>.default_mask_mod¸   sH   ø€ ð
 % Y°Ð%5Ô6¸,ÀyÐRXÐGXÔ:YÒYˆà(¨°FÐ):Ô;¸aÒ?ˆØ! MÑ1ˆ
ØÐr   c                 ó4   •— |‰z   }|‰z   } ‰| |||¦  «        S r   rD   )	rE   rF   rG   rH   Úoffset_qÚ	offset_kvÚ	kv_offsetÚmask_mod_maybe_combinedÚq_offsets	         €€€r   Úmask_modz-make_flex_block_causal_mask.<locals>.mask_modÌ   s.   ø€ Ø˜xÑ'ˆHØ Ñ*ˆIØ*Ð*¨9°hÀÈ)ÑTÔTÐTr   r   )r\   ÚBÚHÚQ_LENÚKV_LENÚdeviceÚ_compile)ÚshapeÚflex_default_block_sizer$   ÚnnÚ
functionalrA   ra   ÚcloneÚfill_ÚcumsumÚtor   r
   )r;   r<   Úquery_lengthÚ
key_lengthr=   r>   Ú
batch_sizeÚtotal_seq_lenÚpad_lenra   rS   rU   r\   rN   rR   rM   rY   rZ   r[   s   `            @@@@@@r   Úmake_flex_block_causal_maskrp   m   sê  øøøøøøø€ ðD !2Ô 7Ñ€J�Øð #Ø"ˆ
Øð %Ø$ˆàÕ5Ñ5¸Ñ:Õ>UÑU€GÝœÔ+×/Ò/Ð0AÈÐQRÐT[Ð^hÑThÐPiÐ/ÑjÔjÐØÔ%€FØ$×*Ò*Ñ,Ô,€LàÐ'à"×(Ò(Ñ*Ô*×0Ò0°Ñ3Ô3×:Ò:¸2Ñ>Ô>ÀÑBÐH\Ñ]ˆ
ðð ð ð ð ð ð,ð ,ð ,ð ,ð ,ð ,ð	ð 	ð 	ð 	ð 	ð 	ð ð mØ"2ÐÐà5IÐ5Q / /ÐWlÐàÐØ˜1”:—=’= Ñ(Ô(ˆØ˜A”J—M’M &Ñ)Ô)ˆ	ð	Uð 	Uð 	Uð 	Uð 	Uð 	Uð 	Uð 	Uð
 +ˆåØØ
Ø
ØØØå+¨GÑ4Ô4Ð4ð	ñ 	ô 	ð 	r   Úhidden_statesÚn_repc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r@   N)rc   ÚexpandÚreshape)rq   rr   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r   Ú	repeat_kvrz   ß   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr   ÚmoduleÚattention_maskÚscalingÚsoftcapÚ	head_maskÚs_auxc	                 ó*  ‡‡‡‡— ‰�t                                d¦  «         |	                     dd¦  «        dk    rt          d¦  «        ‚d }
d Št	          |t
          ¦  «        r|}
n|Š‰�‰d d …d d …d d …d |j        d         …f         Šˆˆˆˆfd„}d}|j        d	         }||d	z
  z  dk    rTt          ||j        d	         |j        d	         z  ¦  «        }t          ||j        d	         |j        d	         z  ¦  «        }d
}|	                     d¦  «        }|j        j	        dk    }t          |||||
||||| j        ¬¦
  «
        }|r |\  }}|                     |j        ¦  «        }n|}d }|                     d	d¦  «                             ¦   «         }||fS )Nzm`flex_attention` does not support `head_mask`. Please set your attention to `eager` if you want this feature.Údropoutg        r   z›`flex_attention` does not support `dropout`. Please use it with inference only (`model.eval()`) or turn off the attention dropout in the respective config.éþÿÿÿc                 ó’  •— ‰�‰t          j        | ‰z  ¦  «        z  } ‰�| ‰|         d         |         |         z   } ‰	�| ‰	|         |         d         d         z   } ‰
�it          j        | dd¬¦  «        j        }t          j        ‰
|z
  ¦  «        }t          j        | |z
  ¦  «        }|                     dd¬¦  «        |z   }||z  } | S )Nr   rB   T)ÚdimÚkeepdim)r$   ÚtanhÚmaxÚvaluesÚexpÚsum)ÚscorerE   rF   rG   rH   Ú
logits_maxÚsinksÚunnormalized_scoresÚ
normalizerr   r€   Ú
score_maskr~   s            €€€€r   Ú	score_modz)flex_attention_forward.<locals>.score_mod  sÝ   ø€ ØÐØ�eœj¨°©Ñ9Ô9Ñ9ˆEØÐ!Ø˜J yÔ1°!Ô4°UÔ;¸FÔCÑCˆEØÐ Ø˜I iÔ0°Ô:¸1Ô=¸aÔ@Ñ@ˆEØÐÝœ 5¨b¸$Ð?Ñ?Ô?ÔFˆJÝ”I˜e jÑ0Ñ1Ô1ˆEÝ"'¤)¨E°JÑ,>Ñ"?Ô"?ÐØ,×0Ò0°RÀÐ0ÑFÔFÈÑNˆJØ'¨*Ñ4ˆEØˆr   Tr@   FÚkernel_optionsÚcpu)r’   Ú
block_maskÚ
enable_gqaÚscaler“   Ú
return_lser#   r   )ÚloggerÚwarning_onceÚgetÚ
ValueErrorÚ
isinstancer   rc   rz   ra   Útyper:   r#   rj   ÚdtypeÚ	transposeÚ
contiguous)r{   r4   r5   r6   r|   r}   r~   r   r€   r   r•   r’   r–   Únum_local_query_headsr“   r˜   Úflex_attention_outputÚattention_outputÚlser‘   s         ```          @r   Úflex_attention_forwardr¦   ë   s  øøøø€ ð ÐÝ×ÒØ{ñ	
ô 	
ð 	
ð ‡z‚z�)˜SÑ!Ô! AÒ%Ð%Ýðañ
ô 
ð 	
ð
 €JØ€JÝ�.¥)Ñ,Ô,ð $Ø#ˆ
ˆ
à#ˆ
àÐØ    1 1 1 a a a¨¨3¬9°R¬=¨Ð 8Ô9ˆ
ðð ð ð ð ð ð ð ð €JØ!œK¨œNÐð 	Ð!6¸Ñ!:Ñ;ÀÒAÐAÝ˜˜Uœ[¨œ^¨s¬y¸¬|Ñ;Ñ<Ô<ˆÝ˜% ¤¨Q¤°5´;¸q´>Ñ!AÑBÔBˆØˆ
à—Z’ZÐ 0Ñ1Ô1€Nà”Ô" eÒ+€Jå;ØØØØØØØØ%ð Ø”ðñ ô Ðð ð Ø 5ÑÐ˜#Ø�fŠf�U”[Ñ!Ô!ˆˆà0ÐØˆà'×1Ò1°!°QÑ7Ô7×BÒBÑDÔDÐØ˜SÐ Ð r   )F)NNNNT)NNNN)$r0   Útypingr   r   r$   Ú	packagingr   Úutilsr   r   Úutils.import_utilsr	   r
   r   Ú!torch.nn.attention.flex_attentionr   rd   r   r   r   Ú
get_loggerr-   r™   r   ÚTensorÚtupler:   ÚintÚOffsetÚboolrp   rz   re   ÚModuleÚfloatr¦   rD   r   r   ú<module>r´      s   ððð ð8 #Ð "Ð "Ð "Ð "Ð "Ð "Ð "à €€€Ø Ð Ð Ð Ð Ð à 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ aÐ að  ÐÑ!Ô!ð _ØgÐgÐgÐgÐgÐgØ^Ð^Ð^Ð^Ð^Ð^Ð^Ð^Ð^Ð^ð 
ˆÔ	˜HÑ	%Ô	%€ð&-ð &-ð &-ð &-ð &-ñ &-ô &-ð &-ðZ ð	ð ØŒ<ðà	Œðð Œ<ðð ˆ5Œ<˜˜uœ|¨U¬\Ð9Ô:Ð:Ô;ðð ð ð ð$ 
ˆuŒ|˜SÐ Ô	!€ð +/ØØØ/3Ø $ðoð oØ”|ðoà" 3œ-ðoð
 �e˜F F˜NÔ+Ô,ðoð ˜Œ~ðoð ðoð oð oð oðd	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð$  $Ø#Ø(,Ø$(ðT!ð T!ØŒHŒOðT!àŒ<ðT!ð 
ŒðT!ð Œ<ð	T!ð
 ˜%œ,¨Ð3Ô4ðT!ð �eŒ_ðT!ð �eŒ_ðT!ð ˜œÔ%ðT!ð �E”LÔ!ðT!ð ˆ5Œ<˜ %¤,Ô/Ð/Ô0ðT!ð T!ð T!ð T!ð T!ð T!r   