§
    Ó˜ºhR™  ã            ,       óŽ  — d dl mZmZmZ d dlZd dlmZ ddlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddgZ G d„ de¦  «        Zd	d
e› de› de› de› de
› de› d�z   e_        dee         dee         dee         dee         dee         dee         dee         dee         dededee ef         dee ef         dee ef         de de dededed ef&d!„Z!dee         dee         dee         dee         dee         dee         dee         dee         dededee ef         dee ef         dee ef         de de dededed ef&d"„Z"dee         dee         dee         dee         dee         dee         dee         dee         dedede de dee ef         de de dededed ed#df(d$„Z# ee!¬%¦  «        	 	 	 	 	 	 	 	 d*dee         dee         dee         dee         dee         dee         d'ee         deded(ee         dee         dee         ded edede de dee ef         de de def*d)„¦   «         Z$dS )+é    )ÚcastÚOptionalÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_device_dtype_check_for_fusedÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ
_fused_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_params_docÚ_stack_if_compilingÚ
_to_scalarÚ_use_grad_for_differentiableÚ_view_as_realÚ
DeviceDictÚDeviceDtypeDictÚ	OptimizerÚParamsTÚAdamÚadamc                   óì   ‡ — e Zd Z	 	 	 	 	 ddddddddœded	eeef         d
eeeef         eeef         f         dededede	e         dededede	e         defˆ fd„Z
ˆ fd„Zd„ Zedd„¦   «         Zˆ xZS )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   FN)ÚforeachÚmaximizeÚ
capturableÚdifferentiableÚfusedÚdecoupled_weight_decayÚparamsÚlrÚbetasÚepsÚweight_decayÚamsgradr"   r#   r$   r%   r&   r'   c                ó  •— t          |t          ¦  «        r:|r|	st          d¦  «        ‚|                     ¦   «         dk    rt          d¦  «        ‚d|k    st          d|› �¦  «        ‚d|k    st          d|› �¦  «        ‚d|d         cxk    rdk     sn t          d	|d         › �¦  «        ‚d|d         cxk    rdk     sn t          d
|d         › �¦  «        ‚d|k    st          d|› �¦  «        ‚t          |d         t          ¦  «        rt          |d         t          ¦  «        sEt          |d         t          ¦  «        rt          |d         t          ¦  «        st          d¦  «        ‚t          |d         t          ¦  «        r@|	s|rt          d¦  «        ‚|d                              ¦   «         dk    rt          d¦  «        ‚t          |d         t          ¦  «        r@|	s|rt          d¦  «        ‚|d                              ¦   «         dk    rt          d¦  «        ‚t          ||||||||	|
||¬¦  «        }t          ¦   «                              ||¦  «         |r)|
rt          d¦  «        ‚d| _	        |rt          d¦  «        ‚d S d S )NúElr as a Tensor is not supported for capturable=False and foreach=Truer   úTensor lr must be 1-elementç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: z0betas must be either both floats or both TensorszKbetas[0] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[0] must be 1-elementzKbetas[1] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[1] must be 1-element)r)   r*   r+   r,   r-   r#   r"   r$   r%   r&   r'   z)`fused` does not support `differentiable`Tz0`fused` and `foreach` cannot be `True` together.)
Ú
isinstancer   Ú
ValueErrorÚnumelÚfloatÚdictÚsuperÚ__init__ÚRuntimeErrorÚ_step_supports_amp_scaling)Úselfr(   r)   r*   r+   r,   r-   r"   r#   r$   r%   r&   r'   ÚdefaultsÚ	__class__s                 €úX/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/torch/optim/adam.pyr9   zAdam.__init__#   s  ø€ õ  �b�&Ñ!Ô!ð 	@Øð ˜zð Ý Ø[ñô ð ð �xŠx‰zŒz˜QŠˆÝ Ð!>Ñ?Ô?Ð?Ø�bŠyˆyÝÐ;°rÐ;Ð;Ñ<Ô<Ð<Ø�cŠzˆzÝÐ<°sÐ<Ð<Ñ=Ô=Ð=Ø�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�lÒ"Ð"ÝÐJ¸LÐJÐJÑKÔKÐKå˜˜aœ¥%Ñ(Ô(ð	QÝ-7¸¸a¼Å%Ñ-HÔ-Hð	Qå˜5 œ8¥VÑ,Ô,ð	Qå1;¸EÀ!¼HÅfÑ1MÔ1Mð	Qõ ÐOÑPÔPÐPÝ�e˜A”h¥Ñ'Ô'ð 	FØð  'ð Ý Øañô ð ð �QŒx�~Š~ÑÔ 1Ò$Ð$Ý Ð!DÑEÔEÐEÝ�e˜A”h¥Ñ'Ô'ð 	FØð  'ð Ý Øañô ð ð �QŒx�~Š~ÑÔ 1Ò$Ð$Ý Ð!DÑEÔEÐEåØØØØ%ØØØØ!Ø)ØØ#9ð
ñ 
ô 
ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*àð 		WØð PÝ"Ð#NÑOÔOÐOØ.2ˆDÔ+ð
 ð WÝ"Ð#UÑVÔVÐVð		Wð 		WðWð Wó    c                 ó$  •— t          ¦   «                              |¦  «         | j        D �]d}|                     dd¦  «         |                     dd¦  «         |                     dd ¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd ¦  «        }|d	         D ]¾}| j                             |g ¦  «        }t          |¦  «        d
k    rŽt          j        |d         ¦  «        stt          |d         ¦  «        }|d         s|d         r*t          j
        |t          |¬¦  «        |j        ¬¦  «        n!t          j
        |t          ¦   «         ¬¦  «        |d<   Œ¿�Œfd S )Nr-   Fr#   r"   r$   r%   r'   r&   r(   r   Ústep©Úis_fused©ÚdtypeÚdevice©rF   )r8   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorr6   Útensorr   rG   )r<   rL   Úgroupr&   ÚpÚp_stateÚstep_valr>   s          €r?   rI   zAdam.__setstate__r   s¦  ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	ñ 	ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×Ò˜Z¨Ñ/Ô/Ð/Ø×Ò˜Y¨Ñ-Ô-Ð-Ø×Ò˜\¨5Ñ1Ô1Ð1Ø×ÒÐ-¨uÑ5Ô5Ð5Ø×ÒÐ5°uÑ=Ô=Ð=Ø×$Ò$ W¨dÑ3Ô3ˆEØ˜8”_ð ð �Øœ*Ÿ.š.¨¨BÑ/Ô/�Ý�w‘<”< 1Ò$Ð$­U¬_¸WÀV¼_Ñ-MÔ-MÐ$Ý$ W¨V¤_Ñ5Ô5�Hð ! Ô.ðOð
 38¸´.ðO�œØ$Ý"3¸UÐ"CÑ"CÔ"CØ#$¤8ðñ ô ð õ #œ\¨(Õ:KÑ:MÔ:MÐNÑNÔNð ˜F‘Oøñ	ð	ð 	r@   c                 ó²  — d}|d         D �]J}	|	j         ��?|t          j        |	¦  «        z  }|                     |	¦  «         |	j         j        rt          d¦  «        ‚|                     |	j         ¦  «         | j        |	         }
t          |
¦  «        dk    rí|d         rt          |	¦  «         |d         s|d         r0t          j	        dt          |d         ¬¦  «        |	j        ¬	¦  «        n!t          j        d
t          ¦   «         ¬¦  «        |
d<   t          j        |	t          j        ¬¦  «        |
d<   t          j        |	t          j        ¬¦  «        |
d<   |d         r#t          j        |	t          j        ¬¦  «        |
d<   |                     |
d         ¦  «         |                     |
d         ¦  «         |d         r|                     |
d         ¦  «         |d         r|
d         j        rt          d¦  «        ‚|d         r1t          j        |d         ¦  «        r|d         st          d¦  «        ‚|                     |
d         ¦  «         �ŒL|S )NFr(   zJAdam does not support sparse gradients, please consider SparseAdam insteadr   r&   r$   © rC   rE   r1   rH   rB   )Úmemory_formatÚexp_avgÚ
exp_avg_sqr-   Úmax_exp_avg_sqr%   zB`requires_grad` is not supported for `step` in differentiable moder"   r)   r/   )ÚgradrO   Ú
is_complexÚappendÚ	is_sparser:   rL   rN   r
   Úzerosr   rG   rQ   Ú
zeros_likeÚpreserve_formatÚrequires_gradrP   )r<   rR   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsÚhas_complexrS   rL   s              r?   Ú_init_groupzAdam._init_groupŠ   s¥  € ð ˆØ�x”ð =	2ñ =	2ˆAØŒvÑ!Ø�uÔ/°Ñ2Ô2Ñ2�Ø ×'Ò'¨Ñ*Ô*Ð*Ø”6Ô#ð Ý&Ødñô ð ð —’˜QœVÑ$Ô$Ð$àœ
 1œ�å�u‘:”: ’?�?Ø˜W”~ð 9Ý5°aÑ8Ô8Ð8ð ! Ô.ðJð
 38¸´.ðJ�œØÝ"3¸UÀ7¼^Ð"LÑ"LÔ"LØ#$¤8ðñ ô ð õ #œ\¨#Õ5FÑ5HÔ5HÐIÑIÔIð ˜&‘Mõ (-Ô'7Ø­Ô)>ð(ñ (ô (�E˜)Ñ$õ +0Ô*:Ø­Ô)>ð+ñ +ô +�E˜,Ñ'ð ˜YÔ'ð å27Ô2BØ­UÔ-Bð3ñ 3ô 3˜Ð.Ñ/ð —’  iÔ 0Ñ1Ô1Ð1Ø×"Ò" 5¨Ô#6Ñ7Ô7Ð7à˜Ô#ð DØ#×*Ò*¨5Ð1AÔ+BÑCÔCÐCØÐ)Ô*ð ¨u°V¬}Ô/Jð Ý&Ø\ñô ð ð ˜)Ô$ðåœ¨¨d¬Ñ4Ô4ðð " ,Ô/ðõ
 'Ø_ñô ð ð ×"Ò" 5¨¤=Ñ1Ô1Ð1ùØÐr@   c                 ó  — |                       ¦   «          d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D ]®}g }g }g }g }g }g }	|d         \  }
}|                      |||||||	¦  «        }t          ||||||	f|d         ||
||d         |d         |d         |d         |d         |d	         |d
         |d         t          | dd¦  «        t          | dd¦  «        |d         dœŽ Œ¯|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr*   r-   r)   r,   r+   r#   r"   r$   r%   r&   Ú
grad_scaleÚ	found_infr'   )r-   rj   Úbeta1Úbeta2r)   r,   r+   r#   r"   r$   r%   r&   rm   rn   r'   )Ú _cuda_graph_capture_health_checkrO   Úenable_gradrJ   rk   r   Úgetattr)r<   ÚclosureÚlossrR   rd   re   rf   rg   rh   ri   ro   rp   rj   s                r?   rB   z	Adam.stepÕ   s®  € ð 	×-Ò-Ñ/Ô/Ð/àˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð )	ð )	ˆEØ-/ÐØ"$ˆEØ%'ˆHØ(*ˆKØ,.ˆOØ(*ˆKØ  œ>‰LˆE�5à×*Ò*ØØ ØØØØØñô ˆKõ Ø ØØØØØðð ˜iÔ(Ø'ØØØ˜”;Ø" >Ô2Ø˜%”LØ˜zÔ*Ø˜iÔ(Ø  Ô.Ø$Ð%5Ô6Ø˜G”nÝ" 4¨°tÑ<Ô<Ý! $¨°TÑ:Ô:Ø',Ð-EÔ'Fð+ð ð ð ð ð0 ˆs   ¬AÁAÁ
A)r   r    r!   r   F©N)Ú__name__Ú
__module__Ú__qualname__r   r   r6   r   ÚtupleÚboolr   r9   rI   rk   r   rB   Ú__classcell__)r>   s   @r?   r   r   "   sŒ  ø€ € € € € ð $(ØCOØØØðMWð #'ØØ Ø$Ø $Ø',ðMWð MWð MWàðMWð �%˜�-Ô ðMWð �U˜5 &˜=Ô)¨5°¸°Ô+?Ð?Ô@ð	MWð
 ðMWð ðMWð ðMWð ˜$”ðMWð ðMWð ðMWð ðMWð ˜Œ~ðMWð !%ðMWð MWð MWð MWð MWð MWð^ð ð ð ð ð0Ið Ið IðV "ð9ð 9ð 9ñ "Ô!ð9ð 9ð 9ð 9ð 9r@   af  Implements Adam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \beta_1, \beta_2
                \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)}          \\
            &\hspace{13mm}      \lambda \text{ (weight decay)},  \: \textit{amsgrad},
                \:\textit{maximize},  \: \epsilon \text{ (epsilon)}                              \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0\leftarrow 0 \text{ (second moment)},\: v_0^{max}\leftarrow 0          \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\

            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm}\textbf{if} \: \lambda \neq 0                                           \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
            &\hspace{10mm} v_t^{max} \leftarrow \mathrm{max}(v_{t-1}^{max},v_t)                  \\
            &\hspace{10mm}\widehat{v_t} \leftarrow v_t^{max}/\big(1-\beta_2^t \big)              \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                  \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_.
    z
    Args:
        að  
        lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR
            is not yet supported for all our implementations. Please use a float
            LR if you are not also specifying fused=True or capturable=True.
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        decoupled_weight_decay (bool, optional): if True, this optimizer is
            equivalent to AdamW and the algorithm will not accumulate weight
            decay in the momentum nor variance. (default: False)
        amsgrad (bool, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        z	
        a=  
    .. Note::
        A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`.
    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ

    r(   re   rf   rg   rh   ri   rm   rn   r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   r'   c                óX  — |€|�J ‚t           j                             ¦   «         rFt          |t          ¦  «        sJ ‚t          |
t          ¦  «        sJ ‚t          |t          ¦  «        sJ ‚nt          |¦  «        }t          |
t          ¦  «        r|
j        |
j        f|
i}nd }t          | ¦  «        D �]w\  }}|s||         n||          }||         }||         }||         }t           j
                             ¦   «         sF|rDt          ¦   «         }|j        j        |j        j        k    r|j        j        |v sJ d|› d�¦   «         ‚|dz  }|dk    r”|r|                     d||z  z
  ¦  «         nv|r]t          |t          ¦  «        rH|j        r)|                     |                     ¦   «         |¦  «        }n/|                     ||¬¦  «        }n|                     ||¬¦  «        }t          j        |¦  «        rot          j        |¦  «        }t          j        |¦  «        }t          j        |¦  «        }|rt          j        ||         ¦  «        ||<   t          j        |¦  «        }|j        }|�3|j        }||f}||vr|
                     ||d¬¦  «        ||<   ||         }n|
}|                     |d|z
  ¦  «         |rxt          |t          ¦  «        rc|j        r-|                     t          j        |¦  «        d|z
  ¬¦  «         n]|                     |¦  «                             ||d|z
  ¬	¦  «         n.|                     |¦  «                             ||d|z
  ¬	¦  «         |s|�rÁ|}|r@t          |
t          ¦  «        r+|
j        rd|
|                     ¦   «         z  z
  } nd|
|z  z
  } nd|
|z  z
  } |r@t          |t          ¦  «        r+|j        rd||                     ¦   «         z  z
  }!nd||z  z
  }!nd||z  z
  }!|| z  }"|"                     ¦   «         }#|!                     ¦   «         }$|rŠ|r||                              ¦   «         }%n||         }%||                              t          j        |%|¦  «        ¦  «         ||                              ¦   «         |$|#z  z                       ||#z  ¦  «        }&n0|                     ¦   «         |$|#z  z                       ||#z  ¦  «        }&|r)|                     |                     ¦   «         |&¦  «         nÙ|                     ||&¦  «         nÂt=          |¦  «        }d|
|z  z
  } d||z  z
  }!|| z  }"|!d
z  }$|rTt          j        ||         |||         ¬¦  «         ||                              ¦   «         |$z                       |¦  «        }&n*|                     ¦   «         |$z                       |¦  «        }&|                     ||&|" ¬	¦  «         |r7t          j        | |         ¦  «        rt          j        ||         ¦  «        ||<   �Œyd S )NúIIf capturable=True, params and state_steps must be on supported devices: ú.r   r   ©ÚalphaT)rG   rF   Únon_blocking)Úweight)Úvalueç      à?)Úout) rO   ÚjitÚis_scriptingr3   r6   r   r   rG   rF   Ú	enumerateÚcompilerÚis_compilingr   ÚtypeÚmul_rc   Úaddcmul_ÚcloneÚaddr]   Úview_as_realÚtoÚlerp_ÚsquareÚnegÚsqrtÚcopy_ÚmaximumÚadd_Úaddcdiv_r   Úview_as_complex)'r(   re   rf   rg   rh   ri   rm   rn   r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   r'   Ú
beta1_dictÚiÚparamr\   rY   rZ   Ústep_tÚcapturable_supported_devicesrG   rF   ÚkeyÚdevice_beta1rB   Úbias_correction1Úbias_correction2Ú	step_sizeÚstep_size_negÚbias_correction2_sqrtr[   Údenoms'                                          r?   Ú_single_tensor_adamr©   Y  sÛ  € ð, Ð )Ð"3Ð"3Ð"3å„y×ÒÑÔð õ ˜"�eÑ$Ô$Ð$Ð$Ð$Ý˜%¥Ñ'Ô'Ð'Ð'Ð'Ý˜%¥Ñ'Ô'Ð'Ð'Ð'Ð'å˜‰^Œ^ˆõ �%�Ñ Ô ð Ø27´,ÀÄÐ1LÈeÐ0Tˆ
ˆ
àˆ
å˜fÑ%Ô%ð VKñ VK‰ˆˆ5Ø'Ð6ˆu�QŒxˆx¨e°A¬h¨YˆØ˜1”+ˆØ  ”^ˆ
Ø˜Q”ˆõ Œ~×*Ò*Ñ,Ô,ð 	°ð 	Ý+LÑ+NÔ+NÐ(à”Ô! V¤]Ô%7Ò7Ð7Ø”LÔ%Ð)EÐEÐEÐEà{Ð\xÐ{Ð{Ð{ñ FÔEÐEð 	�!‰ˆà˜1ÒÐØ%ð ?à—
’
˜1˜r LÑ0Ñ0Ñ1Ô1Ð1Ð1ð "ð ?¥j°½vÑ&FÔ&Fð ?Ø#Ô1ð CØ#Ÿ}š}¨U¯[ª[©]¬]¸LÑIÔI˜˜à#Ÿxšx¨°\˜xÑBÔB˜˜àŸ8š8 E°˜8Ñ>Ô>�DåÔ˜EÑ"Ô"ð 	.ÝÔ% dÑ+Ô+ˆDÝÔ(¨Ñ1Ô1ˆGÝÔ+¨JÑ7Ô7ˆJØð LÝ%*Ô%7¸ÈÔ8JÑ%KÔ%K� Ñ"ÝÔ& uÑ-Ô-ˆEà”ˆàÐ!Ø”KˆEð ˜5�/ˆCØ˜*Ð$Ð$Ø"'§(¢(Ø!¨¸Tð #+ñ #ô #�
˜3‘ð 2<¸C´ˆLˆLà ˆLð 	�Š�d˜A Ñ,Ñ-Ô-Ð-ð ð 	I�j¨µÑ7Ô7ð 	IØÔ"ð Mð × Ò ¥¤¨dÑ!3Ô!3¸AÀ¹IÐ ÑFÔFÐFÐFà—’ Ñ&Ô&×/Ò/°°dÀ!ÀeÁ)Ð/ÑLÔLÐLÐLà�OŠO˜EÑ"Ô"×+Ò+¨D°$¸aÀ%¹iÐ+ÑHÔHÐHàð E	=˜ñ E	=ØˆDð ð 3¥*¨UµFÑ";Ô";ð 3ØÔ&ð 7Ø'(¨5°D·J²J±L´LÑ+@Ñ'@Ð$Ð$à'(¨5°$©;¡Ð$Ð$à#$ u¨d¡{¡?Ð ð ð 3¥*¨UµFÑ";Ô";ð 3ØÔ&ð 7Ø'(¨5°D·J²J±L´LÑ+@Ñ'@Ð$Ð$à'(¨5°$©;¡Ð$Ð$à#$ u¨d¡{¡?Ð àÐ-Ñ-ˆIØ%ŸMšM™OœOˆMà$4×$9Ò$9Ñ$;Ô$;Ð!àð ,à!ð 8Ø%4°QÔ%7×%=Ò%=Ñ%?Ô%?�N�Nà%4°QÔ%7�Nà Ô"×(Ò(­¬°~ÀzÑ)RÔ)RÑSÔSÐSð $ AÔ&×+Ò+Ñ-Ô-Ð1FÈÑ1VÑWß’$�s˜]Ñ*Ñ+Ô+ð �ð
 —O’OÑ%Ô%Ð)>ÀÑ)NÑOß’$�s˜]Ñ*Ñ+Ô+ð ð ð /Ø—’˜wŸ}š}™œ°Ñ6Ô6Ð6Ð6à—’˜w¨Ñ.Ô.Ð.Ð.å˜fÑ%Ô%ˆDà  5¨$¡;™ÐØ  5¨$¡;™ÐàÐ-Ñ-ˆIà$4°cÑ$9Ð!àð Nå”˜o¨aÔ0°*À/ÐRSÔBTÐUÑUÔUÐUð )¨Ô+×0Ò0Ñ2Ô2Ð5JÑJ×PÒPÐQTÑUÔU��à#ŸšÑ*Ô*Ð-BÑB×HÒHÈÑMÔM�à�NŠN˜7 E°)°ˆNÑ<Ô<Ð<ð ð 	K•uÔ'¨¨q¬	Ñ2Ô2ð 	KÝ!&Ô!6°ÀqÔ7IÑ!JÔ!JˆO˜AÑùðmVKð VKr@   c                óˆ  ‡
‡‡‡+— t          | ¦  «        dk    rd S t          ‰t          ¦  «        r8|st          d¦  «        ‚‰                     ¦   «         dk    rt          d¦  «        ‚t          ‰
t          ¦  «        r8|st          d¦  «        ‚‰
                     ¦   «         dk    rt          d¦  «        ‚t          ‰t          ¦  «        r8|st          d¦  «        ‚‰                     ¦   «         dk    rt          d¦  «        ‚t          j                             ¦   «         sI|rGt          d	¬
¦  «        Š+t          ˆ+fd„t          | |¦  «        D ¦   «         ¦  «        sJ d‰+› d�¦   «         ‚|€|�J ‚|r
J d¦   «         ‚t          ‰¦  «        Št          j        | |||||g¦  «        }t          ‰
t          ¦  «        r!t          ‰
j        ¦  «        dk    r	‰
j        ‰
ind }|                     ¦   «         D �]Ñ\  \  }}}}}}}t%          t&          t                   |¦  «        }t%          t&          t                   |¦  «        }t%          t&          t                   |¦  «        }t%          t&          t                   |¦  «        }t%          t&          t                   |¦  «        } |d         j        }!|�|!|vr‰
                     |!d¬¦  «        ||!<   |r||!         n‰
}"|	rH|r4t%          t&          t                   |¦  «        }#t+          |||||#¦  «         nt+          ||||¦  «         |rt          j        |¦  «        }t          j                             ¦   «         s9| d         j        r,t          j        | t          j        dd¬¦  «        d¬¦  «         nt          j        | d¦  «         |dk    rO|rt          j        |d‰|z  z
  ¦  «         n1|rt          j        |||¬¦  «         nt          j        |||¬¦  «        }t          j        ||d|"z
  ¦  «         t          j        |‰¦  «         t          ‰t          j        ¦  «        rt          j        |d‰z
  ¦  «        }$d}%n|}$d‰z
  }%t          j        ||$||%¦  «         ~~$|�r`t          j        ‰
| ¦  «        }&t          j        ‰| ¦  «        }'t          j         |&d¦  «         t          j         |'d¦  «         t          j!        |'¦  «         t          j"        |&‰¦  «         t          j#        |&¦  «         t          j$        |'¦  «         |&}(|'})|rJt%          t&          t                   |¦  «        }#t          j%        |#|¦  «         t          j&        |#¦  «        }*nt          j&        |¦  «        }*t          j"        |*|)¦  «         t          j        |*|¦  «         t          j"        |*|(¦  «         t          j'        |||*¦  «         �Œíˆ
fd„| D ¦   «         }&ˆfd„| D ¦   «         }'tQ          ˆfd„|&D ¦   «         ¦  «        }(d„ |'D ¦   «         })|rJt%          t&          t                   |¦  «        }#t          j%        |#|¦  «         t          j&        |#¦  «        }*nt          j&        |¦  «        }*t          j"        |*|)¦  «         t          j        |*|¦  «         t          j'        |||*|(¦  «         �ŒÓd S )Nr   r/   r   r0   zHbeta1 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta1 must be 1-elementzHbeta2 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta2 must be 1-elementF)Úsupports_xlac              3   ón   •K  — | ]/\  }}|j         j        |j         j        k    o|j         j        ‰v V — Œ0d S rv   )rG   rŒ   )Ú.0rS   rB   r    s      €r?   ú	<genexpr>z%_multi_tensor_adam.<locals>.<genexpr>T  s]   øè è € ð 
ð 
ñ ��4ð ŒHŒM˜Tœ[Ô-Ò-ð >Ø””Ð!=Ð=ð
ð 
ð 
ð 
ð 
ð 
r@   r~   r   z#_foreach ops don't support autogradÚcpuT©rG   r‚   r2   )rG   r€   c                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS ©r   ©r   )r­   rB   ro   s     €r?   ú
<listcomp>z&_multi_tensor_adam.<locals>.<listcomp>ò  ó8   ø€ ð  ð  ð  Ø26��E�Z¨Ñ-Ô-Ñ-Ñ-ð ð  ð  r@   c                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS r²   r³   )r­   rB   rp   s     €r?   r´   z&_multi_tensor_adam.<locals>.<listcomp>õ  rµ   r@   c                 ó    •— g | ]
}‰|z  d z  ‘ŒS )éÿÿÿÿrW   )r­   Úbcr)   s     €r?   r´   z&_multi_tensor_adam.<locals>.<listcomp>ù  s!   ø€ Ð,WÐ,WÐ,WÀ¨b°2©g¸©^Ð,WÐ,WÐ,Wr@   c                 ó   — g | ]}|d z  ‘ŒS )r…   rW   )r­   r¹   s     r?   r´   z&_multi_tensor_adam.<locals>.<listcomp>û  s   € Ð$HÐ$HÐ$H° R¨¡WÐ$HÐ$HÐ$Hr@   ))rN   r3   r   r:   r5   r4   rO   rŠ   r‹   r   ÚallÚzipr   r   Ú"_group_tensors_by_device_and_dtypeÚstrrG   Úvaluesr   Úlistr’   r   Ú_foreach_negÚis_cpuÚ_foreach_add_rQ   Ú_foreach_mul_Ú_foreach_addÚ_foreach_lerp_Ú_foreach_mulÚ_foreach_addcmul_Ú_foreach_powÚ_foreach_sub_Ú_foreach_neg_Ú_foreach_div_Ú_foreach_reciprocal_Ú_foreach_sqrt_Ú_foreach_maximum_Ú_foreach_sqrtÚ_foreach_addcdiv_r   ),r(   re   rf   rg   rh   ri   rm   rn   r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   r'   Úgrouped_tensorsrœ   Údevice_params_Údevice_grads_Údevice_exp_avgs_Údevice_exp_avg_sqs_Údevice_max_exp_avg_sqs_Údevice_state_steps_Ú_Údevice_paramsÚdevice_gradsÚdevice_exp_avgsÚdevice_exp_avg_sqsÚdevice_state_stepsrG   r¢   Údevice_max_exp_avg_sqsÚscaled_device_gradsr„   r£   r¤   r¥   r§   Úexp_avg_sq_sqrtr    s,             ```                              @r?   Ú_multi_tensor_adamrâ     sâ  øøøø€ õ, ˆ6�{„{�aÒÐØˆå�"•fÑÔð <Øð 	ÝØWñô ð ð �8Š8‰:Œ:˜Š?ˆ?ÝÐ:Ñ;Ô;Ð;å�%�Ñ Ô ð ?Øð 	ÝØZñô ð ð �;Š;‰=Œ=˜AÒÐÝÐ=Ñ>Ô>Ð>å�%�Ñ Ô ð ?Øð 	ÝØZñô ð ð �;Š;‰=Œ=˜AÒÐÝÐ=Ñ>Ô>Ð>õ Œ>×&Ò&Ñ(Ô(ð 

¨Zð 

Ý'HØð(
ñ (
ô (
Ð$õ ð 
ð 
ð 
ð 
õ ˜v {Ñ3Ô3ð
ñ 
ô 
ñ 
ô 
ð 	
ð 	
ð
 xÐXtÐwÐwÐwñ	
ô 	
ð 	
ð Ð )Ð"3Ð"3Ð"3àÐDÐDÐDÑDÔDÐDå	�B‰Œ€Bõ  ÔBØ	�˜ +¨ÀÐLñô €Oõ �e�VÑ$Ô$ð	Ý),¨U¬\Ñ):Ô):¸eÒ)CÐ)Cð 
Œ�uÐÐàð ð ×"Ò"Ñ$Ô$ð_ñ _ñ 		ñ 	ØØØØØØØÝ�T¥&œ\¨>Ñ:Ô:ˆÝ�D¥œL¨-Ñ8Ô8ˆÝ�t¥Fœ|Ð-=Ñ>Ô>ˆÝ!¥$¥v¤,Ð0CÑDÔDÐÝ!¥$¥v¤,Ð0CÑDÔDÐà˜qÔ!Ô(ˆØÐ! f°JÐ&>Ð&>Ø!&§¢°Àd Ñ!KÔ!KˆJ�vÑà-7ÐB�z &Ô)Ð)¸Uˆð ð 	Øð Ý)-­dµ6¬lÐ<SÑ)TÔ)TÐ&ÝØ!Ø Ø#Ø&Ø*ñô ð ð õ Ø! <°ÐBTñô ð ð ð 	<Ý Ô-¨lÑ;Ô;ˆLõ Œ~×*Ò*Ñ,Ô,ð 	7Ð1CÀAÔ1FÔ1Mð 	7ÝÔØ"¥E¤L°¸UÐ$CÑ$CÔ$CÈ3ðñ ô ð ð õ ÔÐ 2°AÑ6Ô6Ð6à˜1ÒÐØ%ð 
åÔ# M°1°r¸LÑ7HÑ3HÑIÔIÐIÐIð ð ÝÔ'¨°mÈ<ÐXÑXÔXÐXÐXå#(Ô#5Ø$ m¸<ð$ñ $ô $�Lõ 	Ô˜_¨l¸AÀÑ<LÑMÔMÐMåÔÐ.°Ñ6Ô6Ð6õ �e�Uœ\Ñ*Ô*ð 	Ý"'Ô"4°\À1ÀuÁ9Ñ"MÔ"MÐØˆEˆEà".ÐØ˜‘IˆEåÔØÐ 3°\À5ñ	
ô 	
ð 	
ð
 Øð ñ B	Ý$Ô1°%Ð9KÑLÔLÐÝ$Ô1°%Ð9KÑLÔLÐåÔÐ 0°!Ñ4Ô4Ð4ÝÔÐ 0°!Ñ4Ô4Ð4åÔÐ 0Ñ1Ô1Ð1õ ÔÐ 0°"Ñ5Ô5Ð5ÝÔ&Ð'7Ñ8Ô8Ð8åÔ Ð!1Ñ2Ô2Ð2ð
 )ˆIØ$4Ð!àð JÝ)-­dµ6¬lÐ<SÑ)TÔ)TÐ&åÔ'Ð(>Ð@RÑSÔSÐSõ #(Ô"5Ð6LÑ"MÔ"M��å"'Ô"5Ð6HÑ"IÔ"I�åÔ Ð1FÑGÔGÐGÝÔ °Ñ5Ô5Ð5ÝÔ °Ñ;Ô;Ð;õ Ô# M°?ÀOÑTÔTÐTÑTð ð  ð  ð  Ø:Lð ñ  ô  Ðð ð  ð  ð  Ø:Lð ñ  ô  Ðõ ,Ð,WÐ,WÐ,WÐ,WÐFVÐ,WÑ,WÔ,WÑXÔXˆIà$HÐ$HÐ7GÐ$HÑ$HÔ$HÐ!àð JÝ)-­dµ6¬lÐ<SÑ)TÔ)TÐ&åÔ'Ð(>Ð@RÑSÔSÐSõ #(Ô"5Ð6LÑ"MÔ"M��å"'Ô"5Ð6HÑ"IÔ"I�åÔ Ð1FÑGÔGÐGÝÔ °Ñ5Ô5Ð5ÝÔ#ØØØØñ	ô ð ñ ðu_ð _r@   Úreturnc                óŠ  — | sd S |rt          d¦  «        ‚|�	|j        |ini }|�	|j        |ini }t          |t          ¦  «        r!t	          |j        ¦  «        dk    r	|j        |ind }t          j        | |||||g¦  «        }|                     ¦   «         D �]«\  \  }}\  \  }}}}}}}t          t          t                   |¦  «        }t          t          t                   |¦  «        } t          t          t                   |¦  «        }!t          t          t                   |¦  «        }"t          t          t                   |¦  «        }#d\  }$}%|�+| 
                    ||                     |d¬¦  «        ¦  «        }$|�+| 
                    ||                     |d¬¦  «        ¦  «        }%|�&||vr"|                     |d¬¦  «        ||<   ||         }t          j        |#d¦  «         |st          j        nt          j        }& |&|| |!|"||#|||
|||||$|%¬¦  «         |%�&t          j        |#|%gt#          |#¦  «        z  ¦  «         �Œ­d S )	Nz9Adam with fused=True does not support differentiable=Truer¯   )NNT)r‚   r°   r   )	r-   r)   ro   rp   r,   r+   r#   rm   rn   )r:   rG   r3   r   r¾   r   r½   Úitemsr   rÀ   rK   r’   rO   rÃ   Ú_fused_adam_Ú_fused_adamw_rÊ   rN   )'r(   re   rf   rg   rh   ri   rm   rn   r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   r'   Úgrad_scale_dictÚfound_inf_dictÚlr_dictrÒ   rG   rÙ   rÓ   rÔ   rÕ   rÖ   rß   rØ   rÚ   rÛ   rÜ   rÝ   rÞ   Údevice_grad_scaleÚdevice_found_infÚfuncs'                                          r?   Ú_fused_adamrî     s·  € ð, ð ØˆØð XÝÐVÑWÔWÐWð ,6Ð+AˆÔ	˜JÐ'Ð'Àrð ð *3Ð)>ˆÔ	˜9Ð%Ð%ÀBð õ & b­&Ñ1Ô1ÐWµc¸"¼)±n´nÈÒ6MÐ6MˆŒ�BˆˆÐSWð õ  ÔBØ	�˜ +¨ÀÐLñô €Oð 
×	Ò	Ñ	 Ô	 ð3ñ 3ñ 
	‰ˆ�ñ 
ñ	
ØØØØØ"Øà	å�T¥&œ\¨>Ñ:Ô:ˆÝ�D¥œL¨-Ñ8Ô8ˆÝ�t¥Fœ|Ð-=Ñ>Ô>ˆÝ!¥$¥v¤,Ð0CÑDÔDÐÝ!¥$¥v¤,Ð0CÑDÔDÐà.8Ñ+ÐÐ+ØÐ!Ø /× :Ò :Ø˜
Ÿš f¸4˜Ñ@Ô@ñ!ô !Ðð Ð Ø-×8Ò8Ø˜	Ÿš V¸$˜Ñ?Ô?ñ ô  Ðð Ð 6°Ð#8Ð#8Ø Ÿeše¨6À˜eÑEÔEˆG�F‰OØ˜”ˆBÝÔÐ.°Ñ2Ô2Ð2Ø)?ÐX�uÔ!Ð!ÅUÔEXˆØˆØØØØØ"ØØØØØØ%ØØØ(Ø&ð	
ñ 	
ô 	
ð 	
ð" Ð'ÝÔØ"Ð%5Ð$6½Ð=OÑ9PÔ9PÑ$Pñô ð ùðc3ð 3r@   )Úsingle_tensor_fnFr"   r&   c                ó¨  — |	€2|€0t          | |d¬¦  «        \  }}|rt          |t          ¦  «        r|sd}|	€d}	|€d}t          j                             ¦   «         s(t          d„ |D ¦   «         ¦  «        st          d¦  «        ‚|r-t          j         	                    ¦   «         rt          d¦  «        ‚|	r-t          j         	                    ¦   «         rt          d¦  «        ‚|	r&t          j         	                    ¦   «         st          }n/|r&t          j         	                    ¦   «         st          }nt          } || |||||f|||||||||||
||dœŽ dS )	znFunctional API that performs Adam algorithm computation.

    See :class:`~torch.optim.Adam` for details.
    NF)Ú	use_fusedc              3   óJ   K  — | ]}t          |t          j        ¦  «        V — Œd S rv   )r3   rO   r   )r­   Úts     r?   r®   zadam.<locals>.<genexpr>¢  s?   è è € ð 5ð 5Ø()�
�1•e”lÑ#Ô#ð5ð 5ð 5ð 5ð 5ð 5r@   zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers)r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   rm   rn   r'   )r	   r3   r   rO   rŠ   r‹   r»   r:   r‡   rˆ   rî   râ   r©   )r(   re   rf   rg   rh   ri   r"   r$   r%   r&   rm   rn   rj   r'   r-   ro   rp   r)   r,   r+   r#   rÙ   rí   s                          r?   r   r   q  sÒ  € ðF €}˜˜Ý1Ø�N¨eð
ñ 
ô 
‰
ˆˆ7ð ð 	•z "¥fÑ-Ô-ð 	°jð 	ØˆGØ€}ØˆØ€Øˆõ Œ>×&Ò&Ñ(Ô(ð 
µð 5ð 5Ø-8ð5ñ 5ô 5ñ 2ô 2ð 
õ Ø^ñ
ô 
ð 	
ð ð U•5”9×)Ò)Ñ+Ô+ð UÝÐSÑTÔTÐTØð S•”×'Ò'Ñ)Ô)ð SÝÐQÑRÔRÐRàð #•U”Y×+Ò+Ñ-Ô-ð #ÝˆˆØ	ð #�œ×/Ò/Ñ1Ô1ð #Ý!ˆˆå"ˆà€DØØØØØØðð ØØØØØ!ØØØØ%ØØØ5ð'ð ð ð ð ð r@   )NFFNNNFF)%Útypingr   r   r   rO   r   Ú	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__rÀ   r{   r6   r©   râ   rî   r   rW   r@   r?   ú<module>rø      sÀ  ðà (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (à €€€Ø Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð0 �6Ð
€ðmð mð mð mð mˆ9ñ mô mð mðb$ðJà	ðð ð" 
ð#ð ð$ 
ð%ð ð& 
ð'ð ð( 
ð)ð ð* 
ð+ð ð ñKBð „ðNBKØ�ŒLðBKà�Œ<ðBKð �6ŒlðBKð �f”ð	BKð
 ˜&”\ðBKð �f”ðBKð ˜Ô ðBKð ˜ÔðBKð ðBKð ðBKð �˜�ÔðBKð �˜�ÔðBKð 	ˆe�VˆmÔðBKð ðBKð  
ð!BKð" ð#BKð$ ð%BKð& ð'BKð( !ð)BKð BKð BKð BKðJpØ�ŒLðpà�Œ<ðpð �6Œlðpð �f”ð	pð
 ˜&”\ðpð �f”ðpð ˜Ô ðpð ˜Ôðpð ðpð ðpð �˜�Ôðpð �˜�Ôðpð 	ˆe�VˆmÔðpð ðpð  
ð!pð" ð#pð$ ð%pð& ð'pð( !ð)pð pð pð pðf]Ø�ŒLð]à�Œ<ð]ð �6Œlð]ð �f”ð	]ð
 ˜&”\ð]ð �f”ð]ð ˜Ô ð]ð ˜Ôð]ð ð]ð ð]ð ð]ð ð]ð 	ˆe�VˆmÔð]ð ð]ð  
ð!]ð" ð#]ð$ ð%]ð& ð']ð( !ð)]ð* 
ð+]ð ]ð ]ð ]ð@  ÐÐ1DÐEÑEÔEð #ØØ Ø Ø#'Ø"&ØØ#(ð!Wð WØ�ŒLðWà�Œ<ðWð �6ŒlðWð �f”ð	Wð
 ˜&”\ðWð �f”ðWð �dŒ^ðWð ðWð ðWð �DŒ>ðWð ˜Ô ðWð ˜ÔðWð ðWð  !ð!Wð$ ð%Wð& ð'Wð( ð)Wð* 	ˆe�VˆmÔð+Wð, ð-Wð. 
ð/Wð0 ð1Wð Wð Wñ FÔEðWð Wð Wr@   