ݘh-ddlZddlmZddlZddlmZmZddlmZddlm Z ej e Z Gddej ZGd d ej ZGd d ej ZGd dej ZGddej ZGddej ZGddej ZGddej ZGddej ZGddej ZGddej ZGddeZGdd ej Zid!ed"ed#d$d%fd&ed'ed(ed)d*ifd+ed,ed-ed.ejd/ed0ed1ed2ejd3ed4ejd5ejd6ejejej ej!ed7Z"ee"Z#d8Z$e$d(Z%e$d'Z&e$d!Z'e$d&Z(e$d1Z)e$d6Z*e$d0Z+e$d/Z,dS)9N) OrderedDict)Tensornn)logging)is_torchdynamo_compilingc"eZdZdZdedefdZdS)PytorchGELUTanha& A fast C implementation of the tanh approximation of the GeLU activation function. See https://huggingface.co/papers/1606.08415. This implementation is equivalent to NewGELU and FastGELU but much faster. However, it is not an exact numerical match due to rounding errors. inputreturncDtj|dS)Ntanh) approximate)r functionalgeluselfr s `/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/transformers/activations.pyforwardzPytorchGELUTanh.forward%s}!!%V!<<<N__name__ __module__ __qualname____doc__rrrrr r s@=V=======rr c"eZdZdZdedefdZdS)NewGELUActivationz Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Also see the Gaussian Error Linear Units paper: https://huggingface.co/papers/1606.08415 r r c d|zdtjtjdtjz |dtj|dzzzzzS)N??@Hm?g@)torchrmathsqrtpipowrs rrzNewGELUActivation.forward/sPU{cEJtytw/G/G5S[^c^ghmor^s^sSsKs/t$u$uuvvrNrrrrrr)sH wVwwwwwwwrrcJeZdZdZd deffd ZdedefdZdedefdZxZ S) GELUActivationa Original Implementation of the GELU activation function in Google BERT repo when initially created. For information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional Also see the Gaussian Error Linear Units paper: https://huggingface.co/papers/1606.08415 Fuse_gelu_pythonct|r|j|_dStjj|_dSN)super__init__ _gelu_pythonactrrr)rr+ __class__s rr/zGELUActivation.__init__;s?   *(DHHH})DHHHrr r cf|dzdtj|tjdz zzS)Nr r!r")r$erfr%r&rs rr0zGELUActivation._gelu_pythonBs-s{cEIedinn.D$E$EEFFrc,||Sr-r1rs rrzGELUActivation.forwardExxr)F) rrrrboolr/rr0r __classcell__r2s@rr*r*3s********G&GVGGGGVrr*c"eZdZdZdedefdZdS)FastGELUActivationz} Applies GELU approximation that is slower than QuickGELU but more accurate. See: https://github.com/hendrycks/GELUs r r cZd|zdtj|dzdd|z|zzzzzS)Nr r!g3E?r#)r$rrs rrzFastGELUActivation.forwardNs;U{cEJu|/CsXX]M]`eMeGe/f$g$gghhrNrrrrr<r<IsHiViiiiiiirr<c"eZdZdZdedefdZdS)QuickGELUActivationzr Applies GELU approximation that is fast but somewhat inaccurate. See: https://github.com/hendrycks/GELUs r r c6|tjd|zzS)NgZd;?)r$sigmoidrs rrzQuickGELUActivation.forwardWsu}UU]3333rNrrrrr?r?Rs@4V4444444rr?c<eZdZdZdedeffd ZdedefdZxZS)ClippedGELUActivationa Clip the range of possible GeLU outputs between [min, max]. This is especially useful for quantization purpose, as it allows mapping negatives values in the GeLU spectrum. For more information on this trick, please refer to https://huggingface.co/papers/2004.09602. Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))). See https://huggingface.co/papers/1606.08415 minmaxc||krtd|d|dt||_||_dS)Nzmin should be < max (got min: z, max: )) ValueErrorr.r/rDrE)rrDrEr2s rr/zClippedGELUActivation.__init__hsV 99PcPP#PPPQQ Q rxr c\tjt||j|jSr-)r$cliprrDrE)rrIs rrzClippedGELUActivation.forwardps z$q''48TX666r) rrrrfloatr/rrr9r:s@rrCrC[sw  E77F77777777rrCc2eZdZdZfdZdedefdZxZS)AccurateGELUActivationz Applies GELU approximation that is faster than default and more accurate than QuickGELU. See: https://github.com/hendrycks/GELUs Implemented along with MEGA (Moving Average Equipped Gated Attention) cttjdtjz |_dS)N)r.r/r%r&r'precomputed_constantrr2s rr/zAccurateGELUActivation.__init__|s7 $(Ia$'k$:$:!!!rr r c ~d|zdtj|j|dtj|dzzzzzS)Nr rr#)r$rrQr(rs rrzAccurateGELUActivation.forwardsCU{a%*T-F%RZ]b]fglno]p]pRpJp-q"r"rrssr)rrrrr/rrr9r:s@rrNrNtsn;;;;;tVtttttttttrrNcBeZdZdZfdZdedefdZdedefdZxZS)MishActivationz See Mish: A Self-Regularized Non-Monotonic Activation Function (Misra., https://huggingface.co/papers/1908.08681). Also visit the official repository for the paper: https://github.com/digantamisra98/Mish ctttjj|_dSr-)r.r/rrmishr1rRs rr/zMishActivation.__init__s) =%rr r cj|tjtj|zSr-)r$rrrsoftplusrs r _mish_pythonzMishActivation._mish_pythons'uz"-"8"8"?"?@@@@rc,||Sr-r6rs rrzMishActivation.forwardr7r) rrrrr/rr[rr9r:s@rrVrVs &&&&&A&AVAAAAVrrVc"eZdZdZdedefdZdS)LinearActivationz[ Applies the linear activation function, i.e. forwarding input directly to output. r r c|Sr-rrs rrzLinearActivation.forwards rNrrrrr^r^s@Vrr^ceZdZdZddZdS)LaplaceActivationz Applies elementwise activation based on Laplace function, introduced in MEGA as an attention activation. See https://huggingface.co/papers/2209.10655 Inspired by squared relu, but with bounded range and gradient for better stability 绹۞? ^/ ?c||z |tjdz}ddtj|zzS)Nr"r r!)divr%r&r$r4)rr musigmas rrzLaplaceActivation.forwards@  3!788cEIe,,,--rN)rbrcrrrrrrrrraras2......rraceZdZdZdZdS)ReLUSquaredActivationz` Applies the relu^2 activation introduced in https://huggingface.co/papers/2109.08668v2 cltj|}tj|}|Sr-)rrrelur$square)rr relu_appliedsquareds rrzReLUSquaredActivation.forwards+}))%00 ,|,,rNrhrrrrjrjs-rrjceZdZfdZxZS)ClassInstantierct|}t|tr|n|if\}}|di|S)Nr)r. __getitem__ isinstancetuple)rkeycontentclskwargsr2s rrszClassInstantier.__getitem__sL''%%c**!+GU!;!;Ngg'2 Vs}}V}}r)rrrrsr9r:s@rrqrqs8rrqcjeZdZdZddddejdffd Zdedefd Zdedefd Z d edefd Z xZ S) XIELUActivationz Applies the xIELU activation function introduced in https://arxiv.org/abs/2411.13010 If the user has installed the nickjbrowning/XIELU wheel, we import xIELU CUDA Otherwise, we emit a single warning and use xIELU Python g?r gưFc ttjt jt jt j||dz d|_ tjt jt jt j||z |dz d|_ | dt j||| dt j||||_ t|j |_t|j |_d|_ ddl}tjj|_d} ddlm} | |j|_|dz }n,#t<$r} |d | d z }|j|_Yd} ~ nd} ~ wwxYwt> |dS#t<$r3} t> d tC| Yd} ~ dSd} ~ wwxYw) N)dtyperrbetaepszUsing experimental xIELU CUDA.)allow_in_graphz& Enabled torch._dynamo for xIELU CUDA.z+ Could not enable torch._dynamo for xIELU (z*) - this may result in slower performance.uCUDA-fused xIELU not available (%s) – falling back to a Python version. For CUDA xIELU (experimental), `pip install git+https://github.com/nickjbrowning/XIELU`)"r.r/r Parameterr$logexptensor unsqueezealpha_palpha_nregister_bufferwith_vector_loadsrLr~detachcpuitem _beta_scalarr _eps_scalar_xielu_cuda_obj xielu.opsclassesxieluXIELU torch._dynamor _xielu_cuda_xielu_cuda_fn Exceptionlogger warning_oncestr) r alpha_p_init alpha_n_initr~rr}rrmsgrerrr2s rr/zXIELUActivation.__init__s |EIei \Y^8_8_8_.`.`cd.d$e$e$o$opq$r$rss | Iei \D-@ N N NOORSS T T ^ ^_` a a    VU\$e%D%D%DEEE UELE$B$B$BCCC!2!$)"2"2"4"4"8"8":":"@"@"B"B"G"G"I"IJJ !2!2!6!6!8!8!>!>!@!@!E!E!G!GHH#     #(=#6#<#<#>#>D 2C 7888888&4nT5E&F&F#?? 7 7 7tStttt&*&6###### 7    $ $ $ $ $      jC          sB.J0 I('J0( J2J J0 JJ00 K-:(K((K-rIr c ntj|j}|jtj|jz}t j|dk||z|z|j|zzt jt j ||j |z |z|j|zzS)Nr) rrrZrr~rr$whereexpm1rDr)rrIrrs r _xielu_pythonzXIELUActivation._xielu_pythons-((66)bm44T\BBB{ E aK!Odi!m + [1dh// 0 01 4 ?$)a- O   rc|j}|dkr-|d}|dk-|dkr*|dd|d}||jkr!t d||j|j||j |j |j |j |j }||S)zDFirewall function to prevent torch.compile from seeing .item() callsrTrrz_Warning: xIELU input tensor expects 3 dimensions but got (shape: %s). Reshaping to (shape: %s).)shapedimrviewsizerrrrrrrrr)rrIoriginal_shaperesults rrzXIELUActivation._xielu_cudaseeggkk AAeeggkk 5577Q;;r1affRjj))A QW $ $   q    %-- L L      "  {{>***rr c|jD|jr=ts||Std||S)Nz:torch._dynamo is compiling, using Python version of xIELU.)ris_cudarrrrrrs rrzXIELUActivation.forwards^   + ++-- b**5111##$`aaa!!%(((r) rrrrr$bfloat16r/rrrrr9r:s@rr{r{s  n))))))V v &    +V+++++2)V)))))))))rr{rgelu_10i )rDrE gelu_fastgelu_new gelu_pythonr+Tgelu_pytorch_tanh gelu_accuratelaplace leaky_relulinearrX quick_gelurlrelu2relu6rAsilu)swishrprelurc |tvr t|Std|dtt)Nz function z not found in ACT2FN mapping )ACT2FNKeyErrorlistkeys)activation_strings rget_activationr5sOF""'((h#4hhSWX^XcXcXeXeSfSfhhiiir)-r% collectionsrr$rrutilsrutils.import_utilsr get_loggerrrModuler rr*r<r?rCrNrVr^rarjrqr{ LeakyReLUReLUReLU6SigmoidSiLUTanhPReLUACT2CLSrrrrrrrrrX linear_actrrrrs ###### 888888  H % % = = = = =bi = = =wwwww wwwRY,iiiiiiii44444")44477777BI7772 t t t t tRY t t t RY"ry . . . . . . . .BIk[)[)[)[)[)bi[)[)[)|  N  %s2'>'>? # !   N%6$=>     +   ",    N %  BG  "  RX  rz! " BG# $W G X +   .  ! !jjjn]++ >* % %~f N; ' ' ^L ) ) ~f~f ^H % % r