§
    Ý˜ºho ã                   ó,  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZ d dlmZmZ d dlmZmZ d dlmZ d dl
mZ d dlmZmZ d dlmZ d d	lmZ d dlZd d
lm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl)m2Z2  e0¦   «         rd dl3Z3d dlm4Z4m5Z5m6Z6m7Z7 ddl8m9Z9m:Z:  e$¦   «         rd dl;Z; e(¦   «         rd dl<m=Z=  e&¦   «         o e#¦   «         o e'¦   «         o	 e%¦   «         Z>e>rÏd dl?Z?d dl@mAZAmBZB d dlCmDZD d dlEmFZFmGZG d dlHmIZI d dlJmKZK d dlLmMZM d dlNmOZOmPZPmQZQmRZRmSZS d dlTmUZU d dlVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZe d dlfmgZg d dlhmiZimjZjmkZk  G d„ d egd!¬"¦  «        Zl G d#„ d$eUd!¬"¦  «        Zm G d%„ d&eKd!¬"¦  «        Zn ejel¦  «        Zo ejem¦  «        Zp ejen¦  «        Zqh d'£Zrh d(£Zsh d)£Zt e1ju        ev¦  «        Zwd*d+d,d-œiZx eyex z                    ¦   «         ¦  «        Z{ G d.„ d/ej|        ¦  «        Z}d0efd1„Z~d2ed3d4d5d4fd6„Z€ G d7„ d8¦  «        Z� G d9„ d:¦  «        Z‚e G d;„ d<¦  «        ¦   «         Zƒ G d=„ d>e2¦  «        Z„evd?k    r  e„¦   «         Z…e… †                    ¦   «          dS dS )@é    N)ÚArgumentParserÚ	Namespace)Ú	GeneratorÚIterable)Ú	dataclassÚfield)ÚBytesIO)ÚThread)ÚOptionalÚUnion©Ú
model_info©ÚHF_HUB_OFFLINE)Ú!MODEL_FOR_CAUSAL_LM_MAPPING_NAMESÚ*MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES)Úis_fastapi_availableÚis_librosa_availableÚis_openai_availableÚis_pydantic_availableÚis_uvicorn_availableÚis_vision_availableé   )Ú
AutoConfigÚLogitsProcessorListÚPreTrainedTokenizerFastÚProcessorMixinÚTextIteratorStreamer)Úis_torch_availableÚloggingé   )ÚBaseTransformersCLICommand)ÚAutoProcessorÚBitsAndBytesConfigÚGenerationConfigÚPreTrainedModel)ÚContinuousBatchingManagerÚRequestStatus)ÚImage)ÚFastAPIÚHTTPException)ÚCORSMiddleware)ÚJSONResponseÚStreamingResponse)ÚTranscription)ÚTranscriptionCreateParamsBase)ÚChatCompletionMessageParam)ÚChatCompletionChunkÚChoiceÚChoiceDeltaÚChoiceDeltaToolCallÚChoiceDeltaToolCallFunction)ÚCompletionCreateParamsStreaming)ÚResponseÚResponseCompletedEventÚResponseContentPartAddedEventÚResponseContentPartDoneEventÚResponseCreatedEventÚResponseErrorÚResponseErrorEventÚResponseFailedEventÚResponseInProgressEventÚResponseOutputItemAddedEventÚResponseOutputItemDoneEventÚResponseOutputMessageÚResponseOutputTextÚResponseTextDeltaEventÚResponseTextDoneEvent)ÚResponseCreateParamsStreaming)Ú	BaseModelÚTypeAdapterÚValidationErrorc                   ó   — e Zd ZU dZeed<   dS )Ú)TransformersResponseCreateParamsStreamingz
        OpenAI's ResponseCreateParamsStreaming with an additional field for the generation config (as a json string).
        Úgeneration_configN©Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚ__annotations__© ó    úe/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/transformers/commands/serving.pyrL   rL   w   ó*   € € € € € € ð	ð 	ð ÐÐÑÐÐrV   rL   F)Útotalc                   ó   — e Zd ZU dZeed<   dS )Ú+TransformersCompletionCreateParamsStreamingz�
        OpenAI's CompletionCreateParamsStreaming with an additional field for the generation config (as a json string).
        rM   NrN   rU   rV   rW   r[   r[   ~   rX   rV   r[   c                   óB   — e Zd ZU dZeed<   eed<   dZee	         ed<   dS )Ú%TransformersTranscriptionCreateParamsz
        OpenAI's TranscriptionCreateParamsBase with an additional field for the generation config (as a json string).
        ÚfilerM   FÚstreamN)
rO   rP   rQ   rR   ÚbytesrT   rS   r_   r   ÚboolrU   rV   rW   r]   r]   …   sG   € € € € € € ð	ð 	ð ˆˆ‰ØÐÐÑØ!&ˆ�˜”Ð&Ð&Ñ&Ð&Ð&rV   r]   >   ÚtextÚuserÚstoreÚpromptÚincludeÚ	reasoningÚ
backgroundÚ
truncationÚtool_choiceÚservice_tierÚtop_logprobsÚmax_tool_callsÚprevious_response_id>   ÚnÚstoprc   Úaudiord   ÚlogprobsÚmetadataÚ	functionsÚ
modalitiesÚ
predictionrj   rk   rl   Úfunction_callÚstream_optionsÚresponse_formatÚpresence_penaltyÚreasoning_effortÚweb_search_optionsÚparallel_tool_callsÚmax_completion_tokens>   re   rf   Úlanguagery   Úchunking_strategyÚtimestamp_granularitiesÚqwenz<tool_call>z</tool_call>)ÚstartÚendc                   ó   — e Zd ZdZdZdZdZdS )ÚModalityÚLLMÚVLMÚSTTÚTTSN)rO   rP   rQ   r‡   rˆ   r‰   rŠ   rU   rV   rW   r†   r†   Ó   s"   € € € € € Ø
€CØ
€CØ
€CØ
€C€C€CrV   r†   Úargsc                 ó    — t          | ¦  «        S )z~
    Factory function used to instantiate serving server from provided command line arguments.

    Returns: ServeCommand
    )ÚServeCommand)r‹   s    rW   Úserve_command_factoryrŽ   Ú   s   € õ ˜ÑÔÐrV   ÚreqÚmodel_generation_configr%   Úreturnc                 ó  — |                       d¦  «        �%t          di t          j        | d         ¦  «        ¤Ž}nt	          j        |¦  «        } |j        di |¤Ž}|                     ¦   «         D ]\  }}|�t          |||¦  «         Œ|                       d¦  «        �t          | d         ¦  «        |_
        |                       d¦  «        �t          | d         ¦  «        |_
        |                       d¦  «        �t          | d         ¦  «        |_        |                       d¦  «        �| d         |_        |                       d¦  «        �| d         |_        |                       d¦  «        �:t          | d         ¦  «        |_        t          | d         ¦  «        d	k    rd
|_        |                       d¦  «        �t          | d         ¦  «        |_        |                       d¦  «        �t%          j        | d         ¦  «         |S )a¬  
    Creates a generation config from the parameters of the request. If a generation config is passed in the request,
    it will be used as a baseline for parameterization. Otherwise, we will use the model's default generation config.
    Other parameters in the request will be applied on top of the baseline.

    Args:
        req (`dict`):
            The request which may optionally contain generation parameters.
        model_generation_config (`GenerationConfig`):
            The model's default generation config.
        kwargs (`dict`):
            Additional parameters to set in the generation config.

    Returns:
        The prepared `GenerationConfig` object.
    rM   NÚmax_output_tokensÚ
max_tokensÚfrequency_penaltyÚ
logit_biasrp   Útemperatureg        FÚtop_pÚseedrU   )Úgetr%   ÚjsonÚloadsÚcopyÚdeepcopyÚupdateÚitemsÚsetattrÚintÚmax_new_tokensÚfloatÚrepetition_penaltyÚsequence_biasÚstop_stringsr—   Ú	do_sampler˜   ÚtorchÚmanual_seed)r�   r�   ÚkwargsrM   Únon_standard_kwargsÚkÚvs          rW   Ú!create_generation_config_from_reqr¯   ã   sù  € ð. ‡w‚wÐ"Ñ#Ô#Ð/Ý,ÐTÐT­t¬z¸#Ð>QÔ:RÑ/SÔ/SÐTÐTÐÐå œMÐ*AÑBÔBÐà2Ð+Ô2Ð<Ð<°VÐ<Ð<Ðà#×)Ò)Ñ+Ô+ð -ð -‰ˆˆ1Øˆ=ÝÐ% q¨!Ñ,Ô,Ð,øð ‡w‚wÐ"Ñ#Ô#Ð/Ý+.¨sÐ3FÔ/GÑ+HÔ+HÐÔ(ð ‡w‚wˆ|ÑÔÐ(Ý+.¨s°<Ô/@Ñ+AÔ+AÐÔ(Ø
‡w‚wÐ"Ñ#Ô#Ð/Ý/4°SÐ9LÔ5MÑ/NÔ/NÐÔ,Ø
‡w‚wˆ|ÑÔÐ(Ø*-¨lÔ*;ÐÔ'Ø
‡w‚wˆv�„Ð"Ø),¨V¬ÐÔ&Ø
‡w‚wˆ}ÑÔÐ)Ý(-¨c°-Ô.@Ñ(AÔ(AÐÔ%Ý��]Ô#Ñ$Ô$¨Ò+Ð+Ø*/ÐÔ'Ø
‡w‚wˆwÑÔÐ#Ý"'¨¨G¬Ñ"5Ô"5ÐÔØ
‡w‚wˆv�„Ð"ÝÔ˜#˜fœ+Ñ&Ô&Ð&àÐrV   c                   ó   — e Zd ZdZd„ Zd„ ZdS )Ú	ToolStatez7Lightweight class to keep track of the tool call state.c                 ó.   — |                       ¦   «          d S ©N)Úreset©Úselfs    rW   Ú__init__zToolState.__init__!  s   € Ø�
Š
‰ŒˆˆˆrV   c                 ó>   — d| _         d| _        d| _        d| _        dS )z>Reset the tool call state (assumes we're outside a tool call).Fr   Ú N)Úinside_tool_callÚhas_tool_name_definedÚarg_nesting_levelÚbufferrµ   s    rW   r´   zToolState.reset$  s%   € à %ˆÔØ%*ˆÔ"Ø!"ˆÔØˆŒˆˆrV   N)rO   rP   rQ   rR   r·   r´   rU   rV   rW   r±   r±     s8   € € € € € ØAÐAðð ð ðð ð ð ð rV   r±   c            	       óT   — e Zd ZdZ	 ddddedeed                  fd„Zd	„ Zd
„ Z	d„ Z
dS )Ú
TimedModelz˜
    A class that holds a PreTrainedModel instance and its associated processor.
    Automatically deletes the instances after a specified timeout.
    NÚmodelr&   Útimeout_secondsÚ	processor)r   r   c                 óÜ   — || _         t          |j        ¦  «        | _        || _        || _        t          j        | j        | j        ¦  «        | _	        | j	         
                    ¦   «          d S r³   )rÀ   rS   Úname_or_pathÚ_name_or_pathrÂ   rÁ   Ú	threadingÚTimerÚ_delete_modelÚ_timerrƒ   )r¶   rÀ   rÁ   rÂ   s       rW   r·   zTimedModel.__init__2  s`   € ð ˆŒ
Ý  Ô!3Ñ4Ô4ˆÔØ"ˆŒØ.ˆÔÝ”o dÔ&:¸DÔ<NÑOÔOˆŒØŒ×ÒÑÔÐÐÐrV   c                 ó²   — | j                              ¦   «          t          j        | j        | j        ¦  «        | _         | j                              ¦   «          dS )z2Reset the timer for the deletion of the instances.N)rÉ   ÚcancelrÆ   rÇ   rÁ   rÈ   rƒ   rµ   s    rW   Úreset_timerzTimedModel.reset_timer?  sI   € àŒ×ÒÑÔÐÝ”o dÔ&:¸DÔ<NÑOÔOˆŒØŒ×ÒÑÔÐÐÐrV   c                 óR  — t          | d¦  «        r”| j        ��| `| `d| _        d| _        t          j        ¦   «          t
          j                             ¦   «         rt
          j                             ¦   «          t           
                    | j        › d| j        › d�¦  «         dS dS dS )z>Delete the wrapped model and processor and clean up resources.rÀ   Nz was removed from memory after z seconds of inactivity)ÚhasattrrÀ   rÂ   ÚgcÚcollectr©   ÚcudaÚis_availableÚempty_cacheÚloggerÚinforÅ   rÁ   rµ   s    rW   rÈ   zTimedModel._delete_modelE  s´   € å�4˜Ñ!Ô!ð 	 d¤jÐ&<Ø�
Ø�ØˆDŒJØ!ˆDŒNÝŒJ‰LŒLˆLõ Œz×&Ò&Ñ(Ô(ð )Ý”
×&Ò&Ñ(Ô(Ð(å�KŠKØÔ%ÐrÐrÀdÔFZÐrÐrÐrñô ð ð ð ð	ð 	Ð&<Ð&<rV   c                 ó6   — t          | d¦  «         p| j        du S )z)Check if the instances have been deleted.rÀ   N)rÎ   rÀ   rµ   s    rW   Ú
is_deletedzTimedModel.is_deletedV  s!   € å˜4 Ñ)Ô)Ð)Ð?¨T¬Z¸4Ð-?Ð?rV   r³   )rO   rP   rQ   rR   r¢   r   r   r·   rÌ   rÈ   r×   rU   rV   rW   r¿   r¿   ,  s—   € € € € € ðð ð SWð	ð à ðð ðð ˜EÐ"MÔNÔOð	ð ð ð ðð ð ðð ð ð"@ð @ð @ð @ð @rV   r¿   c                   ó  — e Zd ZU dZ edddi¬¦  «        Zeed<    eddg d	¢d
œ¬¦  «        Ze	e         ed<    eddg d	¢d
œ¬¦  «        Z
e	e         ed<    edddi¬¦  «        Zeed<    edddi¬¦  «        Ze	e         ed<    edddi¬¦  «        Zeed<    edddi¬¦  «        Zeed<    eddddgd
œ¬¦  «        Zeed<    edddi¬¦  «        Zeed<    edddi¬¦  «        Zeed<    ed dd!i¬¦  «        Zeed"<    ed#dd$i¬¦  «        Zeed%<    ed&dd'i¬¦  «        Zeed(<    eddd)i¬¦  «        Ze	e         ed*<    eddd+i¬¦  «        Zeed,<    eddd-i¬¦  «        Zeed.<    eddd/i¬¦  «        Ze	e         ed0<   d1„ ZdS )2ÚServeArgumentsz¥
    Arguments for the serve CLI.

    See the metadata arg for each argument's description -- the metadata will be printed with
    `transformers serve --help`
    ÚautoÚhelpzfDevice to use for inference; will default to `auto` andplace the model on an accelerator if available.)Údefaultrs   ÚdeviceNzA`torch_dtype` is deprecated! Please use `dtype` argument instead.)rÚ   Úbfloat16Úfloat16Úfloat32)rÛ   ÚchoicesÚtorch_dtypez Override the default `torch.dtype` and load the model under this dtype. If `'auto'` is passed, the dtype will be automatically derived from the model's weights.ÚdtypeFz2Whether to trust remote code when loading a model.Útrust_remote_codezÂWhich attention implementation to use; you can run --attn_implementation=flash_attention_2, in which case you must install this manually by running `pip install flash-attn --no-build-isolation`.Úattn_implementationzIWhether to use 8 bit precision for the base model - works only with LoRA.Úload_in_8bitzIWhether to use 4 bit precision for the base model - works only with LoRA.Úload_in_4bitÚnf4zQuantization type.Úfp4Úbnb_4bit_quant_typez#Whether to use nested quantization.Úuse_bnb_nested_quantÚ	localhostz$Interface the server will listen to.Úhosti@  zPort the server will listen to.Úporti,  z@Time in seconds after which a model will be removed from memory.Úmodel_timeoutrÕ   z8Logging level as a string. Example: 'info' or 'warning'.Ú	log_levelz1The default seed for torch, should be an integer.Údefault_seedztWhether to enable CORS. Some apps that make requests from external domains (e.g. Cursor) require CORS to be enabled.Úenable_corsz+Whether to turn on strict input validation.Úinput_validationz€Name of the model to be forced on all requests. This is useful for testing Apps that don't allow changing models in the request.Úforce_modelc                 óJ   — | j         �| j        dk    r| j         | _        dS dS dS )z(Only used for BC `torch_dtype` argument.NrÚ   )râ   rã   rµ   s    rW   Ú__post_init__zServeArguments.__post_init__¼  s5   € ð ÔÐ'¨D¬J¸&Ò,@Ð,@ØÔ)ˆDŒJˆJˆJð (Ð'Ð,@Ð,@rV   )rO   rP   rQ   rR   r   rÝ   rS   rT   râ   r   rã   rä   ra   rå   ræ   rç   rê   rë   rí   rî   r¢   rï   rð   rñ   rò   ró   rô   rö   rU   rV   rW   rÙ   rÙ   [  sÆ  € € € € € € ðð ð �%Øàð >ð
ðñ ô €FˆCð ð ñ ð "' ØàWØAÐAÐAð
ð 
ð"ñ "ô "€K�˜#”ð ð ñ ð !˜5ØðPàAÐAÐAð
ð 
ðñ ô €Eˆ8�CŒ=ð ð ñ ð $˜eØ Ð)]Ð ^ðñ ô Ð�tð ð ñ ð */¨Øàð rð
ð*ñ *ô *Ð˜ #œð ð ñ ð ˜ØØÐeÐfðñ ô €L�$ð ð ñ ð ˜ØØÐeÐfðñ ô €L�$ð ð ñ ð  %˜u¨UÐFZÐhmÐotÐguÐ=vÐ=vÐwÑwÔwÐ˜ÐwÐwÑwØ!& ¨uÀÐHmÐ?nÐ!oÑ!oÔ!oÐ˜$ÐoÐoÑoð �˜k°VÐ=cÐ4dÐeÑeÔe€Dˆ#ÐeÐeÑeØ�˜d¨fÐ6WÐ-XÐYÑYÔY€Dˆ#ÐYÐYÑYØ˜ØØÐ\Ð]ðñ ô €M�3ð ð ñ ð �UØ &Ð*dÐ!eðñ ô €Iˆsð ð ñ ð #( %Ø Ð([Ð\ð#ñ #ô #€L�(˜3”-ð ð ñ ð ˜Øàð&ð
ðñ ô €K�ð ð ñ ð #˜UØàÐBð
ðñ ô Ð�dð ð ñ ð "' Øàð2ð
ð"ñ "ô "€K�˜#”ð ð ñ ð*ð *ð *ð *ð *rV   rÙ   c                   óÀ  — e Zd Zedefd„¦   «         Zdefd„Zdedddd	d
e	fd„Z
defd„Zdefd„Zdefd„Z	 	 	 	 	 	 d2dee         dee         dee         dee         dee         deed                  defd„Zdddefd„Zd„ Z ej        d¬¦  «        deeeef                  fd„¦   «         Zd edeeddf         fd!„Zedd"defd#„¦   «         Zed$efd%„¦   «         Zd edeeddf         fd&„Zd edeeddf         fd'„Zd edeeddf         fd(„Zd ede fd)„Z!ededed*         fd+„¦   «         Z"d,edefd-„Z#d.efd/„Z$d.ede%d"e&f         fd0„Z'd.ede%d"e(f         fd1„Z)dS )3r�   Úparserc                 óz   — t           f}|                      d|¬¦  «        }|                     t          ¬¦  «         dS )z³
        Register this command to argparse so it's available for the transformer-cli

        Args:
            parser: Root parser to register command-specific arguments
        Úserve)Údataclass_types)ÚfuncN)rÙ   Ú
add_parserÚset_defaultsrŽ   )rø   rû   Úserve_parsers      rW   Úregister_subcommandz ServeCommand.register_subcommandÄ  sB   € õ *Ð+ˆØ×(Ò(¨À/Ð(ÑRÔRˆØ×!Ò!Õ'<Ð!Ñ=Ô=Ð=Ð=Ð=rV   r‹   c                 óz  — t           st          d¦  «        ‚|| _        | j        j        dk    | _        | j        j        | _        | j        j        �t          j        | j        j        ¦  «         t          j
        d¦  «        }|                     t          j        | j        j                             ¦   «                  ¦  «         t          j
        d¦  «        }|                     t          j        | j        j                             ¦   «                  ¦  «         i | _        d | _        d | _        d | _        d | _        d S )NzaMissing dependencies for the serving CLI. Please install with `pip install transformers[serving]`Ú
sdpa_pagedÚtransformersz+transformers.generation.continuous_batching)Úserve_dependencies_availableÚImportErrorr‹   rå   Úuse_continuous_batchingrò   rñ   r©   rª   r    Ú
get_loggerÚsetLevelÚ
log_levelsrð   ÚlowerÚloaded_modelsÚ#running_continuous_batching_managerÚlast_messagesÚlast_kv_cacheÚ
last_model)r¶   r‹   Útransformers_loggerÚ	cb_loggers       rW   r·   zServeCommand.__init__Ð  s  € Ý+ð 	ÝØsñô ð ð
 ˆŒ	Ø'+¤yÔ'DÈÒ'TˆÔ$Øœ9Ô0ˆÔàŒ9Ô!Ð-ÝÔ˜dœiÔ4Ñ5Ô5Ð5õ &Ô0°Ñ@Ô@ÐØ×$Ò$¥WÔ%7¸¼	Ô8K×8QÒ8QÑ8SÔ8SÔ%TÑUÔUÐUåÔ&Ð'TÑUÔUˆ	Ø×Ò�7Ô-¨d¬iÔ.A×.GÒ.GÑ.IÔ.IÔJÑKÔKÐKð 57ˆÔØX\ˆÔ0ð "ˆÔØ!ˆÔØˆŒˆˆrV   ÚrequestÚschemaÚ_TypedDictMetaÚ	validatorrI   Úunused_fieldsc                 ó‚  — t                                d|› �¦  «         t          |                     ¦   «         ¦  «        }|j        }||z
  }|r1t                                d|› �¦  «         t          dd|› �¬¦  «        ‚| j        j        r³	 | 	                    |¦  «         nd# t          $ rW}t                                d|                     ¦   «         › �¦  «         t          d|                     ¦   «         ¬¦  «        ‚d}~ww xY w||z  }	|	r3t                                d|	› �¦  «         t          dd|	› �¬¦  «        ‚dS dS )aŽ  
        Validates the request against the schema, and checks for unexpected keys.

        Args:
            request (`dict`):
                The request to validate.
            schema (`_TypedDictMeta`):
                The schema of the request to validate. It is a `TypedDict` definition.
            validator (`TypeAdapter`):
                The validator to use to validate the request. Built from `schema`.
            unused_fields (`set`):
                Fields accepted by `schema`, but not used in `transformers serve`.

        Raises:
            HTTPException: If the request is invalid or contains unexpected or unused fields.
        zValidating request: z Unexpected keys in the request: i¦  )Ústatus_codeÚdetailzValidation error: NzUnused fields in the request: )rÔ   ÚdebugÚsetÚkeysÚ__mutable_keys__Úerrorr+   r‹   ró   Úvalidate_pythonrJ   Úerrors)
r¶   r  r  r  r  Ú
input_keysÚpossible_keysÚunexpected_keysÚeÚunused_fields_in_requests
             rW   Ú_validate_requestzServeCommand._validate_requestð  s  € õ. 	�ŠÐ5¨GÐ5Ð5Ñ6Ô6Ð6õ ˜Ÿš™œÑ(Ô(ˆ
ØÔ/ˆØ$ }Ñ4ˆØð 	nÝ�LŠLÐM¸OÐMÐMÑNÔNÐNÝ¨CÐ8lÐ[jÐ8lÐ8lÐmÑmÔmÐmàŒ9Ô%ð 	ðHØ×)Ò)¨'Ñ2Ô2Ð2Ð2øÝ"ð Hð Hð HÝ—’Ð>°!·(²(±*´*Ð>Ð>Ñ?Ô?Ð?Ý#°¸A¿HºH¹J¼JÐGÑGÔGÐGøøøøðHøøøð
 (2°MÑ'AÐ$Ø'ð Ý—’ÐXÐ>VÐXÐXÑYÔYÐYÝ#Ø #Ð,gÐMeÐ,gÐ,gðñ ô ð ð	ð 	ðð s   ÂB! Â!
DÂ+AC=Ã=Dc                 óV   — |                       |t          t          t          ¬¦  «         d S ©N)r  r  r  r  )r&  rL   Úresponse_validatorÚUNUSED_RESPONSE_FIELDS©r¶   r  s     rW   Úvalidate_response_requestz&ServeCommand.validate_response_request!  s5   € Ø×ÒØÝ<Ý(Ý0ð	 	ñ 	
ô 	
ð 	
ð 	
ð 	
rV   c                 óV   — |                       |t          t          t          ¬¦  «         d S r(  )r&  r[   Úcompletion_validatorÚUNUSED_CHAT_COMPLETION_FIELDSr+  s     rW   Ú validate_chat_completion_requestz-ServeCommand.validate_chat_completion_request)  s5   € Ø×ÒØÝ>Ý*Ý7ð	 	ñ 	
ô 	
ð 	
ð 	
ð 	
rV   c                 óV   — |                       |t          t          t          ¬¦  «         d S r(  )r&  r]   Útranscription_validatorÚUNUSED_TRANSCRIPTION_FIELDSr+  s     rW   Úvalidate_transcription_requestz+ServeCommand.validate_transcription_request1  s5   € Ø×ÒØÝ8Ý-Ý5ð	 	ñ 	
ô 	
ð 	
ð 	
ð 	
rV   r¹   NÚ
request_idÚcontentrÀ   ÚroleÚfinish_reasonÚ
tool_callsr5   r‘   c                 óÞ   — t          |t          t          j        ¦   «         ¦  «        |t          t	          |||¬¦  «        d|¬¦  «        gdd¬¦  «        }d|                     d¬	¦  «        › d
�S )aç  
        Builds a chunk of a streaming OpenAI Chat Completion response.

        IMPORTANT: The serialized chunk won't contain empty fields (fields with `None`). Some downstream apps,
        like Cursor, assume that when the field exists, it has data.

        Args:
            request_id (`str`):
                The request ID.
            content (`str`, *optional*):
                Content of the response from the model.
            model (`str`, *optional*):
                The model that generated the content.
            role (`str`, *optional*):
                The role of the next content, until a new role is defined.
            finish_reason (`str`, *optional*):
                The reason the generation by the model has finished.
            tool_calls (`list[ChoiceDeltaToolCall]`, *optional*):
                Data about the tool calls, when they are triggered.

        Returns:
            `str`: The built chunk, a string containing a JSON string with the payload.
        )r6  r7  r9  r   )ÚdeltaÚindexr8  r¹   zchat.completion.chunk)ÚidÚcreatedrÀ   rá   Úsystem_fingerprintÚobjectúdata: T©Úexclude_noneú

)r2   r¢   Útimer3   r4   Úmodel_dump_json)r¶   r5  r6  rÀ   r7  r8  r9  Úchunks           rW   Úbuild_chat_completion_chunkz(ServeCommand.build_chat_completion_chunk9  s—   € õ@ $ØÝ�œ	™œÑ$Ô$ØåÝ%Ø 'Ø!Ø#-ðñ ô ð
 Ø"/ðñ ô ð
ð  "Ø*ð!
ñ 
ô 
ˆð$ G˜×-Ò-¸4Ð-Ñ@Ô@ÐFÐFÐFÐFrV   ÚresponserH   c                 ó6   — d|                      d¬¦  «        › d�S )a  
        Builds a event of a streaming OpenAI Response response.

        IMPORTANT: The serialized chunk won't contain empty fields (fields with `None`). Some downstream apps,
        like Cursor, assume that when the field exists, it has data.

        Args:
            response (`BaseModel`):
                The response to build an event from. One of the multiple OpenAI Response output types

        Returns:
            `str`: The built chunk, a string containing a JSON string with the payload.
        rA  TrB  rD  )rF  )r¶   rI  s     rW   Úbuild_response_eventz!ServeCommand.build_response_eventm  s'   € ð J˜×0Ò0¸dÐ0ÑCÔCÐIÐIÐIÐIrV   c                 ó´  ‡ — t          ¦   «         }‰ j        r=|                     t          dgddgdg¬¦  «         t                               d¦  «         nt                               d¦  «         |                     d¦  «        dt          fˆ fd„¦   «         }|                     d	¦  «        dt          fˆ fd
„¦   «         }ddlm	} |                     d¦  «        d|fˆ fd„¦   «         }| 
                    d¦  «        |                     d¦  «        ˆ fd„¦   «         ¦   «         }t          j        |‰ j        j        ‰ j        j        ‰ j        j        ¬¦  «         d S )NÚ*T)Úallow_originsÚallow_credentialsÚallow_methodsÚallow_headerszUCORS allow origin is set to `*`. This is not recommended for production environments.zaSome apps may require CORS. Consider launching the server with `--enable-cors` if you see errors.z/v1/chat/completionsr  c                 ó¶   •— ‰                      | ¬¦  «         ‰j        r‰                     | ¦  «        }n‰                     | ¦  «        }t	          |d¬¦  «        S ©N©r  útext/event-stream©Ú
media_type)r0  r  Ú#continuous_batching_chat_completionÚgenerate_chat_completionr.   ©r  Úoutputr¶   s     €rW   Úchat_completionz)ServeCommand.run.<locals>.chat_completion’  sb   ø€ à×1Ò1¸'Ð1ÑBÔBÐBàÔ+ð @Ø×AÒAÀ'ÑJÔJ��à×6Ò6°wÑ?Ô?�Ý$ VÐ8KÐLÑLÔLÐLrV   z/v1/responsesc                 ó|   •— ‰                      | ¬¦  «         ‰                     | ¦  «        }t          |d¬¦  «        S rS  )r,  Úgenerate_responser.   rZ  s     €rW   Ú	responsesz#ServeCommand.run.<locals>.responsesœ  sA   ø€ à×*Ò*°7Ð*Ñ;Ô;Ð;à×+Ò+¨GÑ4Ô4ˆFÝ$ VÐ8KÐLÑLÔLÐLrV   r   )ÚRequestz/v1/audio/transcriptionsc           
   “   óø  •K  — |                       ¦   «         4 ƒd {V —†}t          |d                              ¦   «         ƒ d {V —†|d         ¬¦  «        }t                               d|d         j        › d|d         j        › d|d         j        dz  d›d	�¦  «         d d d ¦  «        ƒd {V —† n# 1 ƒd {V —†swxY w Y   ‰                     |¬
¦  «         ‰ 	                    |¦  «        }t          |d¬¦  «        S )Nr^   rÀ   )r^   rÀ   zReceived file: z; MIME type: z; size: é   z.2fz KiBrT  rU  rV  )Úformr]   ÚreadrÔ   r  ÚfilenameÚcontent_typeÚsizer4  Úgenerate_transcriptionr.   )r  rc  Úparsed_requestr[  r¶   s       €rW   Úaudio_transcriptionsz.ServeCommand.run.<locals>.audio_transcriptions¥  s¯  øè è € ð —|’|‘~”~ð 	ð 	ð 	ð 	ð 	ð 	ð 	¨Ý!FØ# Fœ|×0Ò0Ñ2Ô2Ð2Ð2Ð2Ð2Ð2Ð2Ø˜wœ-ð"ñ "ô "�õ
 —’ð@ d¨6¤lÔ&;ð @ð @È$ÈvÌ,ÔJcð @ð @Ø! &œ\Ô.°Ñ5Ð?ð@ð @ð @ñô ð ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð ×/Ò/¸Ð/ÑGÔGÐGà×0Ò0°Ñ@Ô@ˆFÝ$ VÐ8KÐLÑLÔLÐLs   žA?B/Â/
B9Â<B9z
/v1/modelsc                  óL   •— t          d‰                      ¦   «         dœ¦  «        S )NÚlist)r@  Údata)r-   Úget_gen_modelsrµ   s   €rW   Úget_all_modelsz(ServeCommand.run.<locals>.get_all_models·  s'   ø€ õ  ¨6¸4×;NÒ;NÑ;PÔ;PÐ QÐ QÑRÔRÐRrV   )rí   rî   rð   )r*   rò   Úadd_middlewarer,   rÔ   Úwarning_onceÚpostÚdictÚfastapir`  Úoptionsrš   ÚuvicornÚrunr‹   rí   rî   rð   )r¶   Úappr\  r_  r`  rj  ro  s   `      rW   rw  zServeCommand.run}  sé  ø€ Ý‰iŒiˆð Ôð 	Ø×ÒÝØ"˜eØ"&Ø"˜eØ"˜eð ñ ô ð õ ×ÒØgñô ð ð õ ×ÒØsñô ð ð 
�ŠÐ(Ñ	)Ô	)ð	M¥Tð 	Mð 	Mð 	Mð 	Mð 	Mñ 
*Ô	)ð	Mð 
�Š�/Ñ	"Ô	"ð	M�tð 	Mð 	Mð 	Mð 	Mð 	Mñ 
#Ô	"ð	Mð 	$Ð#Ð#Ð#Ð#Ð#à	�ŠÐ,Ñ	-Ô	-ð	M°ð 	Mð 	Mð 	Mð 	Mð 	Mñ 
.Ô	-ð	Mð" 
�Š�\Ñ	"Ô	"Ø	�Š�Ñ	Ô	ð	Sð 	Sð 	Sð 	Sñ 
Ô	ñ 
#Ô	"ð	Sõ 	Œ�C˜dœiœn°4´9´>ÈTÌYÔM`ÐaÑaÔaÐaÐaÐarV   )Úmaxsizec                 ó`   — g d¢}t           rd„ |D ¦   «         S d„ |D ¦   «         }d„ |D ¦   «         S )a.  
        This is by no means a limit to which models may be instantiated with `transformers serve`: any chat-based
        model working with generate can work.

        This is a limited list of models to ensure we have a discoverable /v1/models endpoint for third-party
        integrations.
        )zMenlo/Jan-nanozMenlo/Jan-nano-128kzQwen/Qwen2.5-0.5B-InstructzQwen/Qwen2.5-3B-InstructzQwen/Qwen2.5-7B-InstructzQwen/Qwen2.5-14B-Instructz meta-llama/Llama-3.1-8B-Instructz meta-llama/Llama-3.2-1B-Instructz!meta-llama/Llama-3.3-70B-InstructzHuggingFaceTB/SmolVLM-Instructz!ibm-granite/granite-vision-3.2-2bzQwen/Qwen2.5-VL-7B-Instructc                 óª   — g | ]P}|d t           j                              ¦   «                              ¦   «         |                     d¦  «        d         dœ‘ŒQS )rÀ   ú/r   ©r=  r@  r>  Úowned_by)ÚdatetimeÚnowÚ	timestampÚsplit©Ú.0rÀ   s     rW   ú
<listcomp>z/ServeCommand.get_gen_models.<locals>.<listcomp>×  si   € ð ð ð ð ð  Ø%Ý'Ô0×4Ò4Ñ6Ô6×@Ò@ÑBÔBØ %§¢¨CÑ 0Ô 0°Ô 3ð	ð ðð ð rV   c                 ó,   — g | ]}t          |¦  «        ‘ŒS rU   r   rƒ  s     rW   r…  z/ServeCommand.get_gen_models.<locals>.<listcomp>á  s    € ÐAÐAÐA°�: eÑ,Ô,ÐAÐAÐArV   c                 ó^   — g | ]*}|j         d |j                             ¦   «         |j        dœ‘Œ+S )rÀ   r}  )r=  Ú
created_atr�  Úauthorrƒ  s     rW   r…  z/ServeCommand.get_gen_models.<locals>.<listcomp>â  sP   € ð ð ð ð ð  œ(Ø%Ø$Ô/×9Ò9Ñ;Ô;Ø %¤ð	ð ðð ð rV   r   )r¶   ÚmodelsÚmodel_infoss      rW   rn  zServeCommand.get_gen_models¾  sy   € ð
ð 
ð 
ˆõ ð 	ðð ð $ðñ ô ð ð BÐA¸&ÐAÑAÔAˆKðð ð )ðñ ô ð rV   r�   c                 ó´  ‡ ‡‡‡	— ‰                       ‰d         ¦  «        Š	‰	‰ j        k    }‰	‰ _        |r*‰ j        �#‰ j                             dd¬¦  «         d‰ _        ‰                      ‰	¦  «        \  }}t          |d¦  «        r|j        n|}t          ‰|j        |j	        |j
        ddd	dd
d¬¦
  «
        Š‰ j        €M|                     ‰d¬¦  «        ‰ _        t          ¦   «         ‰ j        _        ‰ j                             ¦   «          |                     ‰d         dd¬¦  «                             |j        ¦  «        }ˆˆ	ˆˆ fd„} ||d         ¦  «        S )a'  
        Generates an OpenAI Chat Completion using continuous batching.

        Args:
            req (`dict`): The request to generate an OpenAI Chat Completion for.

        Returns:
            `Generator[str, None, None]`: A generator that yields the OpenAI Chat Completion chunks.
        rÀ   NTr   )ÚblockÚtimeoutÚ	tokenizerFr!   rb  é
   Úfifo)	r�   Úeos_token_idÚpad_token_idÚ	use_cacheÚ
num_blocksÚ
block_sizer¨   Úmax_batch_tokensÚ	scheduler)rM   Ú	streamingÚmessagesÚpt)Úreturn_tensorsÚadd_generation_promptc              3   ó¶  •K  — 	 ‰	j                              | ‰                     d¦  «        ‰j        ¬¦  «        }d}‰	                     |d‰¬¦  «        V — ‰	j         D ]§}|j        |k    rŒ‰                     d¦  «        �|s|j        t          j        k    rŒ;d}|j        t          j        k    rdnd }|j        t          j        k    r‰	                     ||‰¬¦  «        V —  d S ‰	                     ||j	        ‰¬	¦  «        V — Œ¨d S # t          $ rG}t                               t          |¦  «        ¦  «         d
t          |¦  «        › d�V — Y d }~d S d }~ww xY w)Nr5  )r5  r£   FÚ	assistant©r7  rÀ   Trp   ©r8  rÀ   )r5  r6  rÀ   údata: {"error": "ú"})r  Úadd_requestrš   r£   rH  r5  Ústatusr(   ÚFINISHEDÚ
next_tokenÚ	ExceptionrÔ   r  rS   )
Ú_inputsr5  Úqueue_is_flushedÚresultr8  r$  rM   Úmodel_id_and_revisionr�   r¶   s
         €€€€rW   Ústream_chat_completionzPServeCommand.continuous_batching_chat_completion.<locals>.stream_chat_completion  sË  øè è € ð!7Ø!ÔE×QÒQØ¨¯ª°Ñ(=Ô(=ÐN_ÔNnð Rñ ô �
ð $)Ð ð ×6Ò6°zÈÐ[pÐ6ÑqÔqÐqÐqÐqà"ÔFð ð �FØÔ(¨JÒ6Ð6Ø Ø—w’w˜|Ñ,Ô,Ð8ÐAQÐ8Ø!œ=­MÔ,BÒBÐBØ$à/3Ð,à.4¬m½}Ô?UÒ.UÐ.U F FÐ[_�MØ”}­Ô(>Ò>Ð>Ø"×>Ò>Ø&°mÐK`ð ?ñ ô ð ð ð ð ˜˜à"×>Ò>Ø'1¸6Ô;LÐTið ?ñ ô ð ð ð ð ð!ð øõ( ð 7ð 7ð 7Ý—’�S ™VœVÑ$Ô$Ð$Ø6­3¨q©6¬6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6øøøøð7øøøs   …CD Ã% D Ä
EÄ<EÅEr   )Úprocess_model_namer  r  rp   Úload_model_and_processorrÎ   r�  r¯   rM   r’  r“  Úinit_continuous_batchingr   Úlogit_processorrƒ   Úapply_chat_templateÚtorÝ   )
r¶   r�   Úmust_discard_cacherÀ   rÂ   r�  Úinputsr­  rM   r¬  s
   ``      @@rW   rX  z0ServeCommand.continuous_batching_chat_completionì  s«  øøøø€ ð !%× 7Ò 7¸¸G¼Ñ EÔ EÐØ2°d´oÒEÐØ/ˆŒØð 	@àÔ7ÐCØÔ8×=Ò=ÀDÐRSÐ=ÑTÔTÐTØ;?�Ô8Ø×8Ò8Ð9NÑOÔOÑˆˆyå+2°9¸kÑ+JÔ+JÐY�IÔ'Ð'ÐPYˆ	å=ØØ$)Ô$;Ø"Ô/Ø"Ô/ØØØØØØð
ñ 
ô 
Ðð Ô3Ð;Ø7<×7UÒ7UØ"3¸tð 8Vñ 8ô 8ˆDÔ4õ H[ÑG\ÔG\ˆDÔ4ÔDØÔ4×:Ò:Ñ<Ô<Ð<ð ×.Ò.¨s°:¬ÈtÐkoÐ.ÑpÔp×sÒsØŒLñ
ô 
ˆð"	7ð "	7ð "	7ð "	7ð "	7ð "	7ð "	7ð "	7ðH &Ð% f¨Q¤iÑ0Ô0Ð0rV   r&   c                 óÊ   — | j         j        }|t          j        ¦   «         v rt          j        }n4|t          j        ¦   «         v rt          j        }nt          d|› �¦  «        ‚|S )NzUnknown modality: )	Ú	__class__rO   r   Úvaluesr†   rˆ   r   r‡   Ú
ValueError)rÀ   Úmodel_classnameÚmodalitys      rW   Úget_model_modalityzServeCommand.get_model_modalityE  sd   € àœ/Ô2ˆØÕHÔOÑQÔQÐQÐQÝ”|ˆHˆHØÕ AÔ HÑ JÔ JÐJÐJÝ”|ˆHˆHåÐC°/ÐCÐCÑDÔDÐDàˆrV   r»  c           	      óN  — g }| D �]}|d         g dœ}|t           j        k    r�t          |d         t          ¦  «        r	|d         }ndt          |d         t          ¦  «        rIg }|d         D ])}|d         dk    r|                     |d         ¦  «         Œ*d                     |¦  «        }||d<   �n\|t           j        k    �rKt          |d         t          ¦  «        r&|d                              d|d         dœ¦  «         �n
|d         D �] }|d         dk    r|d                              |¦  «         Œ+|d         dk    rÉd	|d         d
         v r�t          j	        dd|d         d
         ¦  «        }t          j        t          t          j        |¦  «        ¦  «        ¦  «        }t          j        dd¬¦  «        }	|	j        }
|                     |	j        ¦  «         n|d         d
         }
|d                              d|
dœ¦  «         �Œ|                     |¦  «         �Œ |S )Nr7  ©r7  r6  r6  Útyperb   ú )r¿  rb   Ú	image_urlÚbase64Úurlz^data:image/.+;base64,r¹   z.pngF)ÚsuffixÚdeleteÚimage)r¿  rÃ  )r†   r‡   Ú
isinstancerS   rl  ÚappendÚjoinrˆ   ÚreÚsubr)   Úopenr	   rÂ  Ú	b64decodeÚtempfileÚNamedTemporaryFileÚnameÚsave)rš  r»  Úprocessor_inputsÚmessageÚparsed_messageÚparsed_contentr6  Ú
image_datarÆ  r^   rÃ  s              rW   Ú*get_processor_inputs_from_inbound_messagesz7ServeCommand.get_processor_inputs_from_inbound_messagesQ  sN  € àÐàð '	4ñ '	4ˆGØ&-¨f¤oÀ"ÐEÐEˆNà�8œ<Ò'Ð'õ ˜g iÔ0µ#Ñ6Ô6ð >Ø%,¨YÔ%7�N�NÝ ¨	Ô 2µDÑ9Ô9ð >Ø%'�NØ#*¨9Ô#5ð Cð C˜Ø" 6œ?¨fÒ4Ð4Ø*×1Ò1°'¸&´/ÑBÔBÐBøØ%(§X¢X¨nÑ%=Ô%=�NØ,:�˜yÑ)Ñ)à�Xœ\Ò)Ñ)õ ˜g iÔ0µ#Ñ6Ô6ð \Ø" 9Ô-×4Ò4¸fÈgÐV_ÔN`Ð5aÐ5aÑbÔbÐbÑbà#*¨9Ô#5ð \ñ \˜Ø" 6œ?¨fÒ4Ð4Ø*¨9Ô5×<Ò<¸WÑEÔEÐEÐEØ$ Vœ_°Ò;Ð;Ø'¨7°;Ô+?ÀÔ+FÐFÐFÝ-/¬VÐ4LÈbÐRYÐZeÔRfÐglÔRmÑ-nÔ-n 
Ý(-¬
µ7½6Ô;KÈJÑ;WÔ;WÑ3XÔ3XÑ(YÔ(Y å'/Ô'BÈ&ÐY^Ð'_Ñ'_Ô'_ Ø&*¤i à %§
¢
¨4¬9Ñ 5Ô 5Ð 5Ð 5à&-¨kÔ&:¸5Ô&A à*¨9Ô5×<Ò<ÀgÐVYÐ=ZÐ=ZÑ[Ô[Ð[ùØ×#Ò# NÑ3Ô3Ð3Ñ3ØÐrV   c                 óª  ‡ ‡‡‡‡‡— ‰ j         j        �‰ j         j        |d<   |d         }|d         d         dk    rdS ‰                      |d         ¦  «        Š‰‰ j        k    }‰‰ _        ‰                      ‰¦  «        \  Š}‰                      ‰¦  «        }‰                      ||¦  «        }dŠt          D ],}|‰j        j	        d          
                    ¦   «         v r|Š nŒ-|                     |d|                     d	¦  «        d
dd¬¦  «        }|                     ‰j        ¦  «        }|                     dd¦  «        Šd}	d‰j        j	        d          
                    ¦   «         v rd}	t          ||	d¬¦  «        }
t!          |‰j        ¬¦  «        }d}‰                      |¦  «        r	|s‰ j        }i |¥|
|d|dœ¥Šˆˆˆˆˆ ˆfd„} ||
‰¦  «        S )a  
        Generates an OpenAI Chat Completion using `generate`.

        Args:
            req (`dict`): The request to generate an OpenAI Chat Completion for.

        Returns:
            `Generator[str, None, None]`: A generator that yields the OpenAI Chat Completion chunks.
        NrÀ   rš  éÿÿÿÿr7  rŸ  r   TÚtoolsr›  )r�  rÚ  rœ  Úreturn_dictÚtokenizer5  Úreq_0ÚgptossF©Úskip_special_tokensÚskip_prompt©r�   )ÚstreamerrM   Úreturn_dict_in_generateÚpast_key_valuesc              3   óD  •K  — d}d }d‰j         j        d                              ¦   «         v rd}d}ˆˆfd„}t          |‰¬¦  «        }d}	 |                     ¦   «          t          ¦   «         }‰                     ‰d	‰¬
¦  «        V — | D �]e}d‰j         j        d                              ¦   «         v r-|                     d¦  «        r|d t          d¦  «         …         }||z  }|r||v rd}ŒdŒe‰��Ý| 	                    ¦   «         t          ‰         d         k    rd|_        Œ™| 	                    ¦   «         t          ‰         d         k    r0|                     ¦   «          ‰                     |d d‰¬¦  «        V — Œò|j        �rK|xj        |z  c_        |j        s_t          j        d|j        ¦  «        }	|	€�Œ/|	                     d¦  «        }	d|_        t%          t'          |	¬¦  «        dd|dz   ¬¦  «        }
n·|dk    r�Œxd|j        vr�Œƒ|xj        |                     d¦  «        z  c_        |xj        |                     d¦  «        z  c_        |j        dk     r3d                     |                     d¦  «        d d…         ¦  «        dz   }t%          t'          |¬¦  «        dd¬¦  «        }
‰                     |d |
g‰¬¦  «        V — �ŒE|dk    r‰                     ||‰¬¦  «        V — �Œg‰                     |d‰¬¦  «        V — |                     ¦   «          nS# t0          $ rF}t2                               t7          |¦  «        ¦  «         d t7          |¦  «        › d!�V — Y d }~nd }~ww xY w|                     ¦   «          d S # |                     ¦   «          w xY w)"NFrÞ  r   Tú<|channel|>final<|message|>c                  ó:   •—  ‰j         di | ¤Ž}|j        ‰_        d S ©NrU   ©Úgeneraterå  r  ©r«   Úgenerate_outputrÀ   r¶   s     €€rW   Úgenerate_with_cachezbServeCommand.generate_chat_completion.<locals>.stream_chat_completion.<locals>.generate_with_cacheÔ  ó+   ø€ Ø"0 %¤.Ð":Ð":°6Ð":Ð":�Ø%4Ô%D�Ô"Ð"Ð"rV   ©Útargetr«   r¹   rŸ  r   ú
<|return|>rƒ   r„   r9  )r5  r7  r8  rÀ   z\"name\": \"(.*?)\"r!   )rÐ  ÚfunctionÚ
_tool_call)ró  r<  r¿  r=  z"arguments": {Ú{Ú}éþÿÿÿ)Ú	arguments)ró  r<  r¿  )r5  r7  r9  rÀ   )r6  rÀ   rp   r¡  r¢  r£  )ÚconfigÚarchitecturesr
  r
   rƒ   r±   rH  ÚendswithÚlenÚstripÚ_TOOL_CALL_TOKENSrº   r´   r½   r»   rÊ  ÚsearchÚgroupr5   r6   r¼   ÚcountrÉ  r‚  r¨  rÔ   r  rS   )rã  Ú_request_idÚ
filter_cotÚcot_trace_endrî  ÚthreadÚresultsÚ
tool_stater«  Ú	tool_nameÚtoolr$  Úgeneration_kwargsrÀ   r¬  r5  r¶   Útool_model_familys               €€€€€€rW   r­  zEServeCommand.generate_chat_completion.<locals>.stream_chat_completionÊ  su  øè è € ð ˆJØ ˆMØ˜5œ<Ô5°aÔ8×>Ò>Ñ@Ô@Ð@Ð@Ø!�
Ø =�ðEð Eð Eð Eð Eð Eõ Ð#6Ð?PÐQÑQÔQˆFØˆGðhØ—’‘”�Ý&™[œ[�
ð ×6Ò6°zÈÐ[pÐ6ÑqÔqÐqÐqÐqà&ð Wñ W�Fà 5¤<Ô#=¸aÔ#@×#FÒ#FÑ#HÔ#HÐHÐHØ!Ÿ?š?¨<Ñ8Ô8ð BØ%+Ð,@­s°<Ñ/@Ô/@Ð.@Ð,@Ô%A˜FØ˜vÑ%�Gð "ð %Ø(¨GÐ3Ð3Ø).˜JØ$à$ð )Ñ4à!Ÿ<š<™>œ>Õ->Ð?PÔ-QÐRYÔ-ZÒZÐZØ:>˜JÔ7Ø$ð "Ÿ<š<™>œ>Õ->Ð?PÔ-QÐRWÔ-XÒXÐXØ&×,Ò,Ñ.Ô.Ð.Ø"&×"BÒ"BØ+6Ø%)Ø.:Ø&;ð	 #Cñ #ô #ð ð ð ð %à%Ô6ñ .%Ø&Ð-Ô-°Ñ7Ð-Ô-ð $.Ô#Cð $"Ý,.¬IÐ6LÈjÔN_Ñ,`Ô,` 	Ø#,Ð#4Ù$,à09·²ÀÑ0BÔ0B IØCG 
Ô @Ý':Ý-HÈiÐ-XÑ-XÔ-XØ*+Ø)3Ø'2°\Ñ'Að	("ñ ("ô ("  ð $*¨R¢< <Ù$,ð $4¸:Ô;LÐ#LÐ#LÙ$,ð !+Ð <Ô <ÀÇÂÈSÑ@QÔ@QÑ QÐ <Ô <Ø *Ð <Ô <ÀÇÂÈSÑ@QÔ@QÑ QÐ <Ô <Ø#-Ô#?À!Ò#CÐ#CØ-/¯WªW°V·\²\À#Ñ5FÔ5FÀsÈÀsÔ5KÑ-LÔ-LÈsÑ-R Få':Ý-HÐSYÐ-ZÑ-ZÔ-ZØ*+Ø)3ð("ñ ("ô (" ð #'×"BÒ"BØ+6¸TÈtÈfÐ\qð #Cñ #ô #ð ð ð ñ %ð  ’|�|Ø"×>Ò>Ø'°Ð?Tð ?ñ ô ð ð ð ùð ×6Ò6°{ÐRXÐ`uÐ6ÑvÔvÐvÐvÐvà—’‘”��øÝð 7ð 7ð 7Ý—’�S ™VœVÑ$Ô$Ð$Ø6­3¨q©6¬6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6Ð6øøøøð7øøøð
 —’‘”���ø�—’‘”��øøøs1   ÁKL  ÌN	 Ì 
M0Ì*<M+Í&N	 Í+M0Í0N	 Î	N)r‹   rô   r®  r  r¯  r¼  r×  Ú_MODELS_WITH_TOOL_SUPPORTrù  rú  r
  r²  rš   r³  rÝ   r   r¯   rM   Úis_continuationr  )r¶   r�   rš  r´  rÂ   r»  rÒ  Úsupported_model_familiesrµ  rà  Úgeneration_streamerrM   r  r­  r
  rÀ   r¬  r5  r  s   `             @@@@@rW   rY  z%ServeCommand.generate_chat_completion  sq  øøøøøø€ ð Œ9Ô Ð,Øœ9Ô0ˆC�‰Là9<¸Z¼ˆð �BŒ<˜Ô ;Ò.Ð.ØˆFà $× 7Ò 7¸¸G¼Ñ EÔ EÐØ2°d´oÒEÐà/ˆŒØ×8Ò8Ð9NÑOÔOÑˆˆyà×*Ò*¨5Ñ1Ô1ˆØ×JÒJÈ8ÐU]Ñ^Ô^Ðð !ÐÝ(Að 	ð 	Ð$Ø'¨5¬<Ô+EÀaÔ+H×+NÒ+NÑ+PÔ+PÐPÐPØ$<Ð!Ø�ð Qð ×.Ò.ØØ"&Ø—'’'˜'Ñ"Ô"ØØØð /ñ 
ô 
ˆð —’˜5œ<Ñ(Ô(ˆØ—W’W˜\¨7Ñ3Ô3ˆ
ð #ÐØ�u”|Ô1°!Ô4×:Ò:Ñ<Ô<Ð<Ð<Ø"'Ðå2ØØ 3Øð
ñ 
ô 
Ðõ
 >¸cÐ[`Ô[rÐsÑsÔsÐàˆØ×Ò Ñ$Ô$ð 	/Ð-?ð 	/Ø Ô.ˆMð
Øð
à+Ø!2Ø'+Ø,ð
ð 
ð 
Ððy	ð y	ð y	ð y	ð y	ð y	ð y	ð y	ð y	ð y	ðv &Ð%Ð&9¸:ÑFÔFÐFrV   c                 ó”  ‡ ‡‡
‡‡‡— ‰                       ‰d         ¦  «        Š‰‰ j        k    }‰‰ _        ‰                      ‰¦  «        \  Š}t          ‰d         t          ¦  «        r1d‰v rd‰d         dœgng }|                     d‰d         dœ¦  «         nÇt          ‰d         t          ¦  «        rTd‰v rG‰d         d         d         dk    rd‰d         dœg‰d         ¢}n{‰d         }‰d         |d         d	<   na‰d         }nXt          ‰d         t          ¦  «        r.d‰v rd‰d         dœgng }|                     ‰d         ¦  «         nt          d
¦  «        ‚| 	                    |dd¬¦  «        }| 
                    ‰j        ¦  «        }‰                     dd¦  «        Šd}d‰j        j        d                              ¦   «         v rd}t!          ||d¬¦  «        }t#          ‰‰j        ¬¦  «        }d}‰                      ‰¦  «        r	|s‰ j        }|t+          j        |¦  «        ||d|dœŠ
ˆ
ˆˆˆˆˆ fd„}	 |	|‰¦  «        S )a	  
        Generates an OpenAI Response using `generate`.

        Args:
            req (`dict`): The request to generate an OpenAI Response for.

        Returns:
            `Generator[str, None, None]`: A generator that yields the OpenAI Response events.
        rÀ   ÚinputÚinstructionsÚsystemr¾  rc   r   r7  r6  z%inputs should be a list, dict, or strTr›  )r�  rœ  rn   rÝ  rÞ  Frß  râ  N)rµ  Úattention_maskrã  rM   rä  rå  c              3   óÖ
  •K  — d}d }d‰j         j        d                              ¦   «         v rd}d}ˆˆfd„}t          |‰¬¦  «        }d}d}d}	 |                     ¦   «          t          j        ¦   «         }	t          d|t          d	‰› �|	d
‰‰                     d¦  «        dddiidg g ‰                     dd¦  «        d‰                     d¦  «        ¬¦  «        ¬¦  «        }
|dz  }‰ 	                    |
¦  «        V — t          d|t          d	‰› �|	d‰‰                     d¦  «        dddiidg g ‰                     dd¦  «        d‰                     d¦  «        ¬¦  «        ¬¦  «        }|dz  }‰ 	                    |¦  «        V — t          d||t          d‰› �dddg ¬¦  «        ¬¦  «        }|dz  }‰ 	                    |¦  «        V — t          dd‰› �|||t          dd g ¬!¦  «        ¬"¦  «        }|dz  }‰ 	                    |¦  «        V — d }| D ]Ÿ}d‰j         j        d                              ¦   «         v r-|                     d#¦  «        r|d t!          d#¦  «         …         }||z  }|r
||v rd}d }ŒeŒft#          d$d‰› �||||d d%d&œg¬'¦  «        }|dz  }‰ 	                    |¦  «        V — Œ t%          d(d‰› �||d|d d%d&œg¬)¦  «        }|dz  }‰ 	                    |¦  «        V — t'          d*d‰› �|||t          d|j        g ¬!¦  «        ¬"¦  «        }|dz  }|dz  }‰ 	                    |¦  «        V — t+          d+||t          d‰› �dd,d|j        gg ¬-¦  «        ¬¦  «        }|dz  }|dz  }‰ 	                    |¦  «        V — t/          d.|t          d	‰› �|	d,‰‰                     d¦  «        dddii|j        gdg ‰                     dd¦  «        d‰                     d¦  «        ¬/¦  «        ¬¦  «        }|dz  }‰ 	                    |¦  «        V — |                     ¦   «          �n# t4          $ �r }t6                               d0t;          |¦  «        › �¦  «         t=          d1|t;          |¦  «        ¬2¦  «        }|dz  }‰ 	                    |¦  «        V — t?          d3|t          d	‰› �|	d4‰‰                     d¦  «        dddiig dg dd‰                     d¦  «        tA          d5t;          |¦  «        ¬6¦  «        ¬7¦  «        ¬¦  «        }|dz  }‰ 	                    |¦  «        V — Y d }~nd }~ww xY w|                     ¦   «          d S # |                     ¦   «          w xY w)8NFrÞ  r   Trç  c                  ó:   •—  ‰j         di | ¤Ž}|j        ‰_        d S ré  rê  rì  s     €€rW   rî  zTServeCommand.generate_response.<locals>.stream_response.<locals>.generate_with_cache�  rï  rV   rð  zresponse.createdÚresp_Úqueuedr  Úformatr¿  rb   rI  r}   rÚ   rs   )r=  rˆ  r¥  rÀ   r  rb   r@  rÚ  r[  r}   rj   rs   )r¿  Úsequence_numberrI  r!   zresponse.in_progressÚin_progresszresponse.output_item.addedÚmsg_rÓ  rŸ  )r=  r¿  r¥  r7  r6  )r¿  r  Úoutput_indexÚitemzresponse.content_part.addedÚoutput_textr¹   )r¿  rb   Úannotations)r¿  Úitem_idr  r  Úcontent_indexÚpartrò  zresponse.output_text.deltagš™™™™ùX@)ÚtokenÚlogprob)r¿  r!  r  r  r"  r;  rr   zresponse.output_text.done)r¿  r!  r  r  r"  rb   rr   zresponse.content_part.donezresponse.output_item.doneÚ	completed)r=  r¿  r¥  r7  r6  r   zresponse.completed)r=  rˆ  r¥  rÀ   r  rb   r[  r@  rÚ  r}   rj   rs   z"Exception in response generation: r  )r¿  r  rÓ  zresponse.failedÚfailedÚserver_error)ÚcoderÓ  )r=  rˆ  r¥  rÀ   r  rb   r[  r@  rÚ  r}   rj   rs   r  )!rù  rú  r
  r
   rƒ   rE  r<   r8   rš   rK  r@   rA   rC   r:   rD   rû  rü  rE   rF   r;   rb   rB   r#  r9   r  rÉ  r¨  rÔ   r  rS   r>   r?   r=   )rã  r  r  r  rî  r  r  r  r"  rˆ  Úresponse_createdÚresponse_in_progressÚresponse_output_item_addedÚresponse_content_part_addedr  r«  Úresponse_output_text_deltaÚresponse_output_text_doneÚresponse_content_part_doneÚresponse_output_item_doneÚresponse_completedr$  Úerror_eventÚresponse_failedr
  rÀ   r¬  r�   r5  r¶   s                           €€€€€€rW   Ústream_responsez7ServeCommand.generate_response.<locals>.stream_response†  s‹  øè è € ð ˆJØ ˆMØ˜5œ<Ô5°aÔ8×>Ò>Ñ@Ô@Ð@Ð@Ø!�
Ø =�ðEð Eð Eð Eð Eð Eõ Ð#6Ð?PÐQÑQÔQˆFØˆOØˆLØˆMðNØ—’‘”�Ý!œY™[œ[�
õ $8Ø+Ø$3Ý%Ø/ :Ð/Ð/Ø#-Ø'Ø3Ø%(§W¢W¨^Ñ%<Ô%<Ø&¨°Ð(8Ð9Ø)Ø Ø!Ø,/¯GªGÐ4IÈ5Ñ,QÔ,QØ$*Ø!$§¢¨Ñ!4Ô!4ðñ ô ð$ñ $ô $Ð ð$   1Ñ$�Ø×/Ò/Ð0@ÑAÔAÐAÐAÐAå'>Ø/Ø$3Ý%Ø/ :Ð/Ð/Ø#-Ø,Ø3Ø%(§W¢W¨^Ñ%<Ô%<Ø&¨°Ð(8Ð9Ø)Ø Ø!Ø,/¯GªGÐ4IÈ5Ñ,QÔ,QØ$*Ø!$§¢¨Ñ!4Ô!4ðñ ô ð(ñ (ô (Ð$ð$   1Ñ$�Ø×/Ò/Ð0DÑEÔEÐEÐEÐEõ .JØ5Ø$3Ø!-Ý.Ø. *Ð.Ð.°YÀ}Ð[fÐprðñ ô ð	.ñ .ô .Ð*ð   1Ñ$�Ø×/Ò/Ð0JÑKÔKÐKÐKÐKõ /LØ6Ø/ :Ð/Ð/Ø$3Ø!-Ø"/Ý+°ÀRÐUWÐXÑXÔXð/ñ /ô /Ð+ð   1Ñ$�Ø×/Ò/Ð0KÑLÔLÐLÐLÐLð �Ø&ð Pð P�Fà 5¤<Ô#=¸aÔ#@×#FÒ#FÑ#HÔ#HÐHÐHØ!Ÿ?š?¨<Ñ8Ô8ð BØ%+Ð,@­s°<Ñ/@Ô/@Ð.@Ð,@Ô%A˜FØ˜vÑ%�Gð "ð %Ø(¨GÐ3Ð3Ø).˜JØ&(˜GØ$à$å1GØ9Ø 3 zÐ 3Ð 3Ø(7Ø%1Ø&3Ø$Ø,.¸4Ð"@Ð"@Ð!Að2ñ 2ô 2Ð.ð $ qÑ(�OØ×3Ò3Ð4NÑOÔOÐOÐOÐOÐOõ -BØ4Ø/ :Ð/Ð/Ø$3Ø!-Ø"#Ø Ø(*°tÐ<Ð<Ð=ð-ñ -ô -Ð)ð   1Ñ$�Ø×/Ò/Ð0IÑJÔJÐJÐJÐJõ .JØ5Ø/ :Ð/Ð/Ø$3Ø!-Ø"/Ý+°ÐE^ÔEcÐqsÐtÑtÔtð.ñ .ô .Ð*ð   1Ñ$�Ø Ñ"�Ø×/Ò/Ð0JÑKÔKÐKÐKÐKõ -HØ4Ø$3Ø!-Ý.Ø. *Ð.Ð.Ø&Ø*Ø(Ø!;Ô!@Ð AØ$&ðñ ô ð	-ñ -ô -Ð)ð   1Ñ$�Ø Ñ!�Ø×/Ò/Ð0IÑJÔJÐJÐJÐJõ &<Ø-Ø$3Ý%Ø/ :Ð/Ð/Ø#-Ø*Ø3Ø%(§W¢W¨^Ñ%<Ô%<Ø&¨°Ð(8Ð9Ø 9Ô >Ð?Ø)Ø Ø,/¯GªGÐ4IÈ5Ñ,QÔ,QØ$*Ø!$§¢¨Ñ!4Ô!4ðñ ô ð&ñ &ô &Ð"ð$   1Ñ$�Ø×/Ò/Ð0BÑCÔCÐCÐCÐCà—’‘”�‘øÝð !Añ !Að !AÝ—’ÐJÅ#ÀaÁ&Ä&ÐJÐJÑKÔKÐKÝ0Ø Ø$3Ý ™FœFðñ ô �ð
   1Ñ$�Ø×/Ò/°Ñ<Ô<Ð<Ð<Ð<å"5Ø*Ø$3Ý%Ø/ :Ð/Ð/Ø#-Ø'Ø3Ø%(§W¢W¨^Ñ%<Ô%<Ø&¨°Ð(8Ð9Ø!Ø)Ø Ø,1Ø$*Ø!$§¢¨Ñ!4Ô!4Ý+Ø!/Ý$'¨¡F¤Fðñ ô ðñ ô ð#ñ #ô #�ð,   1Ñ$�Ø×/Ò/°Ñ@Ô@Ð@Ð@Ð@Ð@Ð@Ð@Ð@Ð@øøøøðC!AøøøðH —’‘”���ø�—’‘”��øøøs2   ÁOP. Ð,U Ð.T9Ð9C6T4Ô/U Ô4T9Ô9U ÕU()r®  r  r¯  rÇ  rS   rÈ  rl  rs  r¹  r²  r³  rÝ   rš   rù  rú  r
  r   r¯   rM   r  r  r©   Ú	ones_like)r¶   r�   r´  rÂ   rµ  rà  r  rM   r  r5  r
  rÀ   r¬  r5  s   ``        @@@@rW   r^  zServeCommand.generate_responseG  sé  øøøøøø€ ð !%× 7Ò 7¸¸G¼Ñ EÔ EÐØ2°d´oÒEÐØ/ˆŒØ×8Ò8Ð9NÑOÔOÑˆˆyå�c˜'”l¥CÑ(Ô(ð 	FØM[Ð_bÐMbÐMb˜x°C¸Ô4GÐHÐHÐIÐIÐhjˆFØ�MŠM 6°c¸'´lÐCÐCÑDÔDÐDÐDÝ˜˜Gœ¥dÑ+Ô+ð 	FØ Ð$Ð$Ø�w”< ”? 6Ô*¨hÒ6Ð6Ø'/¸CÀÔ<OÐPÐPÐ`ÐSVÐW^ÔS_Ð`�F�Fà  œ\�FØ+.¨~Ô+>�F˜1”I˜iÑ(Ð(à˜Wœ��Ý˜˜Gœ¥dÑ+Ô+ð 	FØM[Ð_bÐMbÐMb˜x°C¸Ô4GÐHÐHÐIÐIÐhjˆFØ�MŠM˜#˜gœ,Ñ'Ô'Ð'Ð'åÐDÑEÔEÐEà×.Ò.¨vÈTÐbfÐ.ÑgÔgˆØ—’˜5œ<Ñ(Ô(ˆØ—W’WÐ3°WÑ=Ô=ˆ
ð #ÐØ�u”|Ô1°!Ô4×:Ò:Ñ<Ô<Ð<Ð<Ø"'Ðå2ØØ 3Øð
ñ 
ô 
Ðõ
 >¸cÐ[`Ô[rÐsÑsÔsÐàˆØ×Ò Ñ$Ô$ð 	/Ð-?ð 	/Ø Ô.ˆMð Ý#œo¨fÑ5Ô5Ø+Ø!2Ø'+Ø,ð
ð 
Ðða	ð a	ð a	ð a	ð a	ð a	ð a	ð a	ð a	ð a	ðF ˆÐ2°JÑ?Ô?Ð?rV   c                 óF  ‡
‡‡‡— t          ¦   «         st          d¦  «        ‚|                      |d         ¦  «        }|                      |¦  «        \  ŠŠt	          ‰j        dd¬¦  «        }t          |‰j        ¬¦  «        }‰j        j	        }t          j        |d         ¦  «        }t          j        ||d¬¦  «        \  }} ‰||d¬	¦  «                             ‰j        ¦  «        Š
‰
d
                              ‰j        ¦  «        ‰
d
<   ||ddœŠˆ
ˆˆˆfd„}	 |	¦   «         S )a  
        Generates an OpenAI Transcription using the audio file.

        Args:
            req (`dict`): The request containing the audio file and model information.

        Returns:
            `Generator[str, None, None]`: A generator that yields the transcription result.
        z]Missing librosa dependency for audio transcription. Please install with `pip install librosa`rÀ   Trß  râ  r^   )ÚsrÚmonor›  )Úsampling_raterœ  Úinput_features)rã  rM   rä  c               3   óÀ   •K  —  ‰j         di ‰¤‰¤Ž} ‰                     | j        d¬¦  «        d         }t          |¬¦  «        }|                     d¬¦  «        › V — d S )NT)rà  r   )rb   rB  rU   )rë  Úbatch_decodeÚ	sequencesr/   rF  )Úgenerated_idsÚtranscription_textÚtranscriptionÚaudio_inputsÚaudio_modelÚaudio_processorr
  s      €€€€rW   Ú_generate_transcriptionzDServeCommand.generate_transcription.<locals>._generate_transcription“  s   øè è € Ø0˜KÔ0ÐUÐU°<ÐUÐCTÐUÐUˆMØ!0×!=Ò!=¸mÔ>UÐkoÐ!=Ñ!pÔ!pÐqrÔ!sÐÝ)Ð/AÐBÑBÔBˆMØ"×2Ò2ÀÐ2ÑEÔEÐGÐGÐGÐGÐGÐGrV   )r   r  r®  Úload_audio_model_and_processorr   r�  r¯   rM   Úfeature_extractorr:  Úior	   ÚlibrosaÚloadr³  rÝ   rã   )r¶   r�   r¬  r  rM   Úmodel_sampling_rateÚaudio_bytesÚaudio_arrayÚ_rE  rB  rC  rD  r
  s             @@@@rW   rh  z#ServeCommand.generate_transcriptionk  s„  øøøø€ õ $Ñ%Ô%ð 	ÝØoñô ð ð !%× 7Ò 7¸¸G¼Ñ EÔ EÐØ'+×'JÒ'JÐK`Ñ'aÔ'aÑ$ˆ�_å2ØÔ%¸4ÈTð
ñ 
ô 
Ðõ >Ø¨Ô)Fð
ñ 
ô 
Ðð
 .Ô?ÔMÐÝ”j  V¤Ñ-Ô-ˆÝ œ kÐ6IÐPTÐUÑUÔU‰ˆ�QØ&� {ÐBUÐfjÐkÑkÔk×nÒnØÔñ
ô 
ˆð *6Ð6FÔ)G×)JÒ)JÈ;ÔK\Ñ)]Ô)]ˆÐ%Ñ&ð ,Ø!2Ø'+ð
ð 
Ðð	Hð 	Hð 	Hð 	Hð 	Hð 	Hð 	Hð 	Hð 'Ð&Ñ(Ô(Ð(rV   c                 óN  — |                      d¦  «        p|                      d¦  «        }d}| j        €d}ngt          | j        ¦  «        t          |¦  «        k    rd}n?t          t          | j        ¦  «        ¦  «        D ]}| j        |         ||         k    rd} nŒ|| _        |S )aD  
        Determines whether the current request is a continuation of the last request. In other words, if it is the
        same chat session.

        Args:
            req (`dict`): The request to check.

        Returns:
            `True` if the request is a continuation of the last request, `False` otherwise.
        rš  r  TNF)rš   r  rü  Úrange)r¶   r�   rš  Úreq_continues_last_messagesÚis        rW   r  zServeCommand.is_continuation›  sÀ   € ð —7’7˜:Ñ&Ô&Ð:¨#¯'ª'°'Ñ*:Ô*:ˆØ&*Ð#ð ÔÐ%Ø*/Ð'Ð'å�Ô#Ñ$Ô$­¨H©¬Ò5Ð5Ø*/Ð'Ð'õ �3˜tÔ1Ñ2Ô2Ñ3Ô3ð ð �ØÔ% aÔ(¨H°Q¬KÒ7Ð7Ø27Ð/Ø�Eð 8ð &ˆÔØ*Ð*rV   r$   c                 óš   — | j         r)t          d| j        | j        | j        | j        ¬¦  «        }n| j        rt          d¬¦  «        }nd}|S )a  
        Returns the quantization config for the given CLI arguments.

        Args:
            args (`ServeArguments`): The serve arguments. May contain quantization settings, device, etc.

        Returns:
            `Optional[BitsAndBytesConfig]`: The quantization config.
        T)rç   Úbnb_4bit_compute_dtyperê   Úbnb_4bit_use_double_quantÚbnb_4bit_quant_storage)ræ   N)rç   r$   rã   rê   rë   ræ   )r‹   Úquantization_configs     rW   Úget_quantization_configz$ServeCommand.get_quantization_config¹  sv   € ð Ôð 	'Ý"4Ø!à'+¤zØ$(Ô$<Ø*.Ô*CØ'+¤zð#ñ #ô #ÐÐð Ôð 	'Ý"4Ø!ð#ñ #ô #ÐÐð #'Ðà"Ð"rV   Úmodel_idc                 óH   — | j         j        �| j         j        }d|v r|S |› d�S )aR  
        Applies the `force_model` CLI argument and canonicalizes the model name to the format "model_id@revision".
        If the model_id DOESN'T contain an @, it defaults to "model_id@main".

        Args:
            model_id (`str`): The model ID.

        Returns:
            `str`: The canonicalized model name to be used
        Nú@z@main)r‹   rô   )r¶   rY  s     rW   r®  zServeCommand.process_model_nameÖ  s6   € ð Œ9Ô Ð,Ø”yÔ,ˆHØ�(ˆ?ˆ?ØˆOØÐ!Ð!Ð!Ð!rV   r¬  c                 ó&  — | j         }t                               d|› �¦  «         d|v r|                     dd¦  «        \  }}n|d}}t	          j        |||j        ¬¦  «        }|j        dv r|j        nt          t          |j        ¦  «        }|  
                    |¦  «        }||j        |d|j        dœ}|�||d
<   t          j        |fi |¤Ž}	t          t          |	j        d         ¦  «        }
 |
j        |fi |¤Ž}t          |dd	¦  «        €|                     |j        ¦  «        }|j        j        d	u o|j        j        dk    }|j        j        d	uo|j        j        dk     }|s|rd|j        _        t                               d|› �¦  «         ||fS )a  
        Generic method to load a model and a data processor from a model ID and revision, making use of the serve CLI
        arguments.

        Args:
            model_id_and_revision (`str`):
                The model ID and revision to load.
            model_cls (`type[PreTrainedModel]`):
                The model class to load.

        Returns:
            `tuple[PreTrainedModel, Union[ProcessorMixin, PreTrainedTokenizerFast]]`: The loaded model and
            data processor (tokenizer, audio processor, etc.).
        zLoading r[  r!   Úmain)Úrevisionrä   )rÚ   NrÚ   )r^  rå   rã   Ú
device_maprä   NrW  r   Úhf_device_mapé   rb  zLoaded model )r‹   rÔ   rÕ   r‚  r#   Úfrom_pretrainedrä   rã   Úgetattrr©   rX  rå   r   r  rú  r³  rÝ   rM   r£   Ú
max_length)r¶   r¬  r‹   rY  r^  Údata_processorrã   rW  Úmodel_kwargsrù  ÚarchitecturerÀ   Úhas_default_max_lengthÚhas_short_max_new_tokenss                 rW   Ú_load_model_and_data_processorz+ServeCommand._load_model_and_data_processorç  sè  € ð ŒyˆÝ�ŠÐ6Ð4Ð6Ð6Ñ7Ô7Ð7àÐ'Ð'Ð'Ø!6×!<Ò!<¸SÀ!Ñ!DÔ!DÑˆH�h�hà!6¸�hˆHå&Ô6ØØØ"Ô4ð
ñ 
ô 
ˆð #œj¨NÐ:Ð:�”
�
ÅÍÈtÌzÑ@ZÔ@ZˆØ"×:Ò:¸4Ñ@Ô@Ðð !Ø#'Ô#;ØØ Ø!%Ô!7ð
ð 
ˆð Ð*Ø2EˆLÐ.Ñ/åÔ+¨HÐEÐE¸ÐEÐEˆÝ�|¨VÔ-AÀ!Ô-DÑEÔEˆØ,�Ô,¨XÐFÐF¸ÐFÐFˆå�5˜/¨4Ñ0Ô0Ð8Ø—H’H˜Tœ[Ñ)Ô)ˆEð Ô#Ô2°dÐ:Ðg¸uÔ?VÔ?aÐegÒ?gð 	ð Ô#Ô2¸$Ð>ÐpÀ5ÔCZÔCiÐlpÒCpð 	!ð "ð 	:Ð%=ð 	:Ø59ˆEÔ#Ô2å�ŠÐ;Ð$9Ð;Ð;Ñ<Ô<Ð<Ø�nÐ$Ð$rV   c                 óZ  — || j         vs| j         |                              ¦   «         r=|                      |¦  «        \  }}t          || j        j        |¬¦  «        | j         |<   nC| j         |                              ¦   «          | j         |         j        }| j         |         j        }||fS )a\  
        Loads the text model and processor from the given model ID and revision into the ServeCommand instance.

        Args:
            model_id_and_revision (`str`):
                The model ID and revision to load.

        Returns:
            `tuple[PreTrainedModel, PreTrainedTokenizerFast]`: The loaded text model and processor.
        ©rÁ   rÂ   ©	r  r×   rj  r¿   r‹   rï   rÌ   rÀ   rÂ   )r¶   r¬  rÀ   rÂ   s       rW   r¯  z%ServeCommand.load_model_and_processor$  s½   € ð !¨Ô(:Ð:Ð:¸dÔ>PÐQfÔ>g×>rÒ>rÑ>tÔ>tÐ:Ø#×BÒBÐCXÑYÔYÑˆE�9Ý8BØØ $¤	Ô 7Ø#ð9ñ 9ô 9ˆDÔÐ4Ñ5Ð5ð ÔÐ4Ô5×AÒAÑCÔCÐCØÔ&Ð'<Ô=ÔCˆEØÔ*Ð+@ÔAÔKˆIà�iÐÐrV   c                 óZ  — || j         vs| j         |                              ¦   «         r=|                      |¦  «        \  }}t          || j        j        |¬¦  «        | j         |<   nC| j         |                              ¦   «          | j         |         j        }| j         |         j        }||fS )aU  
        Loads the audio model and processor from the given model ID and revision into the ServeCommand instance.

        Args:
            model_id_and_revision (`str`):
                The model ID and revision to load.

        Returns:
            `tuple[PreTrainedModel, ProcessorMixin]`: The loaded audio model and processor.
        rl  rm  )r¶   r¬  rC  rD  s       rW   rF  z+ServeCommand.load_audio_model_and_processor?  s½   € ð !¨Ô(:Ð:Ð:¸dÔ>PÐQfÔ>g×>rÒ>rÑ>tÔ>tÐ:Ø+/×+NÒ+NÐOdÑ+eÔ+eÑ(ˆK˜Ý8BØØ $¤	Ô 7Ø)ð9ñ 9ô 9ˆDÔÐ4Ñ5Ð5ð ÔÐ4Ô5×AÒAÑCÔCÐCØÔ,Ð-BÔCÔIˆKØ"Ô0Ð1FÔGÔQˆOà˜OÐ+Ð+rV   )r¹   NNNNN)*rO   rP   rQ   Ústaticmethodr   r   rÙ   r·   rs  r  r&  r,  r0  r4  r   rS   rl  rH  rK  rw  Ú	functoolsÚ	lru_cacheÚanyrn  r   rX  r†   r¼  r×  rY  r^  rh  ra   r  rX  r®  rj  Útupler   r¯  r   rF  rU   rV   rW   r�   r�   Ã  s  € € € € € Øð	> Nð 	>ð 	>ð 	>ñ „\ð	>ð˜^ð ð ð ð ð@/àð/ð !ð/ð !ð	/ð
 ð/ð /ð /ð /ðb
°ð 
ð 
ð 
ð 
ð
¸ð 
ð 
ð 
ð 
ð
°dð 
ð 
ð 
ð 
ð %'Ø!%Ø#Ø"Ø'+Ø<@ð2Gð 2Gà˜S”Mð2Gð ˜#”ð2Gð ˜Œ}ð	2Gð
 �sŒmð2Gð   ”}ð2Gð ˜TÐ"7Ô8Ô9ð2Gð 
ð2Gð 2Gð 2Gð 2GðhJ¨[ð J¸Sð Jð Jð Jð Jð ?bð ?bð ?bðB €YÔ Ð&Ñ&Ô&ð+  T¨#¨s¨(¤^Ô 4ð +ð +ð +ñ 'Ô&ð+ðZW1°tð W1À	È#ÈtÐUYÈ/Ô@Zð W1ð W1ð W1ð W1ðr ð	Ð"3ð 	¸ð 	ð 	ð 	ñ „\ð	ð ð+ Àxð + ð + ð + ñ „\ð+ ðZFG¨Dð FG°Y¸sÀDÈ$¸Ô5Oð FGð FGð FGð FGðPb@ Tð b@¨i¸¸TÀ4¸Ô.Hð b@ð b@ð b@ð b@ðH	.)¨$ð .)°9¸SÀ$È¸_Ô3Mð .)ð .)ð .)ð .)ð`+ 4ð +¨Dð +ð +ð +ð +ð< ð# nð #¸ÐBVÔ9Wð #ð #ð #ñ „\ð#ð8"¨3ð "°3ð "ð "ð "ð "ð";%ÀCð ;%ð ;%ð ;%ð ;%ðz Ø%(ð à	Ð Ð"9Ð9Ô	:ð ð  ð  ð  ð6,ÀCð ,ÈEÐRcÐesÐRsÔLtð ,ð ,ð ,ð ,ð ,ð ,rV   r�   Ú__main__)‡rÂ  r�   r  Úenumrp  rÏ   rH  r›   rÊ  rÎ  rÆ   rE  Úargparser   r   Úcollections.abcr   r   Údataclassesr   r   r	   r
   Útypingr   r   Úhuggingface_hubr   Úhuggingface_hub.constantsr   r  Ú&transformers.models.auto.modeling_autor   r   Útransformers.utils.import_utilsr   r   r   r   r   r   r¹   r   r   r   r   r   Úutilsr   r    r"   r©   r#   r$   r%   r&   Úgeneration.continuous_batchingr'   r(   rI  ÚPILr)   r  rv  rt  r*   r+   Úfastapi.middleware.corsr,   Úfastapi.responsesr-   r.   Ú openai.types.audio.transcriptionr/   Ú.openai.types.audio.transcription_create_paramsr0   Úopenai.types.chatr1   Ú'openai.types.chat.chat_completion_chunkr2   r3   r4   r5   r6   Ú*openai.types.chat.completion_create_paramsr7   Úopenai.types.responsesr8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   Ú-openai.types.responses.response_create_paramsrG   ÚpydanticrH   rI   rJ   rL   r[   r]   r)  r.  r2  r*  r/  r3  r  rO   rÔ   rþ  rl  r  r  ÚEnumr†   rŽ   rs  r¯   r±   r¿   rÙ   r�   rú   rw  rU   rV   rW   ú<module>rŒ     s‘  ðð €€€Ø €€€Ø €€€Ø €€€Ø Ð Ð Ð Ø 	€	€	€	Ø 	€	€	€	Ø €€€Ø 	€	€	€	Ø €€€Ø Ð Ð Ð Ø €€€Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ø (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø "Ð "Ð "Ð "Ð "Ð "Ð "Ð "à &Ð &Ð &Ð &Ð &Ð &Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4à Ð Ð Ð ðð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 0Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ø (Ð (Ð (Ð (Ð (Ð (ð ÐÑÔð 
ZØ€L€L€Lðð ð ð ð ð ð ð ð ð ð ð ð ZÐYÐYÐYÐYÐYÐYÐYàÐÑÔð Ø€N€N€NàÐÑÔð ØÐÐÐÐÐð ÐÑÔÐkÐ 4Ð 4Ñ 6Ô 6ÐkÐ;OÐ;OÑ;QÔ;QÐkÐViÐViÑVkÔVkð ð  ð qØ€N€N€NØ.Ð.Ð.Ð.Ð.Ð.Ð.Ð.Ø6Ð6Ð6Ð6Ð6Ð6ØAÐAÐAÐAÐAÐAÐAÐAØ>Ð>Ð>Ð>Ð>Ð>Ø\Ð\Ð\Ð\Ð\Ð\Ø<Ð<Ð<Ð<Ð<Ð<ðð ð ð ð ð ð ð ð ð ð ð ð ð ð [ÐZÐZÐZÐZÐZðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð" \Ð[Ð[Ð[Ð[Ð[Ø@Ð@Ð@Ð@Ð@Ð@Ð@Ð@Ð@Ð@ðð ð ð ð Ð4QÐY^ð ñ ô ð ðð ð ð ð Ð6UÐ]bð ñ ô ð ð'ð 'ð 'ð 'ð 'Ð0MÐUZð 'ñ 'ô 'ð 'ð %˜Ð%NÑOÔOÐØ&˜;Ð'RÑSÔSÐØ)˜kÐ*OÑPÔPÐðð ð Ðð %ð %ð %Ð!ð.#ð #ð #Ðð 
ˆÔ	˜HÑ	%Ô	%€ð
 ØØðð ðÐ ð !˜DÐ!2×!7Ò!7Ñ!9Ô!9Ñ:Ô:Ð ðð ð ð ð ˆtŒyñ ô ð ð 	ð ð ð ð ð8Ø	ð8à/ð8ð ð	8ð 8ð 8ð 8ðvð ð ð ð ñ ô ð ð,@ð ,@ð ,@ð ,@ð ,@ñ ,@ô ,@ð ,@ð^ ðd*ð d*ð d*ð d*ð d*ñ d*ô d*ñ „ðd*ðNS,ð S,ð S,ð S,ð S,Ð-ñ S,ô S,ð S,ðl$ ˆzÒÐØˆL‰NŒN€EØ	‡I‚I�K„K€K€K€Kð ÐrV   