蘺h"dZddlmZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlmZdd lmZmZdd lmZmZdd l m!Z!m"Z"m#Z#dd l m$Z%dd l m&Z'ddl(m)Z)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6ddl7m8Z8m9Z9m:Z:m;Z;ddlm?Z?ddl@mAZAddlBmCZCddlDmEZEddlFmGZGddlHmIZImJZJmKZKddlLmMZNdd lOm$ZPddlOmQZQmRZRe rddlSZTejUeVZWeegeeEe!ffeeeXgefe!eEfZYeegeeEe!ffefZZGdde[Z\Gd d!eXZ]Gd"d#eXZ^ ddd+Z_dd.Z`dd0Zadd6Zbdd9Zcdd;ZdddCZeddIZfddKZgddMZhddWZiddYZjdd[Zkdddd\ddcZlddddddgZmddhddkZnddddddlZoddddddnZpddhddoZq dddvZrGdwdxeAdyzZsejtGd{d|Zudd~ZvddZwdZxdddddyddddZydddddyddddZzdZ{e{ez_e{|ddey_dS)z>Utilities for running language models or Chains over datasets.) annotationsN)datetimetimezone)Enum) TYPE_CHECKINGAnyCallableDictListOptionalTupleUnioncast)warn_deprecated)BaseLanguageModel) BaseMessagemessages_from_dict) ChatResult LLMResult)RunnableRunnableConfigRunnableLambdaconfig)utils)EvaluatorCallbackHandlerwait_for_all_evaluators)LangChainTracer)Client) get_git_info)EvaluationResult RunEvaluator) as_runnableis_traceable_function)DatasetDataTypeExample TracerSession)LangSmithError) HTTPError) TypedDict) Callbacks)Chain)load_evaluator) EvaluatorTypePairwiseStringEvaluatorStringEvaluator) evaluation)name_generationprogressceZdZdZdS)InputFormatErrorz(Raised when the input format is invalid.N)__name__ __module__ __qualname____doc__o/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/langchain/smith/evaluation/runner_utils.pyr6r6Hs2222r<r6c"eZdZdZddZddZdS) TestResultz1A dictionary of the results of a single test run.return pd.DataFramec|}d|jD}|d|dS)zReturn quantiles for the feedback scores. This method calculates and prints the quantiles for the feedback scores across all feedback keys. Returns: A DataFrame containing the quantiles for each feedback key. cg|]C}|ds*|ds|dA|DS)inputs.outputs. reference) startswith).0cols r= z5TestResult.get_aggregate_feedback.._sf   ~~i(( ~~j))  ~~k**     r<all)include)axis) to_dataframecolumnsdescribedrop)selfdfto_drops r=get_aggregate_feedbackz!TestResult.get_aggregate_feedbackRs^     z   {{5{))..wQ.???r<c  ddl}n"#t$r}td|d}~wwxYwg}g}|dD]g\}}|d}|d}t |t rd|D} n |i} nd|i} id|d D| } d |vr^t |d t r8| d |d Dn |d | d <| id |D|d |d|dd|| ||i|j||S)z#Convert the results to a dataframe.rNzfPandas is required to convert the results to a dataframe. to install pandas, run `pip install pandas`.resultsfeedbackoutputc i|] \}}d|| S)rEr;rHkvs r= z+TestResult.to_dataframe..xs$HHH1.Q..!HHHr<c i|] \}}d|| S)rDr;r\s r=r_z+TestResult.to_dataframe..s$HHH1=Q==!HHHr<inputrFc i|] \}}d|| S)z reference.r;r\s r=r_z+TestResult.to_dataframe..s'UUUA)a))1UUUr<c.i|]}d|j|jS)z feedback.)keyscore)rHfs r=r_z+TestResult.to_dataframe..s'FFF*15**AGFFFr<Errorexecution_timerun_id)errorrhri)index) pandas ImportErroritemsget isinstancedictupdateappend DataFrame) rSpdeindicesrecords example_idresultrYoutput_rZrs r=rOzTestResult.to_dataframehs/         @   "&y/"7"7"9"9 ' ' Jj)Hjj**G'4(( -HH HHH"G,HHw0E0E0G0GHHHAf$$f[14889HHUU 9L9R9R9T9TUUU&,K%8AkN HHFFXFFF#ZZ00&,-=&>$jj22     NN1    NN: & & & &r|G73333s &!&N)r@rA)r7r8r9r:rVrOr;r<r=r?r?OsF;;@@@@,,4,4,4,4,4,4r<r?c,eZdZdZd fd Zd d ZxZS) EvalErrorz"Your architecture raised an error.rg BaseExceptionkwargsrr@Nonec >tjdd|i|dS)Nrgr;)super__init__)rSrgr __class__s r=rzEvalError.__init__s+//u//////r<namestrcV ||S#t$rtd|dwxYw)Nz%'EvalError' object has no attribute '')KeyErrorAttributeError)rSrs r= __getattr__zEvalError.__getattr__sK R:  R R R !P!P!P!PQQ Q Rs ()rgrrrr@r)rrr@r)r7r8r9r:rr __classcell__)rs@r=r~r~s_,,000000RRRRRRRRr<r~ llm_or_chain_factoryMODEL_OR_CHAIN_FACTORY dataset_namerr@MCFc  t|trD|jj}|j*jjj}t d|d|d|dfdSt|t r|St|tr| fdSt|r>t|r'ttt| fdS |}nk#t$r^tt|}tj|}t d |d t%| fd cYSwxYwtt|t|t r|Sttt|r'ttt| fd St|tsfd SS|S)zForgive the user if they pass in a chain without memory instead of a chain factory. It's a common mistake. Raise a more helpful error message as well.Na$Cannot directly evaluate a chain with stateful memory. To evaluate this chain, pass in a chain constructor that initializes fresh memory each time it is called. This will safegaurd against information leakage between dataset examples. For example: def chain_constructor(): new_memory = z(...) return z*(memory=new_memory, ...) run_on_dataset("z", chain_constructor, ...)cSNr;)chainsr=z(_wrap_in_chain_factory..sur<cSrr;)lcfsr=rz(_wrap_in_chain_factory..ssr<cSrr; runnable_sr=rz(_wrap_in_chain_factory..9r<zWrapping function z as RunnableLambda.cSrr;)wrappedsr=rz(_wrap_in_chain_factory..s7r<cSrr;rsr=rz(_wrap_in_chain_factory..rr<c"tSr)r) constructorsr=rz(_wrap_in_chain_factory..s>+66r<)rpr-rr7memory ValueErrorrrcallabler$r#rr TypeErrorinspect signatureloggerinfor) rr chain_class memory_class_model user_funcsigrrrrrs @@@@@r=_wrap_in_chain_factoryrsO &..3$o.  & 2 <1:L L%1 L L* L L$0 L L L   }}} (*; < < ## (( 3 3"{{{ & ' ' !5 6 6 %#D3G$H$HIII$$$$ $ #))++FF # # #X';<z_get_prompt..s=> > #$Jq#  > > > > > > r<z,Expected list of strings for 'prompts', got rMc3@K|]}t|tVdSrrrs r=rz_get_prompt..s,.S.Saz!S/A/A.S.S.S.S.S.Sr<z)LLM Run expects string prompt input. Got z5LLM Run expects 'prompt' or 'prompts' in inputs. Got rz)LLM Run expects single prompt input. Got z prompts.) r6rprtyper7listrKlennextitervalues)rrprompt_s r= _get_promptrs ><===G6&*C00 "6)**366 (#$ f  &+T22 #> > (.y(9> > > ; ;  #;VI.//8;; # V  tFMMOO,,-- gs # # YiGG  & & Y3.S.S7.S.S.S+S+S YGG"#Wv#W#WXX X LF L L    7||qqz OG O O O   r<List[BaseMessage]cd|stdd|vr |d}nTt|dkr/tt|}ntd|t |t rtd|Dr|g}nCt |t rtd|Dr|}ntd|t|dkrt|dStd t|d |) zGet Chat Messages from inputs. Args: inputs: The input dictionary. Returns: A list of chat messages. Raises: InputFormatError: If the input format is invalid. rmessagesrMzPChat Run expects 'messages' in inputs when example has multiple input keys. Got c3@K|]}t|tVdSr)rprqrs r=rz _get_messages..+s=.. ! 1d......r<c3@K|]}t|tVdSr)rprrs r=rz _get_messages../s=00 ! 1d000000r<zUChat Run expects List[dict] or List[List[dict]] values for 'messages' key input. Got rzMChat Run expects single List[dict] or List[List[dict]] 'messages' input. Got z messages from inputs ) r6rrrrrprrKr)r single_input raw_messagess r= _get_messagesrs ><===Vj) V  D1122  ) & ) )   ,%%  #..%1...++  %~ L$ ' ' C00%1000-- $  3*0 3 3    <A!,q/222 M|,, M MDJ M M   r< first_exampler' input_mapperOptional[Callable[[Dict], Any]]rc|rz||j}t|tsQt|trt d|Ds%t d|dt |ddSdS t|jdS#t $r? t|jYdS#t $rt d|jdwxYwwxYw)Nc3@K|]}t|tVdSrrprrHmsgs r=rz>_validate_example_inputs_for_language_model..Js,IISJsK00IIIIIIr<zWhen using an input_mapper to prepare dataset example inputs for an LLM or chat model, the output must a single string or a list of chat messages. Got: of type .zvExample inputs do not match language model input format. Expected a dictionary with messages or a single prompt. Got: z Please update your dataset OR provide an input_mapper to convert the example.inputs to a compatible format for the llm or chat model you wish to evaluate.) rrprrrKr6rrr)rr prompt_inputs r=+_validate_example_inputs_for_language_modelrBs`#|M$899 ,,,  |T * * IILIIIII #G'GG26l1C1CGGG        , - - - - -    m2333333#   &G*1GGG  s>B CB66#CCrr-cZ|r||j}t|j|}t |t s#t d|dt|d|r,t d|jd|dS|j}t|j|}t|dkrt|jdkrdS|r,t d|jd|dS) z > R&0 #J HHHX]HL999)3It)D)DSioo) P!$PPP )(**E-7u-E-EO))4J/9%/G/GQ%++TK-   )1!)< FD!$ % % %$      r<rsmith_eval.RunEvalConfigrOptional[List[str]] Optional[str]cd}|jr/|j}|r%||vr!td|d|dnQ|rt|dkr |d}n3|1t|dkrtd|d|S)Nz Input key z% not in chain's specified input keys '. Evaluation behavior may be undefined.rMrz#Chain expects multiple input keys: z, Evaluator is likely to fail. Evaluation behavior may be undefined. Specify an input_key in the RunEvalConfig to avoid this warning.) input_keyrwarningr)rrrs r=_determine_input_keyrsI  $  ):55 NNSYSS)SSS     J1,,qM  C OOa$7$7 P* P P P   r<rcd}|jr/|j}|r%||vr!td|d|dnQ|rt|dkr |d}n3|1t|dkrtd|d|S)NzPrediction key z& not in chain's specified output keys rrMrz$Chain expects multiple output keys: zl, Evaluation behavior may be undefined. Specify a prediction_key in the RunEvalConfig to avoid this warning.)prediction_keyrrr)rrrs r=_determine_prediction_keyrsN  .  ><< NNU.UU +UUU     [))Q..$Q  S%5%5%9%9 ;; ; ; ;   r<example_outputsc|jr#|j}|r||vrtd|d|n-|r)t|dkrt|d}nd}|S)NzReference key z! not in Dataset example outputs: rMr) reference_keyrrr)rrrs r=_determine_reference_keyrs ,  }OCC777%477  S11Q66_--a0  r< eval_config7Union[EvaluatorType, str, smith_eval_config.EvalConfig]eval_llmOptional[BaseLanguageModel]rrrrr"c t|ttfr=t|tst|}t||}|j} nkd|i|} t|jfi| }|jj} t|tjr|j p|}|j p|}|j p|}t|trG|j r|td| d|dtj||||||| g} n;t|t$rt'd| dt'd| d | S) N)rrzPMust specify reference_key in smith_eval.RunEvalConfig to use evaluator of type z) with dataset with multiple output keys: r)rrrtagszRun evaluator for z is not implemented. PairwiseStringEvaluators compare the outputs of two different models rather than the output of a single model. Did you mean to use a StringEvaluator instead? See: https://python.langchain.com/docs/guides/evaluation/string/z is not implemented)rpr/rr.r get_kwargsevaluator_typesmith_eval_configSingleKeyEvalConfigrrrr1requires_referencer smith_evalStringRunEvaluatorChainfrom_run_and_data_typer0NotImplementedError) rr rrrrrr evaluator_ eval_type_tagr run_evaluators r=_construct_run_evaluatorrs+ s344 G+}55 5' 44K#KX>>> #) >[%;%;%=%=>#K$>II&II #28 k#4#H I I G#-:I(7I>N'5FM*o..  ( ]-BJ&3JJ7FJJJ  #:QQ   )'R   J 7 8 8  ! Q Q Q Q   " C C C C    r<2Tuple[Optional[str], Optional[str], Optional[str]]clt||}t||}t||}|||fSr)rrr)rrrrrrrs r= _get_keysr;s> %VZ88I.v{CCN,V_EEM nm 33r<List[RunEvaluator]c |g}d\}}} |jsCtd|jDs%|jr4td|jDrt||||\}}} |jD]2} t | |j|||| ||} || 3|jpg} | D]} t| tr|| -t| tr9|tj | ||||| {td| d|S)z Load run evaluators from a configuration. Args: config: Configuration for the run evaluators. Returns: A list of run evaluators. )NNNc8g|]}t|tSr;)rpr/rHrvs r=rJz(_load_run_evaluators..\s"HHH 1m,,HHHr<c8g|]}t|tSr;)rpr1r s r=rJz(_load_run_evaluators.._s"VVVZ?33VVVr<)rrrzUnsupported custom evaluator: z+. Expected RunEvaluator or StringEvaluator.) evaluatorsanycustom_evaluatorsrrr rsrpr"r1rrrr)rrrrrrrrrrrrr$custom_evaluators r=rrGs"N/?,I~}  HHf6GHHH I I    $  VVV=UVVVWW  4= J _4 4 0 >=( - - 0  O          m,,,,06B- & 5 5   ! !"2 3 3 3 3 (/ : :   ! !2II$'#1"/ J    >1A>>>  r<r  callbacksrrrr r'r,Union[str, BaseMessage]c K|||}t|tr||||d{VSt|tr7t d|Dr||||d{VSt d|d t|}||||d{V}n=#t $r0t|}||||d{V}YnwxYw|S)aAsynchronously run the language model. Args: llm: The language model to run. inputs: The input dictionary. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map inputs to the expected format. Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Nr'r c3@K|]}t|tVdSrrrs r=rz_arun_llm..=: : -0JsK ( (: : : : : : r<z%Input mapper returned invalid format 3 Expected a single string or list of chat messages.) rprapredictrrKapredict_messagesr6rr) rrr r'rprompt_or_messagesr llm_outputrs r= _arun_llmr2s.)\&11 (# . . "id& *D 1 1 c: : 4F: : : 7 7 .."id/ #G&GGG   ((F8; )$9E99333333JJ    $V,,H"44ID 5  JJJ  s#-C7D  D r rUnion[Chain, Runnable]Union[dict, str]cK||n ||}t|tr|t|trgt|dkrT|jrMt t |}||||d{V}n0t|pg|}| ||d{V}|S)z%Run a chain asynchronously on inputs.NrMr*r r'r) rpr-rqrrrrracallrainvoke rrr'r rinputs_rrZrunnable_configs r= _arun_chainr=s%,ff,,v2F2FG5%   F w % % F LLA     4(())**{{3)${GGGGGGGG(djbINNN}}W_}EEEEEEEE Mr<)rr'Union[dict, str, LLMResult, ChatResult]c Kt|trdnd}d} t|tr,t||j|d|d|d{V}n5|}t ||j|d|d|d{V}|}nc#t $rV}t |d|jd|jd t|t| }Yd}~nd}~wwxYw|S) aAsynchronously run the Chain or language model. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map the input to the expected format. Returns: A list of outputs. LLMr-Nr r'r& failed for example with inputs  rg) rprr2rr= Exceptionrridreprr~) rrrr chain_or_llmrzrZrrvs r=_arun_llm_or_chainrIs(02CDDQ'F$ *,= > >  )$F^ -) !!!FF)(**E&F^ -) F $$$    ">  a     ### $ MsA8B C8"A C33C8c|||}t|tr||||}nt|tr2t d|Dr||||}nrt d|d t|}||||}n6#t $r)t|}|||}YnwxYw|S)a Run the language model on the example. Args: llm: The language model to run. inputs: The input dictionary. callbacks: The callbacks to use during the run. tags: Optional tags to add to the run. input_mapper: function to map to the inputs dictionary from an Example Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Nr*c3@K|]}t|tVdSrrrs r=rz_run_llm..0r,r<z'Input mapper returned invalid format: r-)r') rprpredictrrKpredict_messagesr6rr) rrr'r rr0r1 llm_prompts llm_messagess r=_run_llmrPs`.)\&11 (# . . 25++"id3>33JJ*D 1 1 c: : 4F: : : 7 7 --"id.JJ#G&GGG   Q%f--KDQQJJ Q Q Q(00L--li-PPJJJ Q s'B??0C21C2Union[Dict, str]cz||n ||}t|trlt|trWt|dkrD|jr=t t |}||||}n*t|pg|}| ||}|S)zRun a chain on inputs.NrMr*r7r) rpr-rqrrrrrrinvoker:s r= _run_chainrTFs%,ff,,v2F2FG5%   ? w % % ? LLA     4(())**sid;;;(djbINNNgo>> Mr<c t|trdnd}d} t|tr&t||j|d|d|}n/|}t ||j|d|d|}|}nm#t $r`}t |j} t |d|j d|jd | d | t| }Yd}~nd}~wwxYw|S) a Run the Chain or language model synchronously. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. Returns: Union[List[dict], List[str], List[LLMResult], List[ChatResult]]: The outputs of the model or chain. r@r-Nr'r r3rArBz Error Type: z , Message: rD) rprrPrrTrErr7rrrFr~) rrrrrHrzrZrrv error_types r=_run_llm_or_chainrW^sb*02CDDQ'F$ *,= > > "${#F^) FF)(**E{#F^) F $$$!WW%  8 8 8 8"> 8 8' 8 845 8 8   ###$ MsA,B C4AC//C4clientr project_nameproject_metadataOptional[Dict[str, Any]]1Tuple[MCF, TracerSession, Dataset, List[Example]]c t||}||}t||j}|st d|d t } | r |pi}i|d| i}|||j|rd|ini|} nf#tt tf$rL} dt| vr| tj } d |d | d |d } t d |d| d} ~ wwxYw|j d| jz}td|d|d|d|j d|| ||fS)N)r) dataset_idzDataset z has no example rows.gitr )reference_dataset_id project_extrametadatazalready exists z+ run_on_dataset( ... project_name="z - z", # Update since z already exists ) z Test project z/ already exists. Please use a different name: z/compare?selectedSessions=z)View the evaluation results for project 'z' at: z View all tests for Dataset z at: T)flush)r read_datasetr list_examplesrFrr create_projectr*r)ruuiduuid4urlprint)rXrrrYrZr  wrapped_modeldatasetrgit_infoprojectrvuid example_msgcomparison_urls r=_prepare_eval_runrrs$++?NNM!!|!<>  /52  " x   '' !(,0864..b% (   z> 2    CFF * *Gjll %(*>s7:PR*S*STTJ &NN:r::",..1A"B"B$..22 ((GC OO $ &),, <4:LGJ(115;GJ(2 H8?GJ(5r<ci}|jD]3}tt|dD]}t|tr]|j}|D]@\\}}}|t|i d|iAut|tr|j }|r(|j r!|j |j z nd} |rt|jnd} |t|ji | | d5tt"tt$f|S)Nr'rY)rhri)ryrrrprlogged_eval_resultsrn setdefaultrrrr latest_runend_time start_time total_secondsrFryr rt) rSr{ccallback eval_results_ryr^runrhris r=_collect_metricsz%_DatasetRunContainer._collect_metricss!#  A q~66  h(@AA#+#?L.:.@.@.B.B*J(33C OORHHOO'O /:: "-C"#&<"6EEGGG!# -09S[[[TF$//H4G0H0H"MMTT.<&, *Dj)+;<<T!WW>> ? ? ? ? ? ? ? ? @s.#= A;*A66A;?AC D *C;;DNrrrrrYrr2rr rrrconcurrency_levelintrZr[c ,|ptj}t|||| \} } } pgjdpiD] \} }d| d|!t|} t| | || j p tj t| d| |tjt| fd| D}|| | |S)N)rZr r_zgit:=rc g|]J}ttjd|jt pg|jdgKS)F)rYrX use_threadingryr)r"rXrymax_concurrency)r'r r)rrrrFr)rHrrXr progress_barrnrr s r=rJz0_DatasetRunContainer.prepare..Is   (' #%,\%&+#*:  -#1#7R%#*:()  ! 1#      r<)rXrnrkrry)r3 random_namerrrbrornrsrrrr&kvrr4ProgressBarCallbackr)clsrXrrrYr2r rrrZrkrlrr]r^ryrrnrs ` ` ` @@@r=preparez_DatasetRunContainer.prepare*sx$D'B'D'D 4E   - 5 5 5 1 wzr%))%006B==?? ( (DAq KKq1 ' ' ' './CDD * 8Z1B1Qhk   !!m\JJJ3CMMBB          ($)   ,s'     r<)rzrr{r|r@rq)r@r|)rzrr@r?)F)rzrrrr@r?)NNNrN)rXrrrrrrYrr2rr rrrrrrZr[r@rx) r7r8r9r:rurrrr classmethodrr;r<r=rxrxs==NNN!!!!,====4     ":>$(8>-II uus9< A  A aggregate_resultsrActr)ddlm}m}||d||dS|dd}t dt |dS)Nr)HTMLdisplayz

Experiment Results:

c |dS)Nz.2fr;)xs r=rz,_display_aggregate_results..zs a::r<right) float_formatjustifyz Experiment Results:)rIPython.displayrr to_stringrj)rrrformatted_strings r=rrrs   11111111344555!""""",66--w7   &''' r<aThe input_mapper argument is deprecated and will be removed in a future release. Please add a RunnableLambda to your chain to map inputs to the expected format instead. Example: def construct_chain(): my_chain = ... input_mapper = {'other_key': 'MyOtherInput', 'my_input_key': x} return input_mapper | my_chain run_on_dataset(..., llm_or_chain_factory=construct_chain) (See https://api.python.langchain.com/en/latest/schema/langchain.schema.runnable.base.RunnableLambda.html)r)r2rrYrZrr Optional[Client]rrrrrc (K| dd} | rtdtd| r(tdd| dd|p t }t ||||||| || } tj| j d  d gttj t| j| | j| j Rd{V} | | | S) Nr0.0.305TmessagependingPThe following arguments are deprecated and will be removed in a future release: rrremovalrZrrrrr)popr_INPUT_MAPPER_DEP_WARNINGrrrxrrunnable_utilsgather_with_concurrencyryromap functoolspartialrIrkrr) rXrrr2rrYrZrr rr containerrzs r=arun_on_datasetrsr::nd33LT +DdSSSS    {{}}         vxxF$,, )-  I)@!  !233   "%.%<)                  M   M7  ; ;;r<c  | dd rtdtd| r(tdd| dd|p t }t |||||| || |d kr( fd t j j D} ntj j d 5} t| tjt j  j j } dddn #1swxYwY | | S) NrrTrrrrrrcFg|]\}}t||jS)r)rWrk)rHrrrrs r=rJz"run_on_dataset..sM     %.%<)       r<rr)rrrrrrxrr~rryr<get_executor_for_configrrrrrWrkr)rXrrr2rrYrZrr rrzexecutorrrs @@r=run_on_datasetrs::nd33LT +DdSSSS    {{}}         vxxF$,, )-  IA     $'y'99;L#M#M    4Y5Fq5I J J h  %)-6-D%1 &%  M                  M7  ; ;;sAD66D:=D:ao Run the Chain or language model on a dataset and store traces to the specified project name. Args: dataset_name: Name of the dataset to run the chain on. llm_or_chain_factory: Language model or Chain constructor to run over the dataset. The Chain constructor is used to permit independent calls on each example without carrying over state. evaluation: Configuration for evaluators to run on the results of the chain concurrency_level: The number of async tasks to run concurrently. project_name: Name of the project to store the traces in. Defaults to {dataset_name}-{chain class name}-{datetime}. project_metadata: Optional metadata to add to the project. Useful for storing information the test variant. (prompt version, model version, etc.) client: LangSmith client to use to access the dataset and to log feedback and run traces. verbose: Whether to print progress. tags: Tags to add to each run in the project. Returns: A dictionary containing the run's project name and the resulting model outputs. For the (usually faster) async version of this function, see :func:`arun_on_dataset`. Examples -------- .. code-block:: python from langsmith import Client from langchain_community.chat_models import ChatOpenAI from langchain.chains import LLMChain from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset # Chains may have memory. Passing in a constructor function lets the # evaluation framework avoid cross-contamination between runs. def construct_chain(): llm = ChatOpenAI(temperature=0) chain = LLMChain.from_string( llm, "What's the answer to {your_input_key}" ) return chain # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) evaluation_config = smith_eval.RunEvalConfig( evaluators=[ "qa", # "Correctness" against a reference answer "embedding_distance", smith_eval.RunEvalConfig.Criteria("helpfulness"), smith_eval.RunEvalConfig.Criteria({ "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" }), ] ) client = Client() run_on_dataset( client, "", construct_chain, evaluation=evaluation_config, ) You can also create custom evaluators by subclassing the :class:`StringEvaluator ` or LangSmith's `RunEvaluator` classes. .. code-block:: python from typing import Optional from langchain.evaluation import StringEvaluator class MyStringEvaluator(StringEvaluator): @property def requires_input(self) -> bool: return False @property def requires_reference(self) -> bool: return True @property def evaluation_name(self) -> str: return "exact_match" def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict: return {"score": prediction == reference} evaluation_config = smith_eval.RunEvalConfig( custom_evaluators = [MyStringEvaluator()], ) run_on_dataset( client, "", construct_chain, evaluation=evaluation_config, ) zrun_on_dataset(zawait arun_on_dataset()r)rrrrr@r)rrr@r)rrr@r)rr'rrr@r)rr'rr-rrr@r)rr'rrrrr@r) rrrrr2rrr&r@r)rrrrr@r)rrrrr@r)rrrrr@r)rrr r rrrr&rrrrrrrrr@r") rrrrrrrrr@r)rrrrrr&rrrrrrr@r) rrrrr rr'r,rrr@r() rr4rrr'r,r rrrr@r5) rr'rrrrrrr@r>) rrrrr'r,r rrrr@r() rr4rrr'r,r rrrr@rQr)rXrrrrrrYrrZr[r rr@r\)r@r)rrAr@r)rXrrrrrr2rrrrYrrZr[rrr rrrr@r)}r: __future__r dataclassesrrloggingrgrrenumrtypingrrr r r r r rrlangchain_core._apirlangchain_core.language_modelsrlangchain_core.messagesrrlangchain_core.outputsrrlangchain_core.runnablesrrrrr<rr!langchain_core.tracers.evaluationrr langchain_core.tracers.langchainrlangsmith.clientr langsmith.envr langsmith.evaluationr!r"langsmith.run_helpersr#r$langsmith.schemasr%r&r'r(langsmith.utilsr)requestsr*typing_extensionsr+langchain.callbacks.managerr,langchain.chains.baser-langchain.evaluation.loadingr.langchain.evaluation.schemar/r0r1langchain.smithr2rlangchain.smith.evaluationrr3r4rlru getLoggerr7rrqrrrEr6r?r~rrrrrrrrrrrrrr2r=rIrPrTrWrrrt dataclassrxrrrrr_RUN_ON_DATASET_DOCSTRINGreplacer;r<r=rsDD"""""" ''''''''                      0/////<<<<<<CCCCCCCC88888888MMMMMMMMMM>>>>>><<<<<<=<<<<<######&&&&&&????????DDDDDDDDGGGGGGGGGGGG******''''''111111''''''777777 544444BBBBBB@@@@@@@@  8 $ $ Ruh' '( dVS[    HRuh// 02C CD33333y333E4E4E4E4E4E4E4E4P R R R R R R R R': : : : : z0 0 0 0 f* * * * \B""""JCCCC(""""J0.$6666r 4 4 4 4????R!%48 555555z!%48 :59 111111x!%48 000000p!%48 :59 333333|26 $ 1515151515h% Q Q Q Q Q Q Q Q h      :(6:"&15 $1<1<1<1<1<1