h2JdZddlZddlZddlmZddlmZddlmZddl Z ddl Z ddl mZddlmZdd lmZmZmZd ed ed efd Zd ed ed e jfdZdZ ddZ ddZ ddZdS)z9Implementation of ARFF parsers: via LIAC-ARFF and pandas.N) OrderedDict) Generator)List)_arff)ArffSparseDataType)_chunk_generatorcheck_pandas_supportget_chunk_n_rows arff_datainclude_columnsreturnctttf}dt|D}t|d|d|dD]a\}}}||vrW|d||d||d||b|S)aObtains several columns from sparse ARFF representation. Additionally, the column indices are re-labelled, given the columns that are not included. (e.g., when including [1, 2, 3], the columns will be relabelled to [0, 1, 2]). Parameters ---------- arff_data : tuple A tuple of three lists of equal size; first list indicating the value, second the x coordinate and the third the y coordinate. include_columns : list A list of columns to include. Returns ------- arff_data_new : tuple Subset of arff data with only the include columns indicated by the include_columns argument. ci|]\}}|| Sr.0 array_idx column_idxs e/var/www/html/services/rag-service/venv/lib/python3.11/site-packages/sklearn/datasets/_arff_parser.py z)_split_sparse_columns..,+"7)Z Irr)list enumeratezipappend)r r arff_data_newreindexed_columnsvalrow_idxcol_idxs r_split_sparse_columnsr$s.*.(@M;D_;U;U"%Yq\9Q<1!N!N@@Wg o % % !  # #C ( ( ( !  # #G , , , !  # #$5g$> ? ? ? rc@t|ddz}|t|f}dt|D}tj|tj}t |d|d|dD]\}}}||vr |||||f<|S)Nrci|]\}}|| Srrrs rrz)_sparse_data_to_array..>rrdtyperr)maxlenrnpemptyfloat64r) r r num_obsy_shaper yr!r"r#s r_sparse_data_to_arrayr17s )A,!#GO,,-G;D_;U;U  +++A!$Yq\9Q<1!N!N99Wg o % %58Ag(11 2 Hrc||}t|dkr ||}n$t|dkr||d}nd}||fS)a Post process a dataframe to select the desired columns in `X` and `y`. Parameters ---------- frame : dataframe The dataframe to split into `X` and `y`. feature_names : list of str The list of feature names to populate `X`. target_names : list of str The list of target names to populate `y`. Returns ------- X : dataframe The dataframe containing the features. y : {series, dataframe} or None The series or dataframe containing the target. rrrN)r*)frame feature_names target_namesXr0s r_post_process_framer7Is^, mA <A ,  \  a   ,q/ "  a4Krc "#$d}||}|dkr tjn tj}|dk } tj||| } ||z##fd| dD"|dkrt d} t | d} t | } t| d}| |g| d }| d  }t|}#fd | D}||g}t| d|D]3}|| || d |4t|dkr)|d|dj|d<| |d t(j}~~i}|jD]\}|d}|dkrd||<.|dkrd||<L|j|||<]||}t1|||\}$n<| d}fd|D}fd|D}t3|t4r|t7d|ddkrd}n|d|dz}t)jt:j|d|}|j |}|dd|f}|dd|f$nt3|tBrtE||}tG|ddz}|t|f} tHj%&|d|d|dff| t(j'}|(}tS||$nt7d tU|"fd!|D}!|!sn\tW|!r/t)j,"$fd"t[|D$nt]|!rt7d#$j/ddkr$ d$$n$j/ddkrd$|dkr|$|dfS|$d"fS)%aARFF parser using the LIAC-ARFF library coded purely in Python. This parser is quite slow but consumes a generator. Currently it is needed to parse sparse datasets. For dense datasets, it is recommended to instead use the pandas-based parser, although it does not always handles the dtypes exactly the same. Parameters ---------- gzip_file : GzipFile instance The file compressed to be read. output_arrays_type : {"numpy", "sparse", "pandas"} The type of the arrays that will be returned. The possibilities ara: - `"numpy"`: both `X` and `y` will be NumPy arrays; - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a pandas Series or DataFrame. columns_info : dict The information provided by OpenML regarding the columns of the ARFF file. feature_names_to_select : list of str A list of the feature names to be selected. target_names_to_select : list of str A list of the target names to be selected. Returns ------- X : {ndarray, sparse matrix, dataframe} The data matrix. y : {ndarray, dataframe, series} The target. frame : dataframe or None A dataframe containing both `X` and `y`. `None` if `output_array_type != "pandas"`. categories : list of str or None The names of the features that are categorical. `None` if `output_array_type == "pandas"`. c3BK|D]}|dVdS)Nutf-8)decode) gzip_filelines r_io_to_generatorz+_liac_arff_parser.._io_to_generators: ' 'D++g&& & & & & ' 'rsparsepandas) return_typeencode_nominalcNi|]!\}}t|tr|v||"Sr) isinstancer)rnamecatcolumns_to_selects rrz%_liac_arff_parser..sI D# c4 &*->%>%> c%>%>%>r attributeszfetch_openml with as_frame=TruedataF)columnscopyT)deepcg|]}|v| SrrrcolrGs r z%_liac_arff_parser..s$TTT33BS;S;S3;S;S;Srrrr) ignore_index)value data_typeintegerInt64nominalcategorycFg|]}t|dSindexintrcol_nameopenml_columns_infos rrPz%_liac_arff_parser..s<% % %  #H-g6 7 7% % % rcFg|]}t|dSrYr[r]s rrPz%_liac_arff_parser..s<$ $ $  #H-g6 7 7$ $ $ rNz6shape must be provided when arr['data'] is a Generatorr-)r(count)shaper(z-Unexpected type for data obtained from arff: ch|]}|vSrr)rr^ categoriess r z$_liac_arff_parser..s+   '/H "   rc g|]j\}}tjtj|ddd||dzft dkS)Or'NrF)rK)r+takeasarraypopastyper\)rir^rer0s rrPz%_liac_arff_parser..s $8 G :>>(#;#;3GGG!!!QQY,..s.??rzAMix of nominal and non-nominal targets is not currently supported)ra)0rCOO DENSE_GENloadr rrkeysnext DataFrame memory_usagesumr r rr*rldtypesconcatfillnar+nanrJlowerr7rDr ValueErrorfromiter itertoolschain from_iterablereshapetupler$r)spr? coo_matrixr-tocsrr1typeallhstackranyrc)%r<output_arrays_typer_feature_names_to_selecttarget_names_to_selectrcr>streamrArBarff_containerpd columns_info columns_names first_rowfirst_df row_bytes chunksizecolumns_to_keepdfsrIr3rvrE column_dtyper6r feature_indices_to_selecttarget_indices_to_selectrb arff_data_Xr.X_shapeis_classificationrerGr0s% ` @@@r_liac_arff_parserrisn''' i ( (F 2X==%))5?K-89NZKN02HH' 5J X%% !"C D D">,#?@@ \..0011 /00 << ]<OO))t)4488:: $Y// UTTT-TTT()$^F%;YGG  D JJ T=u EEoV     s88q==V]]3q6=11CF  #D 1188rv8FF M 2 2D.t4[AL!!##y00 't ##%%22)t $|D1t  V$$" *,B  11#6* % % % % 3% % % !$ $ $ $ 2$ $ $  i + + } LQx2~~a58+;--i88D  4<'DQQQ112AQQQ001AA  5 ) ) / ;TUUK)A,''!+G$= > >?G $$Q+a.+a.!ABj%A  A%i1IJJAAQYQQ     3I   !   " # #   (11G'H'H AA" # # S  71:?? %  AA WQZ1__AX%%!UD  az !!rc "ddl|D]>}|ddrn?i|D]K}||d}|dkrd|<.|dkrd |<Lfd t |D} dd d gd d ddd| d } i| |pi}j|fi|} d|D| _n-#t$r } j d| d} ~ wwxYw||zfd| jD} | | } tj dfd}fd| j D}|D]%}| |j|| |<&t!| ||\}}|dkr||| dfS||}}fd| j D}||d|fS)a^ARFF parser using `pandas.read_csv`. This parser uses the metadata fetched directly from OpenML and skips the metadata headers of ARFF file itself. The data is loaded as a CSV file. Parameters ---------- gzip_file : GzipFile instance The GZip compressed file with the ARFF formatted payload. output_arrays_type : {"numpy", "sparse", "pandas"} The type of the arrays that will be returned. The possibilities are: - `"numpy"`: both `X` and `y` will be NumPy arrays; - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a pandas Series or DataFrame. openml_columns_info : dict The information provided by OpenML regarding the columns of the ARFF file. feature_names_to_select : list of str A list of the feature names to be selected to build `X`. target_names_to_select : list of str A list of the target names to be selected to build `y`. read_csv_kwargs : dict, default=None Keyword arguments to pass to `pandas.read_csv`. It allows to overwrite the default options. Returns ------- X : {ndarray, sparse matrix, dataframe} The data matrix. y : {ndarray, dataframe, series} The target. frame : dataframe or None A dataframe containing both `X` and `y`. `None` if `output_array_type != "pandas"`. categories : list of str or None The names of the features that are categorical. `None` if `output_array_type == "pandas"`. rNr:z@datarSrTrUrVrWc0i|]\}}|v ||Srr)rr#rErvs rrz'_pandas_arff_parser..~s4 GT 6>>  >>rF?%"T\) header index_col na_valueskeep_default_nacomment quotecharskipinitialspace escapecharr(cg|]}|Srr)rrEs rrPz'_pandas_arff_parser..s>>>$>>>rzwThe number of columns provided by OpenML does not match the number of columns inferred by pandas when reading the file.cg|]}|v| SrrrNs rrPz'_pandas_arff_parser..s$PPPss>O7O7Os7O7O7Orz^'(?P.*)'$c`tj|}||S|dS)Ncontents)researchgroup) input_stringmatchsingle_quote_patterns rstrip_single_quotesz0_pandas_arff_parser..strip_single_quotess2 . == = {{:&&&rcDg|]\}}t|j|Sr)rDCategoricalDtyperrEr(rs rrPz'_pandas_arff_parser..s@ D% eR0 1 1 rr@cti|]4\}}t|j||j5Sr)rDrretolistrs rrz'_pandas_arff_parser..sQ D% eR0 1 1 e%%''r)r@r;rz startswithrread_csvrJr{errors ParserErrorrcompilervitemsrFrename_categoriesr7to_numpy)r<rr_rrread_csv_kwargsr=rErdtypes_positionaldefault_read_csv_kwargsr3excrrcategorical_columnsrOr6r0rerGrvrrs @@@@r_pandas_arff_parserr4sp ;;w   % % ' ' 2 27 ; ;  E F#&&*40=     9 , ,#F4LL    ! !Y . .%F4L&':;;U  "  M0L_5JLO BK 5 5_ 5 5E  ?>*=>>> i## @    02HHPPPPemPPPO / "E:&=>>''''' <--// #KK3Z^556IJJc u&=?U V VDAqX%%!UD  zz||QZZ\\1 <--//J az !!sC++ D5DDc|dkrt||||||S|dkrt||||||Std|d)a6Load a compressed ARFF file using a given parser. Parameters ---------- gzip_file : GzipFile instance The file compressed to be read. parser : {"pandas", "liac-arff"} The parser used to parse the ARFF file. "pandas" is recommended but only supports loading dense datasets. output_type : {"numpy", "sparse", "pandas"} The type of the arrays that will be returned. The possibilities ara: - `"numpy"`: both `X` and `y` will be NumPy arrays; - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a pandas Series or DataFrame. openml_columns_info : dict The information provided by OpenML regarding the columns of the ARFF file. feature_names_to_select : list of str A list of the feature names to be selected. target_names_to_select : list of str A list of the target names to be selected. read_csv_kwargs : dict, default=None Keyword arguments to pass to `pandas.read_csv`. It allows to overwrite the default options. Returns ------- X : {ndarray, sparse matrix, dataframe} The data matrix. y : {ndarray, dataframe, series} The target. frame : dataframe or None A dataframe containing both `X` and `y`. `None` if `output_array_type != "pandas"`. categories : list of str or None The names of the features that are categorical. `None` if `output_array_type == "pandas"`. z liac-arffr@zUnknown parser: 'z%'. Should be 'liac-arff' or 'pandas'.)rrr{)r<parser output_typer_rrrcrs rload_arff_from_gzip_filersv    # "      8  "    # "      M M M M   r)N)NN)__doc__r}r collectionsrcollections.abcrtypingrnumpyr+scipyr externalsrexternals._arffrutilsr r r r$ndarrayr1r7rrrrrrrs?? ######%%%%%%000000 ! 48     F ! 48 Z    $L  H"H"H"H"b U"U"U"U"~ P P P P P P r