o 1 i >@sddlmZmZmZmZddlZddlZddl m Z m Z m Z m Z mZddlmZddlmZer6ddlmZeddGd d d eZeddGd d d eZeddGd ddeZeddGdddeZdS)) TYPE_CHECKINGListOptionalTupleN)AbsMaxMaxMeanMinStd) Preprocessor) PublicAPI)Datasetalpha)Z stabilitycbeZdZdZddeedeeeffdd Zddd efd d Z d e j fd dZ ddZ ZS)StandardScalerav Translate and scale each column by its mean and standard deviation, respectively. The general formula is given by .. math:: x' = \frac{x - \bar{x}}{s} where :math:`x` is the column, :math:`x'` is the transformed column, :math:`\bar{x}` is the column average, and :math:`s` is the column's sample standard deviation. If :math:`s = 0` (i.e., the column is constant-valued), then the transformed column will contain zeros. .. warning:: :class:`StandardScaler` works best when your data is normal. If your data isn't approximately normal, then the transformed features won't be meaningful. Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import StandardScaler >>> >>> df = pd.DataFrame({"X1": [-2, 0, 2], "X2": [-3, -3, 3], "X3": [1, 1, 1]}) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> ds.to_pandas() # doctest: +SKIP X1 X2 X3 0 -2 -3 1 1 0 -3 1 2 2 3 1 Columns are scaled separately. >>> preprocessor = StandardScaler(columns=["X1", "X2"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -1.224745 -0.707107 1 1 0.000000 -0.707107 1 2 1.224745 1.414214 1 Constant-valued columns get filled with zeros. >>> preprocessor = StandardScaler(columns=["X3"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -2 -3 0.0 1 0 -3 0.0 2 2 3 0.0 >>> preprocessor = StandardScaler( ... columns=["X1", "X2"], ... output_columns=["X1_scaled", "X2_scaled"] ... ) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 X1_scaled X2_scaled 0 -2 -3 1 -1.224745 -0.707107 1 0 -3 1 0.000000 -0.707107 2 2 3 1 1.224745 1.414214 Args: columns: The columns to separately scale. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. Ncolumnsoutput_columnsc"t||_t|||_dSNsuper__init__rr #_derive_and_validate_output_columnsrselfrr __class__i/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/ray/data/preprocessors/scaler.pyrS  zStandardScaler.__init__datasetr returncCs,|jjt|jd|jjdd|jd|S)N)Z aggregator_fnrcSs t|ddS)Nr)Zddof)r colrrr`s z%StandardScaler._fit..)Zstat_computation_planZadd_aggregatorrr)rr rrr_fitZszStandardScaler._fitdfc.dtjffdd }|j||j<|S)Nscsbjd|jd}jd|jd}|dus|dur%tj|dd<|S|dkr+d}|||S)Nzmean()zstd(r)stats_namenpnan)r(Zs_meanZs_stdrrrcolumn_standard_scalerfs z@StandardScaler._transform_pandas..column_standard_scalerpdZSeriesrZ transformr)rr&r0rr/r_transform_pandasesz StandardScaler._transform_pandascC|jjd|jd|jdSN (columns=z, output_columns=r)r__name__rrr/rrr__repr__xzStandardScaler.__repr__rr8 __module__ __qualname____doc__rstrrrr r%r2 DataFramer3r9 __classcell__rrrrrs $C rcr) MinMaxScaleraScale each column by its range. The general formula is given by .. math:: x' = \frac{x - \min(x)}{\max{x} - \min{x}} where :math:`x` is the column and :math:`x'` is the transformed column. If :math:`\max{x} - \min{x} = 0` (i.e., the column is constant-valued), then the transformed column will get filled with zeros. Transformed values are always in the range :math:`[0, 1]`. .. tip:: This can be used as an alternative to :py:class:`StandardScaler`. Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import MinMaxScaler >>> >>> df = pd.DataFrame({"X1": [-2, 0, 2], "X2": [-3, -3, 3], "X3": [1, 1, 1]}) # noqa: E501 >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> ds.to_pandas() # doctest: +SKIP X1 X2 X3 0 -2 -3 1 1 0 -3 1 2 2 3 1 Columns are scaled separately. >>> preprocessor = MinMaxScaler(columns=["X1", "X2"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 0.0 0.0 1 1 0.5 0.0 1 2 1.0 1.0 1 Constant-valued columns get filled with zeros. >>> preprocessor = MinMaxScaler(columns=["X3"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -2 -3 0.0 1 0 -3 0.0 2 2 3 0.0 >>> preprocessor = MinMaxScaler(columns=["X1", "X2"], output_columns=["X1_scaled", "X2_scaled"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 X1_scaled X2_scaled 0 -2 -3 1 0.0 0.0 1 0 -3 1 0.5 0.0 2 2 3 1 1.0 1.0 Args: columns: The columns to separately scale. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. NrrcrrrrrrrrrzMinMaxScaler.__init__r r r!cs&fddttfD}|j|_S)Ncs g|] }jD]}||qqSr)r).0ZAggr#r/rr s z%MinMaxScaler._fit..)r r aggregater+rr Z aggregatesrr/rr%s zMinMaxScaler._fitr&cr')Nr(csHjd|jd}jd|jd}||}|dkrd}|||S)Nzmin(r)zmax(rr*r+r,)r(Zs_minZs_maxdiffr/rrcolumn_min_max_scalers  z=MinMaxScaler._transform_pandas..column_min_max_scalerr1)rr&rIrr/rr3s zMinMaxScaler._transform_pandascCr4r5r7r/rrrr9r:zMinMaxScaler.__repr__rr;rrrrrB|s $?rBcr) MaxAbsScalera@Scale each column by its absolute max value. The general formula is given by .. math:: x' = \frac{x}{\max{\vert x \vert}} where :math:`x` is the column and :math:`x'` is the transformed column. If :math:`\max{\vert x \vert} = 0` (i.e., the column contains all zeros), then the column is unmodified. .. tip:: This is the recommended way to scale sparse data. If you data isn't sparse, you can use :class:`MinMaxScaler` or :class:`StandardScaler` instead. Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import MaxAbsScaler >>> >>> df = pd.DataFrame({"X1": [-6, 3], "X2": [2, -4], "X3": [0, 0]}) # noqa: E501 >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> ds.to_pandas() # doctest: +SKIP X1 X2 X3 0 -6 2 0 1 3 -4 0 Columns are scaled separately. >>> preprocessor = MaxAbsScaler(columns=["X1", "X2"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -1.0 0.5 0 1 0.5 -1.0 0 Zero-valued columns aren't scaled. >>> preprocessor = MaxAbsScaler(columns=["X3"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -6 2 0.0 1 3 -4 0.0 >>> preprocessor = MaxAbsScaler(columns=["X1", "X2"], output_columns=["X1_scaled", "X2_scaled"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 X1_scaled X2_scaled 0 -2 -3 1 -1.0 -1.0 1 0 -3 1 0.0 -1.0 2 2 3 1 1.0 1.0 Args: columns: The columns to separately scale. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. NrrcrrrrrrrrrzMaxAbsScaler.__init__r r r!cCs dd|jD}|j||_|S)NcSsg|]}t|qSr)r)rCr#rrrrD"sz%MaxAbsScaler._fit..)rrEr+rFrrrr%!s zMaxAbsScaler._fitr&cr')Nr(cs(jd|jd}|dkrd}||S)Nzabs_max(r)rr*rG)r(Z s_abs_maxr/rrcolumn_abs_max_scaler'sz=MaxAbsScaler._transform_pandas..column_abs_max_scalerr1)rr&rKrr/rr3&s zMaxAbsScaler._transform_pandascCr4r5r7r/rrrr94r:zMaxAbsScaler.__repr__rr;rrrrrJs $;rJc sreZdZdZ  ddeedeeefdeeeffdd Z d d d e fd d Z de j fddZddZZS) RobustScalera Scale and translate each column using quantiles. The general formula is given by .. math:: x' = \frac{x - \mu_{1/2}}{\mu_h - \mu_l} where :math:`x` is the column, :math:`x'` is the transformed column, :math:`\mu_{1/2}` is the column median. :math:`\mu_{h}` and :math:`\mu_{l}` are the high and low quantiles, respectively. By default, :math:`\mu_{h}` is the third quartile and :math:`\mu_{l}` is the first quartile. .. tip:: This scaler works well when your data contains many outliers. Examples: >>> import pandas as pd >>> import ray >>> from ray.data.preprocessors import RobustScaler >>> >>> df = pd.DataFrame({ ... "X1": [1, 2, 3, 4, 5], ... "X2": [13, 5, 14, 2, 8], ... "X3": [1, 2, 2, 2, 3], ... }) >>> ds = ray.data.from_pandas(df) # doctest: +SKIP >>> ds.to_pandas() # doctest: +SKIP X1 X2 X3 0 1 13 1 1 2 5 2 2 3 14 2 3 4 2 2 4 5 8 3 :class:`RobustScaler` separately scales each column. >>> preprocessor = RobustScaler(columns=["X1", "X2"]) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 0 -1.0 0.625 1 1 -0.5 -0.375 2 2 0.0 0.750 2 3 0.5 -0.750 2 4 1.0 0.000 3 >>> preprocessor = RobustScaler( ... columns=["X1", "X2"], ... output_columns=["X1_scaled", "X2_scaled"] ... ) >>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP X1 X2 X3 X1_scaled X2_scaled 0 1 13 1 -1.0 0.625 1 2 5 2 -0.5 -0.375 2 3 14 2 0.0 0.750 3 4 2 2 0.5 -0.750 4 5 8 3 1.0 0.000 Args: columns: The columns to separately scale. quantile_range: A tuple that defines the lower and upper quantiles. Values must be between 0 and 1. Defaults to the 1st and 3rd quartiles: ``(0.25, 0.75)``. output_columns: The names of the transformed columns. If None, the transformed columns will be the same as the input columns. If not None, the length of ``output_columns`` must match the length of ``columns``, othwerwise an error will be raised. g?g?Nrquantile_rangercs(t||_||_t|||_dSr)rrrrNr rr)rrrNrrrrr~s  zRobustScaler.__init__r r r!cs|jd}d}|jd}|}|dfdd|||fD}i|_|jD]N|jfdddd }|}||\} }}}d d d tfd d} | |} | |} | |} | |jdd<| |jdd<| |jdd<q&|S)Nrg?r*csg|]}t|qSr)int)rCZ percentile) max_indexrrrDsz%RobustScaler._fit..cs |gSrr)r&r"rrr$s z#RobustScaler._fit..pandas)Z batch_formatdsr ccSs|dd|S)Nr*r)Ztake)rRrSrrr_get_first_valuesz+RobustScaler._fit.._get_first_value low_quantile(r)median(high_quantile()rNcountr+rZ map_batchessortZsplit_at_indicesr?)rr lowZmedhighZ num_recordsZ split_indicesZfiltered_datasetZsorted_dataset_rTZlow_valZmed_valZhigh_valr)r#rPrr%s*        zRobustScaler._fitr&cr')Nr(csbjd|jd}jd|jd}jd|jd}||}|dkr+t|S|||S)NrUr)rVrWr)r+r,r-Z zeros_like)r(Zs_low_qZs_medianZs_high_qrHr/rrcolumn_robust_scalers  z.column_robust_scalerr1)rr&r]rr/rr3s zRobustScaler._transform_pandascCs&|jjd|jd|jd|jdS)Nr6z, quantile_range=z), output_columns=r))rr8rrNrr/rrrr9s zRobustScaler.__repr__)rMN)r8r<r=r>rr?rfloatrrr r%r2r@r3r9rArrrrrL8sG  "rL)typingrrrrnumpyr-rQr2Zray.data.aggregaterrrr r Zray.data.preprocessorr Zray.util.annotationsr Zray.data.datasetr rrBrJrLrrrrs    m`Z