a
    ;Za?                    @   s  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
Z
ddlZddlmZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZ ddlmZm Z  ddlm!Z! ddlm"Z" ddlm#Z# ddlm$Z$ ddlm%Z% ddlm&Z& ddlm'Z' ddlm(Z( ddlm)Z) ddlm*Z* ddlm+Z+ ddlm,Z, ddl-m.Z. ddl-m/Z/ ddl-m0Z0 ddl1m2Z2 ddl1m3Z3 dd l1m4Z4 dd!l1m5Z5 dd"l6m7Z7 dd#l6m8Z8 dd$l6m9Z9 dd%l6m:Z: dd&l6m;Z; dd'l6m<Z< dd(l6m=Z= dd)l6m>Z> dd*l6m?Z? dd+l@mAZAmBZBmCZC dd,l@mDZDmEZE dd-lFmGZG dd.lHmIZI dd/lJmKZKmLZL dd0lMmNZN dd1lOmPZP dd2lQmRZR dd3lSmTZT dd4lUmVZV dd5lWmXZX dd6lWmYZY dd7lZm[Z[ dd8l\m]Z] dd9l1m^Z^ dd:l1m_Z_ dd;l`maZa dd<lmbZb zec W n edy0   dZcY n0 G d=d> d>eXZeG d?d@ d@eeZfG dAdB dBeXZgG dCdD dDegZhG dEdF dFZiejdGZkeekZlemg dHZnemg dIZoeepdJZqdKdL ZrdMdN ZsdOdP ZtdQdR ZudSdT ZvdUdV ZwdWdX ZxdYdZ Zye
jz{d[d\d] Z|d^d_ Z}d`da Z~dbdc Zddde Zdfdg Zdhdi Zdjdk Zdldm Zdndo Zdpdq Zdrds Zdtdu Zdvdw Zdxdy Zdzd{ Zd|d} Zd~d Ze
jz{d[dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZG dd deGZddÄ Zddń ZddǄ ZddɄ Zdd˄ Zdd̈́ Ze
jz{d[ddτ Zddф Zddӄ Ze
jzdejdgddք Zdd؄ Ze
jz{d١e
jzdejddgdd܄ Ze
jz{d١e
jzdejddge
jzdddge
jzdddgdd Zdd Ze
jzddeddddfdeedddddfdeedddddfgdd Zdd Zdd Zdd ZdS )zTest the validation module    N)partial)sleep)
coo_matrix
csr_matrix)FitFailedWarning)FailingClassifier)assert_almost_equal)assert_array_almost_equal)assert_array_equal)assert_allclose)CheckingClassifierMockDataFrame)_num_samples)cross_val_scoreShuffleSplit)cross_val_predict)cross_validate)permutation_test_score)KFold)StratifiedKFold)LeaveOneOut)LeaveOneGroupOut)LeavePGroupsOut)
GroupKFold)GroupShuffleSplit)learning_curve)validation_curve)_check_is_permutation)_fit_and_score)_score)make_regression)load_diabetes)	load_iris)load_digits)explained_variance_score)make_scorer)accuracy_score)confusion_matrix)precision_recall_fscore_support)precision_score)r2_score)mean_squared_error)check_scoring)RidgeLogisticRegressionSGDClassifier)PassiveAggressiveClassifierRidgeClassifier)RandomForestClassifier)KNeighborsClassifier)SVC	LinearSVC)KMeans)SimpleImputer)LabelEncoder)Pipeline)StringIO)BaseEstimator)clone)OneVsRestClassifier)shuffle)make_classification)make_multilabel_classification)OneTimeSplitter)GridSearchCVc                   @   s<   e Zd ZdZdd ZdddZdd Zdd	d
Zdd ZdS )MockImprovingEstimatorz+Dummy classifier to test the learning curvec                 C   s   || _ d| _d | _d S Nr   )n_max_train_sizestrain_sizesX_subset)selfrE    rI   Llib/python3.9/site-packages/sklearn/model_selection/tests/test_validation.py__init__Y   s    zMockImprovingEstimator.__init__Nc                 C   s   || _ |jd | _| S rD   rG   shaperF   rH   rG   Zy_subsetrI   rI   rJ   fit^   s    zMockImprovingEstimator.fitc                 C   s   t d S NNotImplementedErrorrH   XrI   rI   rJ   predictc   s    zMockImprovingEstimator.predictc                 C   s2   |  |rdt| j| j  S t| j| j S d S )Ng       @)_is_training_datafloatrF   rE   rH   rT   YrI   rI   rJ   scoref   s    
zMockImprovingEstimator.scorec                 C   s
   || j u S rP   rG   rS   rI   rI   rJ   rV   m   s    z(MockImprovingEstimator._is_training_data)N)NN	__name__
__module____qualname____doc__rK   rO   rU   rZ   rV   rI   rI   rI   rJ   rC   V   s   

rC   c                       s4   e Zd ZdZd	 fdd	Zdd Zd
ddZ  ZS )!MockIncrementalImprovingEstimatorz*Dummy classifier that provides partial_fitNc                    s   t  | d | _|| _d S rP   )superrK   xexpected_fit_params)rH   rE   rd   	__class__rI   rJ   rK   t   s    z*MockIncrementalImprovingEstimator.__init__c                 C   s
   | j |v S rP   rc   rS   rI   rI   rJ   rV   y   s    z3MockIncrementalImprovingEstimator._is_training_datac              	   K   s   |  j |jd 7  _ |d | _| jrt| jt| }|rNtdt| d| D ]F\}}|| jv rVt|t|krVtd| dt| dt| dqVd S )Nr   zExpected fit parameter(s) z
 not seen.zFit parameter z has length z; expected .)	rF   rM   rc   rd   setAssertionErrorlistitemsr   )rH   rT   yparamsZmissingkeyvaluerI   rI   rJ   partial_fit|   s&    
z-MockIncrementalImprovingEstimator.partial_fit)N)N)r]   r^   r_   r`   rK   rV   rq   __classcell__rI   rI   re   rJ   ra   q   s   ra   c                   @   s<   e Zd ZdZdddZdd Zdd Zdd
dZdd Zd	S )MockEstimatorWithParameterz-Dummy classifier to test the validation curve      ?c                 C   s   d | _ || _d S rP   )rG   param)rH   ru   rI   rI   rJ   rK      s    z#MockEstimatorWithParameter.__init__c                 C   s   || _ |jd | _| S rD   rL   rN   rI   rI   rJ   rO      s    zMockEstimatorWithParameter.fitc                 C   s   t d S rP   rQ   rS   rI   rI   rJ   rU      s    z"MockEstimatorWithParameter.predictNc                 C   s   |  |r| jS d| j S )N   )rV   ru   )rH   rT   rm   rI   rI   rJ   rZ      s    z MockEstimatorWithParameter.scorec                 C   s
   || j u S rP   r[   rS   rI   rI   rJ   rV      s    z,MockEstimatorWithParameter._is_training_data)rt   )NNr\   rI   rI   rI   rJ   rs      s   

rs   c                       s(   e Zd ZdZ fddZdd Z  ZS )%MockEstimatorWithSingleFitCallAllowedz<Dummy classifier that disallows repeated calls of fit methodc                    s&   t | drJ dd| _t ||S )Nfit_called_zfit is called the second timeT)hasattrrx   rb   rO   rN   re   rI   rJ   rO      s    z)MockEstimatorWithSingleFitCallAllowed.fitc                 C   s   t d S rP   rQ   rS   rI   rI   rJ   rU      s    z-MockEstimatorWithSingleFitCallAllowed.predict)r]   r^   r_   r`   rO   rU   rr   rI   rI   re   rJ   rw      s   rw   c                	   @   sH   e Zd ZdZdddZdddZd	d
 Zdd ZdddZdddZ	dS )MockClassifierz-Dummy classifier to test the cross-validationr   Fc                 C   s   || _ || _d S rP   aallow_nd)rH   r|   r}   rI   rI   rJ   rK      s    zMockClassifier.__init__Nc                 C   sT  || _ || _|	| _|
dur"|
|  | jr8|t|d}|jdkrP| jsPtd|dur|jd |jd ksJ d	|jd |jd |dur|jd tt
tksJ d	|jd tt
t|durd}|jd |jd ksJ |	|jd |jd |durPd	}|jtjksPJ |	|jd |jd
 tjd tjd
 | S )zThe dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        N   zX cannot be dr   zKMockClassifier extra fit_param sample_weight.shape[0] is {0}, should be {1}zIMockClassifier extra fit_param class_prior.shape[0] is {0}, should be {1}zRMockClassifier extra fit_param sparse_sample_weight.shape[0] is {0}, should be {1}zUMockClassifier extra fit_param sparse_param.shape is ({0}, {1}), should be ({2}, {3})rv   )	dummy_int	dummy_str	dummy_objr}   reshapelenndim
ValueErrorrM   formatnpuniquerm   P_sparse)rH   rT   rY   sample_weightclass_priorsparse_sample_weightsparse_paramr   r   r   callbackZfmtrI   rI   rJ   rO      sJ    

zMockClassifier.fitc                 C   s&   | j r|t|d}|d d df S )Nr~   r   )r}   r   r   rH   TrI   rI   rJ   rU      s    zMockClassifier.predictc                 C   s   |S rP   rI   r   rI   rI   rJ   predict_proba   s    zMockClassifier.predict_probac                 C   s   ddt | j  S )N      ?rv   )r   absr|   rX   rI   rI   rJ   rZ      s    zMockClassifier.scorec                 C   s   | j | jdS )Nr{   r{   )rH   ZdeeprI   rI   rJ   
get_params  s    zMockClassifier.get_params)r   F)	NNNNNNNNN)NN)F)
r]   r^   r_   r`   rK   rO   rU   r   rZ   r   rI   rI   rI   rJ   rz      s    
         
?
rz   )
      )
r   r   rv   rv   r   r   r   r      r   )
rv   rv   rv   r   r   r   r   r   r   r      c                  C   s  t  } tddD ]}|| _t| tt}t|| tt t	ttd d d g}t| t
|}t|| t
| t| t
t}t|| t
t t| t
|}t|| t
| qdd }t|d} t| t t dd}t|d	} t| tt dd}tt  t| ttd
d W d    n1 s&0    Y  td d d d tjf }t dd} t| |t}t dd} tt  t| |tdd W d    n1 s0    Y  d S )Nir   r~   c                 S   s
   t | tS rP   
isinstancerk   rg   rI   rI   rJ   <lambda>'      z&test_cross_val_score.<locals>.<lambda>check_Xr   cvcheck_yZsklearnscoringT)r}   Fraiseerror_score)rz   ranger|   r   rT   y2r
   rZ   r   column_stackX_sparser   tolistpytestraisesr   newaxis)clfr|   scoresmultioutput_y
list_checkX_3drI   rI   rJ   test_cross_val_score  s2    

0

r   c                  C   s@   t dd\} }tdd}t|dddgid}t|| |d	d
 d S )NTZ
return_X_yauto)ZgammaCrv   r   )Z
param_gridr   )n_jobs)r"   r4   rB   r   )rT   rm   r   ZgridrI   rI   rJ   test_cross_validate_many_jobs;  s    
r   c                  C   s  t dd\} }t }d}tjt|d, t|| |ttttfd W d    n1 sX0    Y  tjt|d& t|| |ttfd W d    n1 s0    Y  tjt|d d  t|| |dd W d    n1 s0    Y  tjt|d d  t|| |d	d W d    n1 s"0    Y  tjt|d( t|| |ttggd W d    n1 sj0    Y  d
}tjtdd" t|| |t	 d W d    n1 s0    Y  tjt|d  t|| |dd W d    n1 s0    Y  tt
}dtj }tjt|d  t|| ||d W d    n1 sB0    Y  tjt|d$ t|| |d|id W d    n1 s0    Y  tjtdd" tt | |dd W d    n1 s0    Y  d S )Nr   random_statez.*must be unique strings.*matchr   zEmpty list.*rI   zDuplicate.*)f1_micror   zB.*scoring is invalid.*Refer to the scoring glossary for details:.*zAn empty dictr   ziScoring failed. The score on this train-test partition for these parameters will be set to %f. Details: 
Zfooz#'mse' is not a valid scoring value.Zmse)r?   rz   r   r   r   r   r%   r)   r&   dictr(   r   nanwarnsUserWarningr4   )rT   rm   	estimatorerror_message_regexpZmulticlass_scorerwarning_messagerI   rI   rJ   )test_cross_validate_invalid_scoring_paramE  sF    $4.082004r   c                  C   sf   t dd\} }tdt fdt fg}t|| |dd}|d }t|tsLJ tdd |D sbJ d S )	NTr   imputer
classifier)return_estimatorr   c                 s   s   | ]}t |tV  qd S rP   )r   r9   ).0r   rI   rI   rJ   	<genexpr>  r   z7test_cross_validate_nested_estimator.<locals>.<genexpr>)r"   r9   r7   rz   r   r   rk   all)rT   rm   ZpipelineresultsZ
estimatorsrI   rI   rJ   $test_cross_validate_nested_estimator  s    r   c               	   C   s  t  } tddd\}}tdd}tddd\}}tddd}|||f|||ffD ]2\}}}	t|	dd}
t|	d	d}g }g }g }g }g }| ||D ]\}}t||| || }	|	|
|	|| ||  |	||	|| ||  |	|
|	|| ||  |	||	|| ||  |	|	 qt
|}t
|}t
|}t
|}t
|}|||||f}t|	||| t|	||| qPd S )
N   r   	n_samplesr   r   linearkernelr   neg_mean_squared_errorr   r2)r   r    r-   r?   r4   r,   splitr<   rO   appendr   array"check_cross_validate_single_metric!check_cross_validate_multi_metric)r   ZX_regZy_regregZX_clfZy_clfr   rT   rm   estZ
mse_scorerZ	r2_scorertrain_mse_scorestest_mse_scorestrain_r2_scorestest_r2_scoresfitted_estimatorstraintestr   rI   rI   rJ   test_cross_validate  sB    
 




r   c                 C   s<  |\}}}}}dD ]\}	}
|	r@t | ||ddd}t|d | nt | ||ddd}t|ts`J t||
kspJ t|d | |	rt | ||dgdd}t|d	 |d nt | ||dgdd}t|tsJ t||
ksJ t|d
 | qt | ||ddd}t|d D ].\}}t|j|| j t|j|| j qd S )N))Tr   )Fr   r   Tr   return_train_scoretrain_scoreF
test_scorer   train_r2test_r2)r   r   r   )	r   r	   r   r   r   	enumerater   Zcoef_Z
intercept_)r   rT   rm   r   r   r   r   r   r   r   Zdict_lenZmse_scores_dictZr2_scores_dictkr   rI   rI   rJ   r     sH    


r   c              	   C   s  |\}}}}}dd }	dt tdd|	f}
h d}|ddh}dD ]X}|
D ]L}|rt| |||d	d
}t|d | t|d | nt| |||dd
}t|tsJ t| |r|n|ksJ t|d | t|d | t	|d t
jksJ t	|d t
jksJ t	|d t
jks&J t	|d t
jks>J t
|d dksVJ t
|d dk snJ t
|d dksJ t
|d dk sNJ qNqDd S )Nc                 S   s"   |  |}t||t|| dS )Nr   r   )rU   r*   r+   )r   rT   rm   y_predrI   rI   rJ   custom_scorer  s    

z8check_cross_validate_multi_metric.<locals>.custom_scorerr   r   >   test_neg_mean_squared_error
score_timer   fit_timer   Ztrain_neg_mean_squared_error)TFTr   Fr   r   r   r   r   r   )r%   r*   unionr   r	   r   r   ri   keystyper   Zndarrayr   )r   rT   rm   r   r   r   r   r   r   r   Zall_scoringZkeys_sans_trainZkeys_with_trainr   r   
cv_resultsrI   rI   rJ   r     s^    	





r   c               	   C   s   t dddd\} }tdd}t tdt t g}d}|D ]}tjt|d  t	|| ||d	 W d    n1 sr0    Y  tjt|d  t
|| ||d	 W d    q:1 s0    Y  q:d S )
N   r   r   )r   	n_classesr   r   r   z*The 'groups' parameter should not be None.r   )r   rT   rm   r   )r?   r4   r   r   r   r   r   r   r   r   r   )rT   rm   r   Z	group_cvserror_messager   rI   rI   rJ   #test_cross_val_score_predict_groups@  s    
.r   z(ignore: Using or importing the ABCs fromc                     s   t t fg} z"ddlm}m} | ||f W n ty>   Y n0 | D ]N\  tt }} fdd}fdd}t||d}t	|||dd qDd S )	Nr   Series	DataFramec                    s
   t |  S rP   r   rg   InputFeatureTyperI   rJ   r   d  r   z-test_cross_val_score_pandas.<locals>.<lambda>c                    s
   t |  S rP   r  rg   
TargetTyperI   rJ   r   e  r   r   r   r   r   )
r   pandasr   r  r   ImportErrorrT   r   r   r   typesr   r  X_dfy_sercheck_dfcheck_seriesr   rI   r  r  rJ   test_cross_val_score_pandasV  s    
r  c                  C   s   t dd} t }|j|j }}td}t| |||d}td}g }|||D ]J\}}tjt	|t
d}	tjt	|t
d}
d|	|< d|
|< |||f qNt| |||d}t|| d S )Nr   r   r   r   dtyperv   )r4   r"   datatargetr   r   r   r   zerosr   boolr   r
   )svmirisrT   rm   ZkfoldZscores_indicesZcv_masksr   r   Z
mask_trainZ	mask_testZscores_masksrI   rI   rJ   test_cross_val_score_maskj  s    
r  c                  C   s   t dd} t }|j|j }}t||j}t| ||}t dd} t| ||}t|| t dd d} t| ||}t|| t dd} t	
t t| || W d    n1 s0    Y  t	
t  t| | | W d    n1 s0    Y  d S )Nprecomputedr   r   c                 S   s   t | |jS rP   )r   dotr   )rc   rm   rI   rI   rJ   r     r   z2test_cross_val_score_precomputed.<locals>.<lambda>)r4   r"   r  r  r   r  r   r   r	   r   r   r   r   )r  r  rT   rm   linear_kernelZscore_precomputedZscore_linearZscore_callablerI   rI   rJ    test_cross_val_score_precomputed}  s     




*r  c               	      s   t  } tjd }ttt}ttdgtdgtdgffdd}tt	d}d dt
  fdd	}t|t|d
| || |d}t| tt|d d S )Nr   rv   )r   rv   rM   r   *   Z42c                    s.   | j  ksJ | jksJ | jks*J d S rP   )r   r   r   )r   Z	DUMMY_INTZ	DUMMY_OBJZ	DUMMY_STRrI   rJ   assert_fit_params  s    z:test_cross_val_score_fit_params.<locals>.assert_fit_paramsr   )r   r   r   r   r   r   r   r   
fit_params)rz   rT   rM   r   r   r   rm   r   r   eyeobjectonesfullr   )r   r   r   ZW_sparser   r"  r$  rI   r!  rJ   test_cross_val_score_fit_params  s*    
$	
r)  c                     s~   t  } g   fdd}tjdd* t|}t| tt|dd}W d    n1 sR0    Y  t|g d t dkszJ d S )Nc                    s     | |f dS )Nr   )r   )Zy_testZ	y_predictZ_score_func_argsrI   rJ   
score_func  s    z3test_cross_val_score_score_func.<locals>.score_funcTrecordr   )r   r   )r   r   r   )	rz   warningscatch_warningsr%   r   rT   rm   r
   r   )r   r+  r   rZ   rI   r*  rJ   test_cross_val_score_score_func  s    0r0  c                  C   sH   G dd d} t t t|  t W d    n1 s:0    Y  d S )Nc                   @   s   e Zd ZdS )z4test_cross_val_score_errors.<locals>.BrokenEstimatorN)r]   r^   r_   rI   rI   rI   rJ   BrokenEstimator  s   r1  )r   r   	TypeErrorr   rT   )r1  rI   rI   rJ   test_cross_val_score_errors  s    r3  c                  C   s|   t  } tdd}t|| j| j}t|g dd t|| j| jdd}t|g dd t|| j| jdd}t|g dd d S )Nr   r   )
ףp=
?r   r4  r4  r   r   accuracyr   Zf1_weighted)r"   r4   r   r  r  r	   )r  r   r   Z	zo_scoresZ	f1_scoresrI   rI   rJ   3test_cross_val_score_with_score_func_classification  s    
r6  c            	      C   s   t ddddd\} }t }t|| |}t|g dd t|| |dd	}t|g dd t|| |d
d	}tg d}t||d tt}t|| ||d	}t|g dd d S )Nr   r   r   r   )r   
n_featuresn_informativer   )gGz?r4  r4  gGz?gq=
ףp?r   r   r   r   )g(\؇gzGIgGz&qg\(qg)\G)r    r-   r   r	   r   r   r%   r$   )	rT   rm   r   r   Z	r2_scoresZneg_mse_scoresZexpected_neg_mser   Z	ev_scoresrI   rI   rJ   /test_cross_val_score_with_score_func_regression  s    r9  c               
   C   s  t  } | j}t|}| j}tdd}td}t|||d|dd\}}}|dksRJ t|dd	 t|||d|dt	|j
d
d\}	}
}|	|ksJ ||ksJ tdd}td}t|||d|dt	|j
d
d\}	}
}|	|ksJ ||ksJ dd }t|}t|||d||d
d\}}
}t|dd t|dd ttt|d}t|||d|dd\}}}|dk snJ |dks|J d S )Nr   r   r   r   r5  )n_permutationsr   r   g?        rv   r   )r:  r   r   groupsr   c                 S   s"   | |k  | |k   | jd  S rD   )sumrM   )Zy_truer   rI   rI   rJ   custom_score+  s    z,test_permutation_score.<locals>.custom_scored   )r:  r   r   r   g(\?g{Gz?r   rt   皙?)r"   r  r   r  r4   r   r   r   r   r'  sizer%   modaranger   )r  rT   r   rm   r  r   rZ   r   ZpvalueZscore_group_Zpvalue_groupZ
svm_sparseZ	cv_sparser>  scorerrI   rI   rJ   test_permutation_score  sd    




rF  c                  C   st   t jdt jddd} t j| dd d f< t ddg| jd d }tdtd	t jd
fdt	 fg}t
|| | d S N   r  r   r~   r   r   rv   r   mean)ZstrategyZmissing_valuesr   )r   rC  float64r   r   repeatrM   r9   r7   rz   r   rT   rm   prI   rI   rJ   &test_permutation_test_score_allow_nans@  s    rN  c               	   C   s   t ddd} t dgd dgd  }tdgd}d}tjt|d	 t|| | W d    n1 sl0    Y  d
}tjt|d	* t|| |dt 	did W d    n1 s0    Y  t|| |dt 	did d S )Nr?  r   r   r   rv   r   rd   9Expected fit parameter\(s\) \['sample_weight'\] not seen.r   2Fit parameter sample_weight has length 1; expectedr#  )
r   rC  r   r   r   r   r   rj   r   r'  rT   rm   r   err_msgrI   rI   rJ   &test_permutation_test_score_fit_paramsN  s    *8rT  c                  C   st   t jdt jddd} t j| dd d f< t ddg| jd d }tdtd	t jd
fdt	 fg}t
|| | d S rG  )r   rC  rJ  r   r   rK  rM   r9   r7   rz   r   rL  rI   rI   rJ   test_cross_val_score_allow_nans]  s    rU  c            	      C   s  t ddgddgddgddgddgddgddgddgddgddgg
} t ddgddgddgddgddgddgddgddgddgddgg
}tdd	}ttd
d}ttdd}ttdd}t|| ||d}t|| ||d}t|| ||d}t|g d t|g d t|g d d S )Nr   r   r   r   rv   r~   )Zn_neighborsmicro)ZaverageZmacroZsamplesr   )rv   rt         ?rt   gUUUUUU?)rv   rt   rY  rt         ?)r   r   r3   r%   r)   r   r   )	rT   rm   r   Zscoring_microZscoring_macroZscoring_samplesZscore_microZscore_macroZscore_samplesrI   rI   rJ   test_cross_val_score_multilabelk  s4    >
r[  c                  C   s  t dd\} }t }t }t|}|| |D ].\}}|| | ||  || | ||< q0t|| ||d}t	|| t|| |}t
|t
|ksJ t }t|| ||d}t
|t
|ksJ |  }||t|k9 }t|}t|||}t	t
|t
| tt | }t
|t
|ks&J G dd d}	tt" t|| ||	 d W d    n1 sh0    Y  tdd\} }d}
tjt|
d, ttdd	| |d
tdd W d    n1 s0    Y  d S )NTr   r   c                   @   s   e Zd ZdddZdS )z%test_cross_val_predict.<locals>.BadCVNc                 s   s0   t dD ]"}tg dtg dfV  qd S )Nr   )r   rv   r   r   )r   r            )r   r   r   )rH   rT   rm   r<  irI   rI   rJ   r     s    z+test_cross_val_predict.<locals>.BadCV.split)NN)r]   r^   r_   r   rI   rI   rI   rJ   BadCV  s   r`  zNumber of classes in training fold \(2\) does not match total number of classes \(3\). Results may not be appropriate for your use case.r   	liblinearsolverr   r   methodr   )r!   r   r-   r   Z
zeros_liker   rO   rU   r   r	   r   r   copyZmedianr   r6   r   r   r   r"   r   RuntimeWarningr.   )rT   rm   r   r   Zpreds2r   r   predsZXspr`  r   rI   rI   rJ   test_cross_val_predict  sF    

2ri  c                  C   sp  t dddd\} }ttdd| |dd}|jd	ks6J td
d\} }ttdd| |dd}|jdkshJ | d d } |d d }d}tjt|d( tt | |dt	dd W d    n1 s0    Y  t
d
d\} }tddd}t|| |dd}|jdksJ t|}| | ||  } }d}tjt|d( t|| |t	dddd W d    n1 sb0    Y  d S )Nr   2   r   r   r   r   ra  rb  decision_functionre  )rj  Tr      r   r?  zOnly 1 class/es in training fold, but 2 in overall dataset. This is not supported for decision_function with imbalanced folds. To fix this, use a cross-validation technique resulting in properly stratified foldsr   rd  r   Zovo)r   Zdecision_function_shape)i  -   zOutput shape \(599L?, 21L?\) of decision_function does not match number of classes \(7\) in fold. Irregular decision_function .*r   n_splitsr   re  )r?   r   r.   rM   r"   r   r   r   r1   r   r#   r4   r   Zargsort)rT   rm   rh  r   r   Zindr   rI   rI   rJ   .test_cross_val_predict_decision_function_shape  s8    $
rt  c                  C   sl   t dddd\} }ttdd| |dd}|jd	ks6J td
d\} }ttdd| |dd}|jdkshJ d S )Nr   rj  r   rk  ra  rb  r   rm  rj  r   Tr   rn  r?   r   r.   rM   r"   rT   rm   rh  rI   rI   rJ   *test_cross_val_predict_predict_proba_shape  s    rx  c                  C   sl   t dddd\} }ttdd| |dd}|jd	ks6J td
d\} }ttdd| |dd}|jdkshJ d S )Nr   rj  r   rk  ra  rb  predict_log_probarm  ru  Tr   rn  rv  rw  rI   rI   rJ   .test_cross_val_predict_predict_log_proba_shape  s    rz  c            
      C   sb  t  } | j| j }}t|}t||d d d g}tddd}t|||}|jdksZJ t|||}|jdkstJ t|||}t	|jd t|||}t	|jd dd }t
|d	}t|| | }t
|d
}t||| }ttdd| | dd}ttdd|| dd}|d d d d tjf }dd }	t
|	d	}t|||}t	|jd d S )Nr~   Fr   )Zfit_interceptr   )ro  )ro  r   c                 S   s
   t | tS rP   r   rg   rI   rI   rJ   r   -  r   z4test_cross_val_predict_input_types.<locals>.<lambda>r   r   ra  rb  rl  rm  c                 S   s
   | j dkS )Nr   )r   rg   rI   rI   rJ   r   D  r   )r"   r  r  r   r   r   r-   r   rM   r
   r   r   r.   r   )
r  rT   rm   r   r   r   predictionsr   r   Zcheck_3drI   rI   rJ   "test_cross_val_predict_input_types  sF    


r|  c                     s   t t fg} z"ddlm}m} | ||f W n ty>   Y n0 | D ]N\  tt }} fdd}fdd}t||d}t	|||dd qDd S )	Nr   r   c                    s
   t |  S rP   r  rg   r  rI   rJ   r   X  r   z/test_cross_val_predict_pandas.<locals>.<lambda>c                    s
   t |  S rP   r  rg   r  rI   rJ   r   Y  r   r  r   r   )
r   r  r   r  r   r	  rT   r   r   r   r
  rI   r  rJ   test_cross_val_predict_pandasJ  s    
r}  c                  C   s   t ddddddd\} }d|d< tddd}tdd}t|| |\}}t|| ||d	d
}||d  d dkstJ t||d  d d df dksJ t||d  d d ddf dksJ t||d  dksJ t|j	ddt
|jdd d S )Nr?  r   r   rv   )r   r7  n_redundantr8  n_clusters_per_classr   ra  r   rc  rq  r   rs  Zaxis   )Zdecimal)r?   r.   r   rk   r   r   r   r   r	   r=  r'  rM   )rT   rm   r   r   r   r   Z
yhat_probarI   rI   rJ   !test_cross_val_predict_unbalanced^  s$    
	
&*r  c                  C   sf   t  } tjd}|dd}t| |d ddd}t|d d df | t| |d ddd}t|| d S )	Nr   r?  r   r   rU   )rm   r   re  r   r   )rz   r   randomRandomStateZrandr   r   )Zmock_classifierrngrT   Zy_hatZy_hat_probarI   rI   rJ   test_cross_val_predict_y_nonet  s    
r  c                  C   sX   t  } | j| j }}t }dtt|jd i}t||||dd}t	|t
d d S )Nr   r   r   )r$  r   )r"   r  r  rz   r   r   r%  rM   r   r
   r'  )r  rT   rm   r   r$  r|   rI   rI   rJ   &test_cross_val_score_sparse_fit_params  s    r  c                  C   s  d} d}t | ddddddd\}}t| |d |  }dD ]}tjdd	@}t|||t|d
tddd|dd\}}}	}
}W d    n1 s0    Y  t|dkrt	d|d j
 |jdksJ |	jdksJ |
jdksJ |jdksJ t|tddd t|jddtddd t|	jddtddd |
jdksNJ |jdks^J tjdd	<}t|||t|| dtddd|d\}}}W d    n1 s0    Y  t|dkrt	d|d j
 t|| t||	 q:d S )Nr   r   rv   r   r   r   r7  r8  r~  r   r  r   FTTr,  rq  皙?r   r   )r   rF   r>   Zreturn_timesUnexpected warning: %r)r   r   r   r  ffffff?rJ  rr  r   )r   rF   r>   )r?   rC   r.  r/  r   r   r   linspacer   RuntimeErrormessagerM   r
   r	   rI  r  rA   )r   rr  rT   rm   r   shuffle_trainwrF   train_scorestest_scoresZ	fit_timesZscore_timesZtrain_sizes2Ztrain_scores2Ztest_scores2rI   rI   rJ   test_learning_curve  sl    
	

,
r  c               
   C   s   t dddddddd\} }td}t|| d dtdd	d
d\}}}t|tddd
 t|jddtdd	d
 t|jddtdd	d
 d S )Nr   rv   r   r   r  r   r   r  r   r   )rm   r   rF   r  r  )r?   rC   r   r   r  r
   r	   rI  rT   rD  r   rF   r  r  rI   rI   rJ    test_learning_curve_unsupervised  s     
	r  c               	   C   s   t dddddddd\} }td}tj}t t_z6t|| |ddd\}}}W tj }tj  |t_ntj }tj  |t_0 d	|v sJ d S )
Nr   rv   r   r   r  r   r   )r   verbosez[learning_curve])r?   rC   sysstdoutr:   r   getvalueclose)rT   rm   r   Z
old_stdoutrF   r  r  outrI   rI   rJ   test_learning_curve_verbose  s.    
	




r  c               	   C   s`   t dddddddd\} }td}tt  t|| |dd W d    n1 sR0    Y  d S )Nr   rv   r   r  T)exploit_incremental_learningr?   rC   r   r   r   r   rT   rm   r   rI   rI   rJ   5test_learning_curve_incremental_learning_not_possible  s    

r  c                  C   s   t dddddddd\} }td}dD ]v}t|| |dd	td
dd|d\}}}t|tddd t|jddtddd t|jddtd
dd q&d S )Nr   rv   r   r   r  r   r  r   Tr  r   r   )r   r  rF   r>   r  r  r?   ra   r   r   r  r
   r	   rI  )rT   rm   r   r  rF   r  r  rI   rI   rJ   (test_learning_curve_incremental_learning  s.    
		r  c                  C   s   t dddddddd\} }td}t|| d ddtd	d
dd\}}}t|tddd t|jddtdd
d t|jddtd	d
d d S )Nr   rv   r   r   r  r   r   Tr  r   r   )rm   r   r  rF   r  r  r  r  rI   rI   rJ   5test_learning_curve_incremental_learning_unsupervised  s*    
	r  c            
   	   C   s   t dddddddd\} }tddd}tdd d	d
}t|| ||ddd\}}}t|| |d|d	d\}}}	t|| t|jdd|jdd t|jdd|	jdd d S )Nr   rv   r   r   r  r@  r   r   Fmax_itertolr>   r   T)rF   r   r  )r   rF   r  r  )r?   r   r  r0   r   r
   r	   rI  )
rT   rm   rF   r   train_sizes_inctrain_scores_inctest_scores_inctrain_sizes_batchtrain_scores_batchtest_scores_batchrI   rI   rJ   <test_learning_curve_batch_and_incremental_learning_are_equal6  sD    
		
r  c               	   C   sj  t dddddddd\} }td}tt& t|| |dddgd W d    n1 sX0    Y  tt& t|| |dd	d
gd W d    n1 s0    Y  tt& t|| |dddgd W d    n1 s0    Y  tt& t|| |dddgd W d    n1 s0    Y  tt& t|| |dddgd W d    n1 s\0    Y  d S )Nr   rv   r   r   r  r   r   r   rF   r;  r   r  g?   r  r  rI   rI   rJ   0test_learning_curve_n_sample_range_out_of_bounds]  s(    
	4446r  c                  C   s   t dddddddd\} }td}d}tjt|d2 t|| |dtdd	dd
\}}}W d    n1 sl0    Y  t|ddg d S )Nr   rv   r   r   r  zzRemoved duplicate entries from 'train_sizes'. Number of ticks will be less than the size of 'train_sizes': 2 instead of 3.r   gQ?r   r  )	r?   rC   r   r   rg  r   r   r  r
   )rT   rm   r   r   rF   rD  rI   rI   rJ   1test_learning_curve_remove_duplicate_sample_sizest  s"    
	*r  c               
   C   s   t dddddddd\} }td}tdd}t|| ||td	d
dd\}}}t|tddd t|jddtdd
d t|jddtd	d
d d S )Nr   rv   r   r   r  r   r   rq  r  r   r   r  r  r  )	r?   rC   r   r   r   r  r
   r	   rI  )rT   rm   r   r   rF   r  r  rI   rI   rJ   (test_learning_curve_with_boolean_indices  s"    
	
r  c                  C   s  t ddgddgddgddgd	d
gddgddgddgddgddgddgd	d
gddgddgddgg} t g d}t g d}tdd dd}tdd}t|| ||dt ddd|ddd	\}}}t|jddt g d t|jddt g d  t	t
2 t|| ||dt ddd|d!d" W d    n1 s>0    Y  t|| ||dt ddd|dddd#
\}}	}
t|	jdd|jdd t|
jdd|jdd d S )$Nrv   r   r   r   r   r\  r]  r^     r                       r   	   r   )rv   rv   rv   r   r   r   rv   rv   r   r   r   rv   r   r   r   )rv   rv   rv   rv   rv   rv   r   r   r   r   r   r   r   r   r   Fr  rq  333333?r   T)r   r   rF   r<  r>   r   r  )rY  r  Iq?)r  rZ  rZ  r   )r   r   rF   r<  r   )r   r   rF   r<  r>   r   r  )r   r   r0   r   r   r  r	   rI  r   r   r   )rT   rm   r<  r   r   r  r  r  r  r  r  rI   rI   rJ    test_learning_curve_with_shuffle  s    
&r  c               
   C   s   t ddd} t dgd dgd  }tdgd}d}tjt|d	  t|| |d
d W d    n1 sp0    Y  d}tjt|d	, t|| |d
dt 	did W d    n1 s0    Y  t|| |d
dt 	did d S )Nr?  r   r   r   rv   r   rO  rP  r   r   r   rQ  )r   r$  )
r   rC  r   r   r   r   r   rj   r   r'  rR  rI   rI   rJ   test_learning_curve_fit_params  s    .$r  c                  C   s  t dddddddd\} }tddg}d}tjt|d	0 t|| |d
dtddddd W d    n1 sn0    Y  d}tjt|d	< t|| |d
dtdddddtd
id W d    n1 s0    Y  t|| |d
dtdddddtdid d S )Nr   rv   r   r   r  r   r   rP  r   r   Tr  r   r   r   )r   r  rF   r   z2Fit parameter sample_weight has length 3; expected)r   r  rF   r   r$  )	r?   ra   r   r   rj   r   r   r  r'  )rT   rm   r   rS  rI   rI   rJ   3test_learning_curve_incremental_learning_fit_params  sV    
	$
$r  c               	   C   s   t dddddddd\} }tddd}tjdd*}tt | |d|dd	\}}W d    n1 sd0    Y  t|dkrtd
|d j	 t
|jdd| t
|jddd|  d S )Nr   rv   r   r  r   Tr,  ru   
param_nameparam_ranger   r  r  )r?   r   r  r.  r/  r   rs   r   r  r  r	   rI  )rT   rm   r  r  r  r  rI   rI   rJ   test_validation_curve/  s.    
	(r  c               	   C   sF   t dddddddd\} }tddd}tt | |d|dd\}}d S )Nr   rv   r   r  r   ru   r  )r?   r   r  r   rw   )rT   rm   r  rD  rI   rI   rJ   %test_validation_curve_clone_estimatorJ  s$    

r  c               
   C   s   d} d}t ddd\}}ttddd||dg dt|| d	d
}ttt|dd d f d  ttddd||dg dt|ddd
}ttt|dd d f d  ttddd||dg dt|dd
}tt	|t	| d S )Nr?  r   r   r   r   r   r   )r  r  r@  r@  r  r  )r   r   rv   r   r   Trr  r>   rq  )
r?   r   r4   rA   r	   r   Zvsplithstackr   r   )r   rr  rT   rm   Zscores1Zscores2Zscores3rI   rI   rJ   +test_validation_curve_cv_splits_consistency`  s<    

"

"

r  c                  C   s   t ddd} t dgd dgd  }tdgd}d}tjt|d	( t|| |d
g ddd W d    n1 sx0    Y  d}tjt|d	4 t|| |d
g dddt 	did W d    n1 s0    Y  t|| |d
g dddt 	did d S )Nr?  r   r   r   rv   r   rO  rP  r   Z	foo_param)rv   r   r   r   )r  r  r   rQ  )r  r  r   r$  )
r   rC  r   r   r   r   r   rj   r   r'  rR  rI   rI   rJ    test_validation_curve_fit_params  sB    $	$	r  c                  C   sv   t jd} t d}| | t|ds.J tt |ddrDJ d|d< t|drZJ tt |dfdrrJ d S )Nr   r?     )r   r  r  rC  r>   r   deleter  )r  rM  rI   rI   rJ   test_check_is_permutation  s    

r  c                  C   sj   t dddddd\} }t| }t|}ttdd}t|| |dd	}t|||dd	}| }t|| d S )
Nr   rv   FT)r   n_labelsZallow_unlabeledZreturn_indicatorr   r   r   r   r   )r@   r   r=   r4   r   Ztoarrayr	   )rT   rm   r   Zy_sparseZclassifrh  Zpreds_sparserI   rI   rJ   (test_cross_val_predict_sparse_prediction  s    
r  c           
   	   C   s   t ddd}|jdkr6|dkr(t|fn
t|df}n|j}t|}|||D ]6\}}t| || || } t	| ||| ||< qR||d |d |
dfD ]}	tt| ||	||d| qd	S )
z@Helper for tests of cross_val_predict with binary classificationr   Fr  rv   rl  r   strrd  N)r   r   r   rM   r   r  r   r<   rO   getattrastyper   r   )
r   rT   rm   re  r   	exp_shapeexpected_predictionsr   r   tgrI   rI   rJ   check_cross_val_predict_binary  s    
 
r  c              	   C   s   t ddd}ttjj}||dd}tjt|tt|f|| tjd}tj|dd\}}	|	||	D ]P\}
}t
| ||
 |	|
 } t| ||| }t|	|
 }||t||< qj||d	 |d
 |dfD ]}tt| ||||d| qdS )zDHelper for tests of cross_val_predict with multiclass classificationr   Fr  r   rl  ry  r   r  TZreturn_inverserv   r   r  rd  N)r   r   finforJ  minr(  r   ri   r   r   r<   rO   r  ix_r  r   r   )r   rT   rm   re  r   	float_mindefault_valuesr  rD  y_encr   r   
fold_predsZ
i_cols_fitr  rI   rI   rJ   "check_cross_val_predict_multiclass  s&    r  c                    s  t ddd}ttjj}||dd} jd }g }t|D ]`}	tt dd|	f }
|
dkrt|d	krtt|f}nt||
f}|	tj
||| tjd
 q< fddt jd D }tj|dd}|||D ]\}}t| || || } t| ||| }t|D ]^}	t|| dd|	f }||	 jdkrN||	 ||	 |< nt||}||	 ||	 |< qqԈ  d  d  dfD ]T}t| ||||d}t|t|ksJ tt|D ]}t|| ||  qqdS )zCheck the output of cross_val_predict for 2D targets using
    Estimators which provide a predictions as a list with one
    element per class.
    r   Fr  r   r  rv   Nr   rl  r  c                    s8   g | ]0}t j d d |f ddd d d t jf qS )NTr  rv   )r   r   r   )r   r_  rm   rI   rJ   
<listcomp>   s   z6check_cross_val_predict_multilabel.<locals>.<listcomp>r  r  rd  )r   r   r  rJ  r  rM   r   r   ri   r   r(  Zconcatenater   r<   rO   r  r   r   r  r  r   r   )r   rT   rm   re  r   r  r  Z	n_targetsZexpected_predsZi_colZn_classes_in_labelr  Z
y_enc_colsr  r   r   r  Z	fold_colsidxr  Zcv_predict_outputr_  rI   r  rJ   "check_cross_val_predict_multilabel  sD    

r  c                 C   s,   t ddd\}}dD ]}t| ||| qd S )Nr   r   )r   r   rl  r   ry  )r?   r  )r   rT   rm   re  rI   rI   rJ   *check_cross_val_predict_with_method_binary9  s    r  c                 C   sB   t  }|j|j }}t||dd\}}dD ]}t| ||| q*d S )Nr   r   r  )r"   r  r  r>   r  )r   r  rT   rm   re  rI   rI   rJ   .check_cross_val_predict_with_method_multiclassA  s
    r  c                   C   s    t tdd ttdd d S )Nra  rb  )r  r.   r  rI   rI   rI   rJ   "test_cross_val_predict_with_methodI  s    r  c                  C   sN   t  } | j| j }}t||dd\}}dD ]}tddd}t|||| q*d S )Nr   r   r  logr   )Zlossr   )r"   r  r  r>   r/   r  )r  rT   rm   re  r   rI   rI   rJ   &test_cross_val_predict_method_checkingP  s    r  c                  C   s`   t  } | j| j }}t||dd\}}ttddddddgid	d
}dD ]}t|||| qHd S )Nr   r   r   ra  r  r   r  rv   r   r   r  )r"   r  r  r>   rB   r.   r  )r  rT   rm   r   re  rI   rI   rJ   /test_gridsearchcv_cross_val_predict_with_method[  s    r  c                  C   sL   d} d}t | d|ddd\}}ttddd	}d
D ]}t||||d q2d S )Nr?  r   r   r   r   r   r  r   r7  r   ra  r   )rc  r   )r   rl  rm  )r@   r=   r.   r  )Zn_sampr   rT   rm   r   re  rI   rI   rJ   1test_cross_val_predict_with_method_multilabel_ovrf  s    

r  c                   @   s   e Zd Zdd ZdS )RFWithDecisionFunctionc                 C   s2   |  |}d}t|ts J |dd |D }|S )Nz?This helper should only be used on multioutput-multiclass tasksc                 S   s.   g | ]&}|j d  dkr&|dddf n|qS )rv   r   Nr~   r  )r   rM  rI   rI   rJ   r  |  r   z<RFWithDecisionFunction.decision_function.<locals>.<listcomp>)r   r   rk   )rH   rT   ZprobsmsgrI   rI   rJ   rl  x  s
    
z(RFWithDecisionFunction.decision_functionN)r]   r^   r_   rl  rI   rI   rI   rJ   r  t  s   r  c               	   C   s   d} t dd| ddd\}}|d d df  |d d df 7  < d	D ]R}tddd
}t * td t||||d W d    qB1 s0    Y  qBd S )Nr   r?  r   r   r   r  r   rv   r   ry  rl  Zn_estimatorsr   ignorerm  )r@   r  r.  r/  simplefilterr  )r   rT   rm   re  r   rI   rI   rJ   0test_cross_val_predict_with_method_multilabel_rf  s    

$

r  c               	   C   s   t jd} | jdddd}t g d}tdd}dD ]D}t ( td	 t	|||| W d    q81 sr0    Y  q8d S )
Nr   rv   )r  r   rA  )r   rv   r   r   rv   r   r   rv   r   r   rv   r   r   r   ra  rb  r  r  )
r   r  r  normalr   r.   r.  r/  r  r  )r  rT   rm   r   re  rI   rI   rJ   -test_cross_val_predict_with_method_rare_class  s    


r  c               	   C   s   t jd} | jdddd}t ddgddgddgddgddgg}dD ]R}tddd}t * td	 t	||||d
 W d    qH1 s0    Y  qHd S )Nr   rv   )r   r   r  r   )r   ry  r   r  r  rm  )
r   r  r  r  r   r  r.  r/  r  r  )r  rT   rm   re  r   rI   rI   rJ   ;test_cross_val_predict_with_method_multilabel_rf_rare_class  s    (

r  c                 C   s   t t||g}t||}|| |D ]|\}}	|| | ||  || |	 }
|dkrlt t|	|f}nt t|	|ft |jj	}|
|d d |j
f< |||	< q(|S )Nr   )r   r  r   r  r   rO   r(  r  r  r  Zclasses_)rT   rm   r   classesr   re  r  funcr   r   Zexpected_predictions_Zexp_pred_testrI   rI   rJ   get_expected_predictions  s    

r  c                  C   s  t ddd} t dd tdD }d}tdd}td	d}t }g d
}|D ]}tdd}t|| |||d}	t	| |||||}
t
|
|	 t|| |||d}	t	| |||||}
t
|
|	 tt tdddd}t|| |||d}	||}t	| |||||}
t
|
|	 qTd S )NrH  r?  r   c                 S   s   g | ]}|d  qS )r   rI   )r   rc   rI   rI   rJ   r    r   z7test_cross_val_predict_class_subset.<locals>.<listcomp>r   r   rq  r   r  ra  rb  rd  r   r   )r   rC  r   r   r   r   r8   r.   r   r  r	   r>   rK  Zfit_transform)rT   rm   r  Zkfold3Zkfold4lemethodsre  r   r{  r  rI   rI   rJ   #test_cross_val_predict_class_subset  s4    





r   c                     sb  t  } | j| j }}t }tjddd}|d |  tj	|j
tjdtj	|j
ddtjd zt||| fd	d
d tt( t|||fdd
d W d    n1 s0    Y  W d\ tdD ]:}zt|j
 W  q^W q ty   td Y q0 qnPd\ tdD ]<}zt|j
 W  q\W n tyV   td Y n0 q0 d S )NwbF)moder  s   Hello world!!!!!r  rI   r)rM   r  r  c                    s    S rP   rI   r   rT   rm   )rZ   rI   rJ   r     r   z#test_score_memmap.<locals>.<lambda>r   c                    s    S rP   rI   r  )r   rI   rJ   r     r   )NNr   r   )r"   r  r  rz   tempfileZNamedTemporaryFilewriter  r   ZmemmapnamerJ  r   r   r   r   r   osunlinkWindowsErrorr   )r  rT   rm   r   ZtfrD  rI   )rZ   r   rJ   test_score_memmap  s4    
8r  c                     s   t t fg} z"ddlm}m} | ||f W n ty>   Y n0 | D ]^\ t }|j|j }} || }} fdd}fdd}	t	||	d}
t
|
|| qDd S )Nr   r   c                    s
   t |  S rP   r  rg   r  rI   rJ   r     r   z4test_permutation_test_score_pandas.<locals>.<lambda>c                    s
   t |  S rP   r  rg   r  rI   rJ   r     r   r  )r   r  r   r  r   r	  r"   r  r  r   r   )r  r   r  r  rT   rm   r  r  r  r  r   rI   r  rJ   "test_permutation_test_score_pandas  s    
r  c               
   C   s  t t j} tdd}td}| |d t d d dd d g	}ddi}tjtdd t	|i | W d    n1 sr0    Y  t
d	}tjt|d  t| |d
dd W d    n1 s0    Y  tjt|d  t| |d
dd W d    n1 s0    Y  tjt|d" t| ||d
dd W d    n1 s:0    Y  tjt|d* t| ||dt jgd
dd W d    n1 s0    Y  |  dksJ d S )Nrv   r   r  r   r   r   z%Failing classifier failed as requiredr   zerror_score must be the string 'raise' or a numeric value. (Hint: if using 'raise', please make sure that it has been spelled correctly.)r   zunvalid-stringr   r   Z	parameter)r  r  r   r   r;  )r   FAILING_PARAMETERr   rC  r'  r   r   r   r   r   reescaper   r   r   r   rZ   )failing_clfrT   rm   fit_and_score_argsfit_and_score_kwargsr   rI   rI   rJ   test_fit_and_score_failing  s6    

,..2&
r  c                  C   s|   t ddd\} }tddd}tt | \}}|| |t ||dg}dddd d	d
}t|i |}|d |d ksxJ d S )Nr   r   r   r   r   r?  r  )r  r  T)
parametersr$  Zreturn_parametersr  )r?   r4   nextr   r   r   r   )rT   rm   r   r   r   r  r  resultrI   rI   rJ   test_fit_and_score_workingF  s    r  r   c                 C   s   t t j}tdd}td}|||g}d| d}tjd|d  dtjd	}tj	t
|d
 t|i | W d    n1 s0    Y  d}td|d  d| tj}tj	t
|d
 t|i | W d    n1 s0    Y  d S )Nrv   r   r  r]  r  zh7 fits failed.+total of 7.+The score on these train-test partitions for these parameters will be set to r   rh   )flagsr   z1ValueError: Failing classifier failed as requiredzMThe score on these train-test partitions for these parameters will be set to z.+)r   r  r   rC  r'  r  compileDOTALLr   r   r   r   )r   r  rT   rm   Zcross_validate_argsZcross_validate_kwargsr   r   rI   rI   rJ   )test_cross_validate_failing_fits_warningsU  s0    



,r  c                 C   s   t |d S rP   )r   )r   rT   rm   	error_msgrI   rI   rJ   _failing_scorerx  s    r  zignore:lbfgs failed to converger   c              	   C   s   t dd\}}tdd||}d}tt|d}| dkr|tjt|d$ t|||d	|| d
 W d    q1 sp0    Y  nVd|  }tj	t
|d. t|||d	|| d
}t||  W d    n1 s0    Y  d S )NTr   r   r  "This scorer is supposed to fail!!!r  r   r   r   )r   r   r   [Scoring failed. The score on this train-test partition for these parameters will be set to )r"   r.   rO   r   r  r   r   r   r   r   r   r   )r   rT   rm   r   r  failing_scorerwarning_msgr   rI   rI   rJ   #test_cross_val_score_failing_scorer|  s$    &r%  r   TFwith_multimetricc              
   C   s  t dd\}}tdd||}d}tt|d}|r@||d}n|}| dkrtjt|d	& t|||d
||| d W d    n1 s0    Y  nnd|  }	tj	t
|	d	F t|||d
||| d}
|
D ]}d|v rt|
| |  qW d    n1 s0    Y  d S )NTr   r   r  r   r!  )Zscore_1Zscore_2r   r   r   )r   r   r   r   r"  r   )r"   r.   rO   r   r  r   r   r   r   r   r   r   )r   r   r&  rT   rm   r   r  r#  r   r$  r   ro   rI   rI   rJ   "test_cross_validate_failing_scorer  sF    	&	r'  c                 C   s   dS )Ng;pΈ^@rI   )r_  jr   rI   rI   rJ   three_params_scorer  s    r)  z:train_score, scorer, verbose, split_prg, cdt_prg, expectedr   )rv   r   r   rv   zS\[CV\] END .................................................... total time=   0.\ds)Zsc1Zsc2r   ze\[CV 2/3\] END  sc1: \(train=3.421, test=3.421\) sc2: \(train=3.421, test=3.421\) total time=   0.\dsr   zW\[CV 2/3; 1/1\] END ....... sc1: \(test=3.421\) sc2: \(test=3.421\) total time=   0.\dsc              	   C   s   t ddd\}}tddd}	tt |\}
}|	||||
||d d g	}|||d}t|i | |  \}}|d}t|dkrt	||d	 sJ nt	||d sJ d S )
Nr   r   r   r   r   )r   Zsplit_progressZcandidate_progress
r   rv   )
r?   r4   r  r   r   r   Z
readouterrr   r  r   )Zcapsysr   rE  r  Z	split_prgZcdt_prgZexpectedrT   rm   r   r   r   r  r  r  rD  ZoutlinesrI   rI   rJ   test_fit_and_score_verbosity  s    #
r,  c                  C   sZ   d} dd }d d d |g}t jt| d  t|dtji W d    n1 sL0    Y  d S )Nz&scoring must return a number, got Nonec                 S   s   d S rP   rI   )r   ZX_testrI   rI   rJ   two_params_scorer
	  s    z%test_score.<locals>.two_params_scorerr   r   )r   r   r   r   r   r   )r   r-  r  rI   rI   rJ   r   	  s
    r   c                  C   sj   dd } t dddd\}}tdd}||| t|||d| d	}g d
}|D ]}d||v sNJ qNd S )Nc                 S   s2   |  |}t||}|d |d |d |d dS )N)r   r   r*  )rv   r   )rv   rv   Ztnfpfntp)rU   r'   )r   rT   rm   r   cmrI   rI   rJ   r   	  s    

zPtest_callable_multimetric_confusion_matrix_cross_validate.<locals>.custom_scorer(   r   r   )r   r7  r   r   r   )r   r   r.  ztest_{})r?   r5   rO   r   r   )r   rT   rm   r   r   Zscore_namesr  rI   rI   rJ   9test_callable_multimetric_confusion_matrix_cross_validate	  s    
r4  c                  C   s   t  } | j| j }}t||j}tdd}td  }t	|||dd W d    n1 s\0    Y  |rnJ G dd dt}|dd}d}tjt
|d  t	|||dd W d    n1 s0    Y  d S )	Nr  r   r   r   c                   @   s   e Zd Zdd ZdS )z1test_validation_pairwise.<locals>.IncorrectTagSVMc                 S   s   ddiS )NZpairwiseFrI   )rH   rI   rI   rJ   
_more_tags1	  s    z<test_validation_pairwise.<locals>.IncorrectTagSVM._more_tagsN)r]   r^   r_   r5  rI   rI   rI   rJ   IncorrectTagSVM0	  s   r6  z;_pairwise was deprecated in 0.24 and will be removed in 1.1r   )r"   r  r  r   r  r   r4   r   r   r   FutureWarning)r  rT   rm   r  r  r-  r6  r  rI   rI   rJ   test_validation_pairwise#	  s    
.
r8  )r`   r  r  r  r  r.  	functoolsr   timer   r   Znumpyr   Zscipy.sparser   r   Zsklearn.exceptionsr   Z)sklearn.model_selection.tests.test_searchr   Zsklearn.utils._testingr   r	   r
   r   Zsklearn.utils._mockingr   r   Zsklearn.utils.validationr   Zsklearn.model_selectionr   r   r   r   r   r   r   r   r   r   r   r   r   r   Z#sklearn.model_selection._validationr   r   r   Zsklearn.datasetsr    r!   r"   r#   Zsklearn.metricsr$   r%   r&   r'   r(   r)   r*   r+   r,   Zsklearn.linear_modelr-   r.   r/   r0   r1   Zsklearn.ensembler2   Zsklearn.neighborsr3   Zsklearn.svmr4   r5   Zsklearn.clusterr6   Zsklearn.imputer7   Zsklearn.preprocessingr8   Zsklearn.pipeliner9   ior:   Zsklearn.baser;   r<   Zsklearn.multiclassr=   Zsklearn.utilsr>   r?   r@   Z$sklearn.model_selection.tests.commonrA   rB   r
  	NameErrorrC   ra   rs   rw   rz   r'  rT   r   r   rm   r   r%  r   r   r   r   r   r   r   r   r   Zmarkfilterwarningsr  r  r  r)  r0  r3  r6  r9  rF  rN  rT  rU  r[  ri  rt  rx  rz  r|  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  Zparametrizer   r  r  r%  r'  r)  r,  r   r4  r8  rI   rI   rI   rJ   <module>   s~  
W
*
@/0I

#B845

	='N0.&4)

)
"

0

 
