a
    ;Za                     @   s  d dl Z d dlZd dlmZ d dlmZ d dlmZm	Z	 d dl
mZ d dlmZmZ d dlmZmZmZmZ d dlmZ d d	lmZ d d
lmZmZmZ d dlmZ d dlmZmZ d dl m!Z! d dl"m#Z#m$Z$m%Z% d dl&m'Z'm(Z( d dl)m*Z* d dl+m,Z, d dl-m.Z.m/Z/ d dlm0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl9m:Z:m;Z; d dl9m<Z<m=Z= d dl9m>Z>m?Z? e j@dddd ZAe jBCdd d!ge jBCd"d#d$gd%d& ZDe jBCd"d#d$gd'd( ZEe jBCd"d#d$gd)d* ZFd+d, ZGe jBCd"d#d$gd-d. ZHe jBCdd d!ge jBCd"d#d$gd/d0 ZIe jBCdd d!ge jBCd"d#d$gd1d2 ZJe jBCdd d!ge jBCd"d#d$ge jBCd3eKd4d5d6 ZLd7d8 ZMd9d: ZNe jBCdd d!gd;d< ZOd=d> ZPd?d@ ZQe jBCd"d#d$gdAdB ZRe jBCd"d#d$gdCdD ZSe jBCd"d#d$gdEdF ZTeeUdGe jBCdHejVWdIXdJdKd4ejVWdIXdJdKd4dLgdMdN ZYe j@dOdP ZZe j@dQdR Z[dSdT Z\e jBCdUe ]e,dVdWd4e ]e,dVdWdXgdYdZ Z^d[d\ Z_d]d^ Z`d_d` Zae j@dddadb Zbe j@dddcdd Zcdedf Zde jBCdgdhdigdjdk Zee jBCdldKdmge jBCdndodpgdqdr Zfdsdt Zge jBCdudvdwgdxdy Zhdzd{ Zie jBCdgdhdigd|d} Zjd~d Zke jBCdd d!ge jBCd"d#d$gdd Zle jBCdd d!ge jBCd"d#d$gdd ZmdS )    N)assert_allclose)sparse)BaseEstimatorclone)DummyClassifier)LeaveOneOuttrain_test_split)assert_array_almost_equalassert_almost_equalassert_array_equalignore_warnings)softmax)NotFittedError)make_classification
make_blobs	load_iris)LabelEncoder)KFoldcross_val_predict)MultinomialNB)RandomForestClassifierRandomForestRegressorVotingClassifier)LogisticRegressionLinearRegression)DecisionTreeClassifier)	LinearSVC)Pipelinemake_pipeline)StandardScaler)IsotonicRegression)DictVectorizer)SimpleImputer)brier_score_loss)CalibratedClassifierCV_CalibratedClassifier)_sigmoid_calibration_SigmoidCalibration)calibration_curveCalibrationDisplaymodule)Zscopec                  C   s   t dddd\} }| |fS )N      *   	n_samples
n_featuresrandom_state)r   )Xy r4   =lib/python3.9/site-packages/sklearn/tests/test_calibration.pydata,   s    r6   methodsigmoidisotonicensembleTFc                 C   s8  d}| \}}t jjddj|jd}|| 8 }|d | |d | |d |   }}}	||d  ||d   }
}t j|||	d}||
d d df }t	||jd |d}t
t ||| W d    n1 s0    Y  ||
ft|t|
ffD ]0\}}t	||d|d	}|j|||	d ||d d df }t||t||ksZJ |j||d |	d ||d d df }t|| |j|d
| d |	d ||d d df }t|| |j||d d
 |	d ||d d df }|dkrt|d|  n"t||t|d d
 |ks J q d S )Nd   r-   seedsizesample_weight   cvr:      r7   rD   r:      r8   )nprandomRandomStateuniformr?   minr   fitpredict_probar$   pytestraises
ValueErrorr   
csr_matrixr#   r	   )r6   r7   r:   r/   r2   r3   rA   X_trainy_trainsw_trainX_testy_testclfprob_pos_clfcal_clfZthis_X_trainthis_X_testprob_pos_cal_clfZprob_pos_cal_clf_relabeledr4   r4   r5   test_calibration2   sB    (*




r]   c                 C   sV   | \}}t  }t|d|d}tt ||| W d    n1 sH0    Y  d S )NZfoor7   r:   )r   r$   rO   rP   rQ   rM   )r6   r:   r2   r3   rX   Zclf_invalid_methodr4   r4   r5   test_calibration_bad_methodo   s
    r_   c                 C   sP   | \}}t t |d}tt ||| W d    n1 sB0    Y  d S )N)r:   )r$   r   rO   rP   RuntimeErrorrM   )r6   r:   r2   r3   Zclf_base_regressorr4   r4   r5   test_calibration_regressory   s    ra   c                 C   s<   | \}}t dd}||| |jd j}t|ts8J d S NrG   rD   r   )r$   rM   calibrated_classifiers_base_estimator
isinstancer   )r6   r2   r3   	calib_clfZbase_estr4   r4   r5   "test_calibration_default_estimator   s
    
rh   c                 C   sp   | \}}d}t |d}t||d}t|jt s2J |jj|ksBJ ||| |rV|nd}t|j|kslJ d S )NrE   )n_splitsrC   rB   )r   r$   rf   rD   ri   rM   lenrd   )r6   r:   r2   r3   ZsplitsZkfoldrg   Zexpected_n_clfr4   r4   r5   test_calibration_cv_splitter   s    
rk   c                 C   s   d}| \}}t jjddjt|d}|d | |d | |d |   }}}	||d  }
tdd}t|||d}|j|||	d ||
}||| ||
}t j	
|| }|dksJ d S )	Nr;   r-   r<   r>   r1   r^   r@   皙?)rH   rI   rJ   rK   rj   r   r$   rM   rN   ZlinalgZnorm)r6   r7   r:   r/   r2   r3   rA   rS   rT   rU   rV   re   calibrated_clfZprobs_with_swZprobs_without_swZdiffr4   r4   r5   test_sample_weight   s    (


ro   c                 C   s   | \}}t ||dd\}}}}tdd}	t|	|d|d}
|
|| |
|}t|	|d|d}||| ||}t|| dS )zTest parallel calibrationr-   rl   rG   )r7   Zn_jobsr:   rB   N)r   r   r$   rM   rN   r   )r6   r7   r:   r2   r3   rS   rV   rT   rW   re   Zcal_clf_parallelZprobs_parallelZcal_clf_sequentialZprobs_sequentialr4   r4   r5   test_parallel_execution   s    


rp   r=   rG   c                 C   s  dd }t dd}tdd|ddd	\}}d
||d
k< t|jd }|d d d
 |d d d
  }}	|dd d
 |dd d
  }
}|||	 t|| d|d}|||	 ||
}ttj	|ddt
t|
 d||
|  k rdk sn J ||
|d||
| ksJ ||t||
|d}||||d}|d| k sLJ tddd}|||	 ||
}||||d}t|| d|d}|||	 ||
}||||d}|d| k sJ d S )Nc                 S   s*   t ||  }t || d |jd  S )NrG   r   )rH   Zeyesumshape)y_trueZ
proba_pred	n_classesZY_onehotr4   r4   r5   multiclass_brier   s    z5test_calibration_multiclass.<locals>.multiclass_brier   rl   i  r;   
         .@r/   r0   r1   ZcentersZcluster_stdrG   r   rB   rE   rF   Zaxisg?gffffff?)rt   皙?   r-   )n_estimatorsr1   )r   r   rH   uniquerr   rM   r$   rN   r   rq   onesrj   Zscorer   decision_functionr   )r7   r:   r=   ru   rX   r2   r3   rt   rS   rT   rV   rW   rZ   probasZuncalibrated_brierZcalibrated_brierZ	clf_probsZcal_clf_probsr4   r4   r5   test_calibration_multiclass   s<    



""

r   c                  C   sh   G dd d} t dddddd\}}t ||}|  }t||g|jd}||}t|d	|j  d S )
Nc                   @   s   e Zd Zdd ZdS )z9test_calibration_zero_probability.<locals>.ZeroCalibratorc                 S   s   t |jd S )Nr   )rH   zerosrr   selfr2   r4   r4   r5   predict  s    zAtest_calibration_zero_probability.<locals>.ZeroCalibrator.predictN)__name__
__module____qualname__r   r4   r4   r4   r5   ZeroCalibrator  s   r   2   rw   rv   rx   ry   )re   calibratorsclasses      ?)r   r   rM   r%   classes_rN   r   Z
n_classes_)r   r2   r3   rX   
calibratorrZ   r   r4   r4   r5   !test_calibration_zero_probability  s    



r   c               
   C   s  d} t d|  ddd\}}tjjddj|jd}|| 8 }|d|  |d|  |d|    }}}|| d	|   || d	|   || d	|     }}}	|d	|  d |d	|  d  }
}t }t|d
d}t	
t ||| W d   n1 s0    Y  |||| ||
dddf }||
ft|t|
ffD ]\}}dD ]}t||d
d}|	dfD ]v}|j|||d ||}||}|dddf }t|tddgtj|dd  t||t||ksjJ qjqPqDdS )z*Test calibration for prefitted classifiersr      r,   r-   r.   r<   r>   NrG   prefitrc   rB   )r9   r8   )r7   rD   r@   r   rz   )r   rH   rI   rJ   rK   r?   rL   r   r$   rO   rP   r   rM   rN   r   rR   r   r   arrayZargmaxr#   )r/   r2   r3   rA   rS   rT   rU   ZX_calibZy_calibZsw_calibrV   rW   rX   Z	unfit_clfrY   Zthis_X_calibr[   r7   rZ   swy_proby_predr\   r4   r4   r5   test_calibration_prefit,  s<    (
"*

"
r   c                 C   s   | \}}t dd}t||ddd}||| ||}t|||ddd}|dkr^td	d
}nt }||| ||| ||}	||	}
t	|d d df |
 d S )Nrv   rl   r   FrF   r   )rD   r7   r9   Zclip)Zout_of_boundsrB   )
r   r$   rM   rN   r   r    r'   r   r   r   )r6   r7   r2   r3   rX   rZ   Z
cal_probasZunbiased_predsr   Zclf_dfZmanual_probasr4   r4   r5   test_calibration_ensemble_false[  s    



r   c                  C   s   t g d} t g d}t ddg}t|t| |d ddt |d |  |d    }t | || }t||d	 t	t
( t t | | f| W d
   n1 s0    Y  d
S )z0Test calibration values with Platt sigmoid model)rE   r   )rB   r   gj=ɿgY90(?r   r   r   rB   r,   N)rH   r   r	   r&   Zexpr'   rM   r   rO   rP   rQ   vstack)ZexFZexYZAB_lin_libsvmZlin_probZsk_probr4   r4   r5   test_sigmoid_calibrationt  s    "r   c            
      C   s  t g d} t g d}t| |dd\}}t| |d ddd\}}t|t|ksZJ t|dksjJ t|ddg t|d	d
g t|| t|| tt" tdgdgdd W d   n1 s0    Y  t g d}t g d}t||ddd\}}	t|t|	ksJ t|dks.J t|ddg t|	d	dg tt t||dd W d   n1 sz0    Y  dS )z Check calibration_curve function)r   r   r   rB   rB   rB   )        rm   皙?皙??r   rG   )n_binsT)r   	normalizer   rB   rm   r   r{   gF)r   N)r   r   r   r   rB   rB   )r   rm   r   g      ?r   r   quantiler   strategygUUUUUU?r   Z
percentile)r   )rH   r   r(   rj   r
   rO   rP   rQ   )
rs   r   	prob_true	prob_predZprob_true_unnormalizedZprob_pred_unnormalizedZy_true2Zy_pred2Zprob_true_quantileZprob_pred_quantiler4   r4   r5   test_calibration_curve  s2    


0
r   c                 C   sf   t dddddd\}}tj|d< tdt fdtd	d
fg}t|dd| d}||| || dS )z$Test that calibration can accept nanrw   rG   r   r-   )r/   r0   Zn_informativeZn_redundantr1   )r   r   ZimputerZrfrB   )r}   r9   )rD   r7   r:   N)	r   rH   nanr   r"   r   r$   rM   r   )r:   r2   r3   rX   Zclf_cr4   r4   r5   test_calibration_nan_imputer  s    


r   c                 C   sl   d}t dd|d\}}tddd}t|dt | d	}||| ||}t|jd
dt	|j
d  d S )NrG   rw   rE   )r/   r0   rt   r   rv   Cr1   r8   rF   rB   rz   r   )r   r   r$   r   rM   rN   r	   rq   rH   r   rr   )r:   Znum_classesr2   r3   rX   Zclf_probZprobsr4   r4   r5   test_calibration_prob_sum  s    

r   c                 C   s   t jdd}t d}tddd}t|dt | d}||| t|j	D ]\}}|
|}| rt|d d |f t t| t |d d d |f dksJ t |d d |d	 d f dksJ qLt |d	|jd  sLJ qLd S )
Nrw   rE   r   rv   r   r8   rF   r   rB   )rH   rI   randnZaranger   r$   r   rM   	enumeraterd   rN   r   r   rj   allZallcloserr   )r:   r2   r3   rX   rZ   iZcalibrated_classifierZprobar4   r4   r5   test_calibration_less_classes  s    


 "(r   )categoryr2   r-      rE   r,   c                 C   s2   g d}G dd dt }t| }|| | dS )z;Test that calibration accepts n-dimensional arrays as input)rB   r   r   rB   rB   r   rB   rB   r   r   rB   r   r   rB   r   c                   @   s    e Zd ZdZdd Zdd ZdS )z>test_calibration_accepts_ndarray.<locals>.MockTensorClassifierz*A toy estimator that accepts tensor inputsc                 S   s   t || _| S )N)rH   r~   r   )r   r2   r3   r4   r4   r5   rM     s    zBtest_calibration_accepts_ndarray.<locals>.MockTensorClassifier.fitc                 S   s   | |jd djddS )Nr   r   rB   rz   )Zreshaperr   rq   r   r4   r4   r5   r     s    zPtest_calibration_accepts_ndarray.<locals>.MockTensorClassifier.decision_functionN)r   r   r   __doc__rM   r   r4   r4   r4   r5   MockTensorClassifier  s   r   N)r   r$   rM   )r2   r3   r   rn   r4   r4   r5    test_calibration_accepts_ndarray  s    

r   c                  C   s,   dddddddddg} g d}| |fS )NZNYZadult)stateZageZTXVTchild)rB   r   rB   r4   )	dict_dataZtext_labelsr4   r4   r5   r      s    r   c                 C   s,   | \}}t dt fdt fg}|||S )NZ
vectorizerrX   )r   r!   r   rM   )r   r2   r3   Zpipeline_prefitr4   r4   r5   dict_data_pipeline  s
    r   c                 C   sf   | \}}|}t |dd}||| t|j|j t|dr@J t|drNJ || || dS )aR  Test that calibration works in prefit pipeline with transformer

    `X` is not array-like, sparse matrix or dataframe at the start.
    See https://github.com/scikit-learn/scikit-learn/issues/8710

    Also test it can predict without running into validation errors.
    See https://github.com/scikit-learn/scikit-learn/issues/19637
    r   rc   n_features_in_N)r$   rM   r   r   hasattrr   rN   )r   r   r2   r3   rX   rg   r4   r4   r5   test_calibration_dict_pipeline  s    	
r   zclf, cvrB   r   r   c                 C   s   t ddddd\}}|dkr(| ||} t| |d}||| |dkrht|j| j |j| jksJ n.t |j}t|j| |j|jd ksJ d S )	Nrw   rE   rG   rv   r/   r0   rt   r1   r   rc   rB   )r   rM   r$   r   r   r   r   rr   )rX   rD   r2   r3   rg   r   r4   r4   r5   test_calibration_attributes.  s    	r   c                  C   s   t ddddd\} }tdd| |}t|dd	}d
}tjt|d, || d d d df | W d    n1 sv0    Y  d S )Nrw   rE   rG   rv   r   rB   r   r   rc   zAX has 3 features, but LinearSVC is expecting 5 features as input.matchr   )r   r   rM   r$   rO   rP   rQ   )r2   r3   rX   rg   msgr4   r4   r5   2test_calibration_inconsistent_prefit_n_features_inF  s    r   c                 C   s|   | \}}t dd||}tt |jd j}W d    n1 sF0    Y  t||jd jD ]\}}||u sbJ qbd S rb   )	r$   rM   rO   ZwarnsFutureWarningrd   Zcalibrators_zipr   )r6   r2   r3   rg   r   Zclf1Zclf2r4   r4   r5   )test_calibrated_classifier_cv_deprecationS  s    *r   c                  C   sV   t ddddd\} }tdd tdD d	d
}|| | t|dd}|| | d S )Nrw   rE   rG   rv   r   c                 S   s   g | ]}d t | t fqS )lr)strr   ).0r   r4   r4   r5   
<listcomp>h      z5test_calibration_votingclassifier.<locals>.<listcomp>r   Zsoft)Z
estimatorsZvotingr   )re   rD   )r   r   rangerM   r$   )r2   r3   Zvoterg   r4   r4   r5   !test_calibration_votingclassifierb  s    r   c                   C   s
   t ddS )NTZ
return_X_y)r   r4   r4   r4   r5   	iris_datar  s    r   c                 C   s    | \}}||dk  ||dk  fS )NrG   r4   )r   r2   r3   r4   r4   r5   iris_data_binaryw  s    r   c           
      C   s   |\}}|\}}t  ||}d}tjt|d t||| W d    n1 sT0    Y  t ||}	d}tjt|d t|	|| W d    n1 s0    Y  t }	tt	 t|	|| W d    n1 s0    Y  d S )Nz)'estimator' should be a fitted classifierr   z/response method predict_proba is not defined in)
r   rM   rO   rP   rQ   r)   from_estimatorr   r   r   )
pyplotr   r   r2   r3   ZX_binaryZy_binaryZregr   rX   r4   r4   r5   #test_calibration_display_validation}  s    ,,r   constructor_namer   from_predictionsc                 C   s   |\}}t  }||| ||}|dkrnd}tjt|d t||| W d    q1 sb0    Y  n>d}tjt|d t|| W d    n1 s0    Y  d S )Nr   z"to be a binary classifier, but gotr   z-y should be a 1d array, got an array of shape)	r   rM   rN   rO   rP   rQ   r)   r   r   )r   r   r   r2   r3   rX   r   r   r4   r4   r5   #test_calibration_display_non_binary  s    
.r   r   rw   r   rK   r   c                 C   sR  |\}}t  ||}tj|||||dd}||d d df }t||||d\}	}
t|j|	 t|j|
 t|j	| |j
dksJ dd l}t|j|jjsJ |j dksJ t|j|jjsJ t|j|jjsJ |j dksJ |j dksJ dd	g}|j  }t|t|ks0J |D ]}| |v s4J q4d S )
Nr   )r   r   ZalpharB   r   r   r   zMean predicted probabilityzFraction of positivesPerfectly calibrated)r   rM   r)   r   rN   r(   r   r   r   r   estimator_nameZ
matplotlibrf   Zline_linesZLine2DZ	get_alphaax_ZaxesZAxesZfigure_ZfigureZFigureZ
get_xlabelZ
get_ylabel
get_legend	get_textsrj   get_text)r   r   r   r   r2   r3   r   vizr   r   r   Zmplexpected_legend_labelslegend_labelslabelsr4   r4   r5    test_calibration_display_compute  s2    
r   c           	      C   sz   |\}}t t t }||| t|||}|jdg}|j 	 }t
|t
|ks\J |D ]}| |v s`J q`d S )Nr   )r   r   r   rM   r)   r   r   r   r   r   rj   r   )	r   r   r2   r3   rX   r   r   r   r   r4   r4   r5   $test_plot_calibration_curve_pipeline  s    
r   zname, expected_label)NZ_line1)my_estr   c           
      C   s   t g d}t g d}t g }t||||d}|  |d u rJg n|g}|d |j  }t|t|ks|J |D ]}	|		 |v sJ qd S )Nr   rB   rB   r   r   r   r   g?r   r   )
rH   r   r)   plotappendr   r   r   rj   r   )
r   nameZexpected_labelr   r   r   r   r   r   r   r4   r4   r5   'test_calibration_display_default_labels  s    

r   c           	      C   s   t g d}t g d}t g }d}t||||d}|j|ksHJ d}|j|d |dg}|j  }t|t|ksJ |D ]}|	 |v sJ qd S )Nr   r   zname oner   zname twor   r   )
rH   r   r)   r   r   r   r   r   rj   r   )	r   r   r   r   r   r   r   r   r   r4   r4   r5   )test_calibration_display_label_class_plot  s    
r   c                 C   s  |\}}d}t  ||}||d d df }tt| }| dkrL|||fn||f}	||	d|i}
|
j|kspJ |d |
  |dg}|
j	 
 }t|t|ksJ |D ]}| |v sJ q|d d}|
j|d t|t|ksJ |D ]}| |v sJ qd S )	Nzmy hand-crafted namerB   r   r   r   r   Zanother_namer   )r   rM   rN   getattrr)   r   closer   r   r   r   rj   r   )r   r   r   r2   r3   Zclf_namerX   r   constructorparamsr   r   r   r   r4   r4   r5   ,test_calibration_display_name_multiple_calls  s*    


r   c           	      C   sj   |\}}t  ||}t ||}t|||}tj||||jd}|j d }|ddksfJ d S )N)ZaxrB   r   )r   rM   r   r)   r   r   Zget_legend_handles_labelscount)	r   r   r2   r3   r   Zdtr   Zviz2r   r4   r4   r5   !test_calibration_display_ref_line$  s    r   c                 C   sP  t dd\}}t |}|dd |dd  }}t|d }tj|jd d |jd f|jd}||dddddf< ||dddddf< tj|jd d |jd}||ddd< ||ddd< t }t	|| |dd	}t
|}	|	j|||d
 ||| t|	j|jD ]\}
}t|
jj|jj q|	|}||}t|| dS )zrCheck that passing repeating twice the dataset `X` is equivalent to
    passing a `sample_weight` with a factor 2.Tr   Nr;   rG   r   rB   )dtyper7   r:   rD   r@   )r   r   fit_transformrH   Z	ones_liker   rr   r   r   r$   r   rM   r   rd   r   re   coef_rN   )r7   r:   r2   r3   rA   ZX_twiceZy_twicere   calibrated_clf_without_weightscalibrated_clf_with_weightsest_with_weightsest_without_weightsy_pred_with_weightsy_pred_without_weightsr4   r4   r5   ?test_calibrated_classifier_cv_double_sample_weights_equivalence1  s>    $

r  c                 C   s  t dd\}}t |}t|dd |dd f}t|dd |dd f}t|}d|ddd< t }t|| |dd	}t	|}|j
|||d
 |
|ddd |ddd  t|j|jD ]\}}	t|jj|	jj q||}
||}t|
| dS )z|Check that passing removing some sample from the dataset `X` is
    equivalent to passing a `sample_weight` with a factor 0.Tr   N(   r   Z   rB   rG   r   r@   )r   r   r   rH   r   ZhstackZ
zeros_liker   r$   r   rM   r   rd   r   re   r   rN   )r7   r:   r2   r3   rA   re   r   r   r   r   r  r  r4   r4   r5   >test_calibrated_classifier_cv_zeros_sample_weights_equivalencec  s6    
 

r  )nrO   ZnumpyrH   Znumpy.testingr   Zscipyr   Zsklearn.baser   r   Zsklearn.dummyr   Zsklearn.model_selectionr   r   Zsklearn.utils._testingr	   r
   r   r   Zsklearn.utils.extmathr   Zsklearn.exceptionsr   Zsklearn.datasetsr   r   r   Zsklearn.preprocessingr   r   r   Zsklearn.naive_bayesr   Zsklearn.ensembler   r   r   Zsklearn.linear_modelr   r   Zsklearn.treer   Zsklearn.svmr   Zsklearn.pipeliner   r   r   Zsklearn.isotonicr    Zsklearn.feature_extractionr!   Zsklearn.imputer"   Zsklearn.metricsr#   Zsklearn.calibrationr$   r%   r&   r'   r(   r)   Zfixturer6   ZmarkZparametrizer]   r_   ra   rh   rk   ro   rp   r   r   r   r   r   r   r   r   r   r   r   rI   rJ   r   r   r   r   r   Zparamr   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r4   r4   r4   r5   <module>   s   

;
	


=/
%











)

"0