机器学习（模型评估——各类交叉验证方法使用）

冷不防 2024-04-17 23:19 17阅读 0赞

一：K折交叉验证

from sklearn.model_selection import cross_val_score
    from sklearn.datasets import load_iris
    from sklearn.linear_model import LogisticRegression
    iris = load_iris()
    logreg = LogisticRegression()
    scores = cross_val_score(logreg, iris.data, iris.target, cv=3)
    print("Cross-validation scores: {}".format(scores))

![在这里插入图片描述][20190902201951510.png]  
注意有另一种类型[StratifiedKFold][]

#添加随机化
    from sklearn.model_selection import KFold
    kfold = KFold(n_splits=3, shuffle=True, random_state=0)
    cross_val_score(logreg, iris.data, iris.target, cv=kfold)

二：Leave-one-out cross-validation

#适用小数据集，大数据集很耗时间
    from sklearn.model_selection import LeaveOneOut
    loo = LeaveOneOut()
    scores = cross_val_score(logreg, iris.data, iris.target, cv=loo)
    print("Number of cv iterations: ", len(scores))
    print("Mean accuracy: {:.2f}".format(scores.mean()))

三：Shuffle-split cross-validation

from sklearn.model_selection import ShuffleSplit
    shuffle_split = ShuffleSplit(test_size=.5, train_size=.5, n_splits=10)
    scores = cross_val_score(logreg, iris.data, iris.target, cv=shuffle_split)

四：Cross-validation with groups

from sklearn.model_selection import GroupKFold
    # create synthetic dataset
    X, y = make_blobs(n_samples=12, random_state=0)
    # assume the first three samples belong to the same group,
    # then the next four, etc.
    groups = [0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3]
    scores = cross_val_score(logreg, X, y, groups, cv=GroupKFold(n_splits=3))

五：Simple Grid Search（寻找模型的最佳参数设置）

# naive grid search implementation
    from sklearn.svm import SVC
    X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, random_state=0)
    print("Size of training set: {} size of test set: {}".format(
    X_train.shape[0], X_test.shape[0]))
    best_score = 0
    for gamma in [0.001, 0.01, 0.1, 1, 10, 100]:
    	for C in [0.001, 0.01, 0.1, 1, 10, 100]:
    		# for each combination of parameters, train an SVC
    		svm = SVC(gamma=gamma, C=C)
    		svm.fit(X_train, y_train)
    		# evaluate the SVC on the test set
    		score = svm.score(X_test, y_test)
    		# if we got a better score, store the score and parameters
    		if score > best_score:
    		best_score = score
    		best_parameters = {'C': C, 'gamma': gamma}
    print("Best score: {:.2f}".format(best_score))
    print("Best parameters: {}".format(best_parameters))

![在这里插入图片描述][20190903090029885.png]  
六：使用train，validation，test去验证模型

from sklearn.svm import SVC
    # split data into train+validation set and test set
    X_trainval, X_test, y_trainval, y_test = train_test_split(
    iris.data, iris.target, random_state=0)
    # split train+validation set into training and validation sets
    X_train, X_valid, y_train, y_valid = train_test_split(
    X_trainval, y_trainval, random_state=1)
    print("Size of training set: {} size of validation set: {} size of test set:"
    " {}\n".format(X_train.shape[0], X_valid.shape[0], X_test.shape[0]))
    best_score = 0
    for gamma in [0.001, 0.01, 0.1, 1, 10, 100]:
    	for C in [0.001, 0.01, 0.1, 1, 10, 100]:
    		# for each combination of parameters, train an SVC
    		svm = SVC(gamma=gamma, C=C)
    		svm.fit(X_train, y_train)
    		# evaluate the SVC on the test set
    		score = svm.score(X_valid, y_valid)
    		# if we got a better score, store the score and parameters
    		if score > best_score:
    		best_score = score
    		best_parameters = {'C': C, 'gamma': gamma}
    # rebuild a model on the combined training and validation set,
    # and evaluate it on the test set
    svm = SVC(**best_parameters)
    svm.fit(X_trainval, y_trainval)
    test_score = svm.score(X_test, y_test)
    print("Best score on validation set: {:.2f}".format(best_score))
    print("Best parameters: ", best_parameters)
    print("Test set score with best parameters: {:.2f}".format(test_score))

![在这里插入图片描述][20190903092153142.png]  
七：Grid Search with Cross-Validation

for gamma in [0.001, 0.01, 0.1, 1, 10, 100]:
    	for C in [0.001, 0.01, 0.1, 1, 10, 100]:
    		# for each combination of parameters,
    		# train an SVC
    		svm = SVC(gamma=gamma, C=C)
    		# perform cross-validation
    		scores = cross_val_score(svm, X_trainval, y_trainval, cv=5)
    		# compute mean cross-validation accuracy
    		score = np.mean(scores)
    		# if we got a better score, store the score and parameters
    		if score > best_score:
    		best_score = score
    		best_parameters = {'C': C, 'gamma': gamma}
    # rebuild a model on the combined training and validation set
    svm = SVC(**best_parameters)
    svm.fit(X_trainval, y_trainval)

八：GridSearchCV（直接实现类似上面七的操作）

from sklearn.model_selection import GridSearchCV
    from sklearn.svm import SVC
    
    param_grid = {'C': [0.001, 0.01, 0.1, 1, 10, 100],
    'gamma': [0.001, 0.01, 0.1, 1, 10, 100]}
    
    grid_search = GridSearchCV(SVC(), param_grid, cv=5)#注意，cv参数可以像cross-validation，设置各种划分类型
    X_train, X_test, y_train, y_test = train_test_split(
    	iris.data, iris.target, random_state=0)
    grid_search.fit(X_train, y_train)
    print("Test set score: {:.2f}".format(grid_search.score(X_test, y_test)))
    
    print("Best parameters: {}".format(grid_search.best_params_))
    print("Best cross-validation score: {:.2f}".format(grid_search.best_score_)) #mean cross-validation accuracy

![在这里插入图片描述][20190903094920694.png]

print("Best estimator:\n{}".format(grid_search.best_estimator_))

![在这里插入图片描述][20190903095132419.png]

#查看上面运行的详细结果
    import pandas as pd
    # convert to DataFrame
    results = pd.DataFrame(grid_search.cv_results_)
    # show the first 5 rows
    display(results.head())

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70]  
九：Search over spaces that are not grids

param_grid = [{'kernel': ['rbf'],
    'C': [0.001, 0.01, 0.1, 1, 10, 100],
    'gamma': [0.001, 0.01, 0.1, 1, 10, 100]},
    {'kernel': ['linear'],
    'C': [0.001, 0.01, 0.1, 1, 10, 100]}]
    grid_search = GridSearchCV(SVC(), param_grid, cv=5)
    grid_search.fit(X_train, y_train)
    print("Best parameters: {}".format(grid_search.best_params_))
    print("Best cross-validation score: {:.2f}".format(grid_search.best_score_))

![在这里插入图片描述][20190903100149928.png]  
十：Nested cross-validation（嵌套的交叉验证）  
耗时非常长，因为param\_grid里的参数组合有36种，所以下面代码共测试了36 \* 5 \* 5 = 900个模型

scores = cross_val_score(GridSearchCV(SVC(), param_grid, cv=5),
    iris.data, iris.target, cv=5)
    print("Cross-validation scores: ", scores)
    print("Mean cross-validation score: ", scores.mean())

![在这里插入图片描述][20190903101022328.png]

#该函数实现了上面一样的操作
    def nested_cv(X, y, inner_cv, outer_cv, Classifier, parameter_grid):
    	outer_scores = []
    	# for each split of the data in the outer cross-validation
    		# (split method returns indices)
    		for training_samples, test_samples in outer_cv.split(X, y):
    			# find best parameter using inner cross-validation
    			best_parms = {}
    			best_score = -np.inf
    			# iterate over parameters
    			for parameters in parameter_grid:
    				# accumulate score over inner splits
    				cv_scores = []
    				# iterate over inner cross-validation
    				for inner_train, inner_test in inner_cv.split(
    					X[training_samples], y[training_samples]):
    					# build classifier given parameters and training data
    					clf = Classifier(**parameters)
    					clf.fit(X[inner_train], y[inner_train])
    					# evaluate on inner test set
    					score = clf.score(X[inner_test], y[inner_test])
    					cv_scores.append(score)
    				# compute mean score over inner folds
    				mean_score = np.mean(cv_scores)
    				if mean_score > best_score:
    					# if better than so far, remember parameters
    					best_score = mean_score
    					best_params = parameters
    		# build classifier on best parameters using outer training set
    		clf = Classifier(**best_params)
    		clf.fit(X[training_samples], y[training_samples])
    		# evaluate
    		outer_scores.append(clf.score(X[test_samples], y[test_samples]))
    	return np.array(outer_scores)

十一：活用上面各个验证方法中的n\_jobs参数，利用CPU并行提升运行速度

[20190902201951510.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/d91380eaa9df475c98963fad760a86db.png
[StratifiedKFold]: https://scikit-learn.org/stable/modules/cross_validation.html#stratified-k-fold
[20190903090029885.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/07bb0ea9dcb743d0b88ab2bd33bcb9eb.png
[20190903092153142.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/73701bf04f9e465cbcb69fe045cd16da.png
[20190903094920694.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/29417c0cf5474963b716215d9996ac19.png
[20190903095132419.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/55ebf78fda9048ca913b607537f7ead5.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6bGRt_size_16_color_FFFFFF_t_70]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/d0dbcb8a8b9046bda42df62fd7106d1e.png
[20190903100149928.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/cb1f7fc44e0e41749d664443abd3d703.png
[20190903101022328.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/17/b67bb0420fda4fc1a4ce86c39b1ee0e6.png