常见的机器学习算法（一）线性回归

偏执的太偏执、 2023-02-11 15:26 14阅读 0赞

线性回归模型可以使用两种方法进行训练：

① 梯度下降法；

② 正态方程(封闭形式解)：![format_png][]

其中 X 是一个矩阵，其形式为![format_png 1][]，包含所有训练样本的维度信息。而正态方程需要计算![format_png 2][]的转置。这个操作的计算复杂度介于![format_png 3][]）和![format_png 4][]之间，而这取决于所选择的实现方法。因此，如果训练集中数据的特征数量很大，那么使用正态方程训练的过程将变得非常缓慢。

**（一）用梯度下降法训练**

1.用0 (或小的随机值)来初始化权重向量和偏置量

2.计算输入的特征与权重值的线性组合，这可以通过矢量化和矢量传播来对所有训练样本进行处理：![format_png 5][]

其中 X 是所有训练样本的维度矩阵，其形式为![format_png 6][]；· 表示点积。

3.用均方误差计算训练集上的损失：

![format_png 7][]

4.对每个参数，计算其对损失函数的偏导数：

![format_png 8][]

所有偏导数的梯度计算如下：

![format_png 9][]

5.更新权重向量和偏置量：

![format_png 10][]

其中，![format_png 11][]表示学习率。

import numpy as np
    import matplotlib.pyplot as plt
    'train_test_split将矩阵随机划分为训练子集和测试子集，并返回划分好的训练集测试集样本和训练集测试集标签'
    from sklearn.model_selection import train_test_split
    
    np.random.seed(123)
    x = 2 * np.random.rand(500, 1)
    y = 5 + 3 * x + np.random.rand(500, 1)
    '数据集打出来看一下'
    fig = plt.figure(figsize=(8,6))
    plt.scatter(x, y)
    plt.title('Dataset')
    plt.xlabel('First feature')
    plt.ylabel('Second feature')
    plt.show()
    
    x_train, x_test, y_train, y_test = train_test_split(x, y)
    print('Shape x_train: ', x_train.shape)
    print('Shape y_train: ', y_train.shape)
    print('Shape x_test: ', x_test.shape)
    print('Shape y_test: ', y_test.shape)
    
    class LinearRegression:
        def __init__(self):
            pass
    
        '用梯度下降方法训练'
        def train_gradient_descent(self, x, y, l_r=0.01, n_iters=100):
            n_samples, n_features = x.shape
            self.weight = np.zeros(shape=(n_features, 1))
            # print('weight: ', self.weight)
            self.bias = 0
            costs = []
            for i in range(n_iters):
                y_predict = np.dot(x, self.weight) + self.bias
                cost = (1 / n_samples) * np.sum((y_predict - y)**2)
                costs.append(cost)
    
                if i % 100 == 0:
                    print('Cost at iteration {}:{}'.format(i, cost))
    
                d_dw = (2 / n_samples) * np.dot(x.T, (y_predict - y))
                d_db = (2 / n_samples) * np.sum((y_predict - y))
    
                self.weight = self.weight - l_r * d_dw
                self.bias = self.bias - l_r * d_db
            return self.weight, self.bias, costs
    
        def predict(self, x):
            return np.dot(x, self.weight) + self.bias
    
    regressor = LinearRegression()
    
    w_trained, b_trained, costs = regressor.train_gradient_descent(
        x_train, y_train, l_r=0.005, n_iters=600)
    fig = plt.figure(figsize=(8,6))
    plt.plot(np.arange(600), costs)
    plt.title('Development of cost during training')
    plt.xlabel('Number of iterations')
    plt.ylabel('Cost')
    plt.show()
    
    '测试'
    n_samples, _ = x_train.shape
    n_samples_test, _ = x_test.shape
    y_p_train = regressor.predict(x_train)
    y_p_test = regressor.predict(x_test)
    error_train = (1 / n_samples) * np.sum((y_p_train - y_train)**2)
    error_test = (1 / n_samples_test) * np.sum((y_p_test - y_test)**2)
    print('Error on training set: ', np.round(error_train, 4))
    print('Error on test set: ', np.round(error_test, 4))
    
    '可视化测试预测'
    fig = plt.figure(figsize=(8,6))
    plt.scatter(x_train, y_train)
    plt.scatter(x_test, y_p_test)
    plt.xlabel('First feature')
    plt.ylabel('Second feature')
    plt.show()

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 1][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 2][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 3][]

**(二) 用正规方程（normal equation）训练**

import numpy as np
    import matplotlib.pyplot as plt
    'train_test_split将矩阵随机划分为训练子集和测试子集，并返回划分好的训练集测试集样本和训练集测试集标签'
    from sklearn.model_selection import train_test_split
    
    np.random.seed(123)
    x = 2 * np.random.rand(500, 1)
    y = 5 + 3 * x + np.random.rand(500, 1)
    '数据集打出来看一下'
    fig = plt.figure(figsize=(8,6))
    plt.scatter(x, y)
    plt.title('Dataset')
    plt.xlabel('First feature')
    plt.ylabel('Second feature')
    plt.show()
    
    x_train, x_test, y_train, y_test = train_test_split(x, y)
    print('Shape x_train: ', x_train.shape)
    print('Shape y_train: ', y_train.shape)
    print('Shape x_test: ', x_test.shape)
    print('Shape y_test: ', y_test.shape)
    
    class LinearRegression:
        def __init__(self):
            pass
    
        '用正规方程normal equation方法做训练'
        def train_normal_equation(self,x,y):
            self.weight = np.dot(np.dot(np.linalg.inv(np.dot(x.T, x)), x.T), y)
            self.bias = 0
            return self.weight, self.bias
    
        def predict(self, x):
            return np.dot(x, self.weight) + self.bias
    
    n_samples, _ = x_train.shape
    n_samples_test, _ = x_test.shape
    x_b_train = np.c_[np.ones((n_samples)), x_train]#np.c_按行连接两个矩阵，把两矩阵左右相加，要求行数相等
    x_b_test = np.c_[np.ones((n_samples_test)), x_test]
    reg_normal = LinearRegression()
    w_trained = reg_normal.train_normal_equation(x_b_train, y_train)
    
    '测试'
    y_p_train = reg_normal.predict(x_b_train)
    y_p_test = reg_normal.predict(x_b_test)
    error_train = (1 / n_samples) * np.sum((y_p_train - y_train)**2)
    error_test = (1 / n_samples_test) * np.sum((y_p_test - y_test)**2)
    print('Error on training set: ', np.round(error_train, 4))
    print('Error on test set: ', np.round(error_test, 4))
    
    '可视化测试预测'
    fig = plt.figure(figsize=(8,6))
    plt.scatter(x_train, y_train)
    plt.scatter(x_test, y_p_test)
    plt.xlabel('First feature')
    plt.ylabel('Second feature')
    plt.show()

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 4][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 5][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 6][]

**（三）直接调用sklearn的API**

from sklearn.linear_model import LinearRegression         # 线性回归 #
    module = LinearRegression()
    module.fit(x, y)
    module.score(x, y)
    module.predict(test)

完整如下：

import numpy as np
    import matplotlib.pyplot as plt
    from sklearn.linear_model import LinearRegression#直接调用API
    from sklearn.model_selection import train_test_split
    
    np.random.seed(123)
    x = 2 * np.random.rand(500, 1)
    y = 5 + 3 * x + np.random.rand(500, 1)
    '数据集打出来看一下'
    fig = plt.figure(figsize=(8, 6))
    plt.scatter(x, y)
    plt.title('Dataset')
    plt.xlabel('First feature')
    plt.ylabel('Second feature')
    plt.show()
    
    '原始数据x,y经train_test_split处理，得到用于训练和验证的数据集x_train, x_test, y_train, y_test'
    x_train, x_test, y_train, y_test = train_test_split(x, y)
    print('Shape x_train: ', x_train.shape)
    print('Shape y_train: ', y_train.shape)
    print('Shape x_test: ', x_test.shape)
    print('Shape y_test: ', y_test.shape)
    
    module = LinearRegression()
    module.fit(x_test, y_test)#fit拟合处理
    
    '测试'
    n_samples, _ = x_train.shape
    n_samples_test, _ = x_test.shape
    y_p_train = module.predict(x_train)
    y_p_test = module.predict(x_test)
    error_train = (1 / n_samples) * np.sum((y_p_train - y_train) ** 2)
    error_test = (1 / n_samples_test) * np.sum((y_p_test - y_test) ** 2)
    print('Error on training set: ', np.round(error_train, 4))
    print('Error on test set: ', np.round(error_test, 4))
    
    '可视化'
    fig = plt.figure(figsize=(8, 6))
    plt.scatter(x_train, y_train)
    plt.scatter(x_test, y_p_test)
    plt.xlabel('First feature')
    plt.ylabel('Second feature')
    plt.show()

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 7][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 8][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 9][]

[format_png]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE11OFRPaDFkQzI3S2ljQ1FEZFo1Q0s5anEzTGs5V2dIQUJsQ1VDQUZmOG51UGhwaWJlV1BoNFdjUS82NDA_d3hfZm10PXBuZw?x-oss-process=image/format,png
[format_png 1]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1OTjRRdDRmSDN1ZGYyR3d3aWNxeFhpY2VzYlBPUWFYMFpRMjRRYXNzMEZyR0xzanlIdnozbUMxZy82NDA_d3hfZm10PXBuZw?x-oss-process=image/format,png
[format_png 2]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1BU3cxVkJBSmt5TVdmeDllWHRRNkE2QnJxSHhDcUNzcG9rTzNpYnEzVjJkaWFaZm1UYU1VN0Rjdy82NDA_d3hfZm10PXBuZw?x-oss-process=image/format,png
[format_png 3]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1senRYUFV4UWM3aGJVaWF4NkhRV2lidXNKaGF0VmVWNHF5WWliU1JvS2FVTkdBQlFvbWJ1NDBpYU9BLzY0MD93eF9mbXQ9cG5n?x-oss-process=image/format,png
[format_png 4]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE11OGZMdzBpYjlEY1dzaWE5SG5YaEhYaWJTNXRoU2lhTk44OHdxN01XV002bnB2UlFoUmQyQzRuTWdnLzY0MD93eF9mbXQ9cG5n?x-oss-process=image/format,png
[format_png 5]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1zcFdSNVZKQjZpYzVJeWViMlREYUFVUlRGa0UxZ2RRcjQ3YjVRRTl3bDdXME1BRFFraWJUbXNrdy82NDA_d3hfZm10PXBuZw?x-oss-process=image/format,png
[format_png 6]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1pYUJyYzN6YlZ6bDFaN3ZZOGJ5NEpvbTZjQVMwcXNEbzRQRmUxcnByZlFJdGpMYjdSbEtUMUd3LzY0MD93eF9mbXQ9cG5n?x-oss-process=image/format,png
[format_png 7]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1iZjdqVkJlUmxTYVRUQ2EwTmZHcE03dEhRY1FEZmY1bFM4V3dpY0hOWEQyMU4waEtvWDk0NmJBLzY0MD93eF9mbXQ9cG5n?x-oss-process=image/format,png
[format_png 8]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1ldDc4aWFHSE01WmlhOEtsRDV5SlhidEJpYkIwY3FBdFRnQm9hbWZab0pNaWF4RHJpY1YzZlVFZjlTQS82NDA_d3hfZm10PXBuZw?x-oss-process=image/format,png
[format_png 9]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1kMGVHZ0xGUVhzMXQwa01vWllGRVBCZmlhRlp1M09ORWpRYkZGN2VjVFpGSWVpYVNGaWFydjVnRncvNjQwP3d4X2ZtdD1wbmc?x-oss-process=image/format,png
[format_png 10]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1jR2lheE9LUjNaTzBTdWZsV0d6V0RMMTVpYTltM1hXbDVidmJzbmhDa2U4R3pwVmhxaWFnRzhpYTFRLzY0MD93eF9mbXQ9cG5n?x-oss-process=image/format,png
[format_png 11]: https://imgconvert.csdnimg.cn/aHR0cHM6Ly9tbWJpei5xcGljLmNuL21tYml6X3BuZy9CblNORWFmaWNGQVl6QjdDeXhTU0picVVEbkRnZTFqRE1zREtBVmljYldnOEFKU2lidGN3TUljQlBNNXg0UG9TVncxSXhxU3VOcXZQTUpJYzlkbUlmREhQZy82NDA_d3hfZm10PXBuZw?x-oss-process=image/format,png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200524112027869.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/20200524112108303.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 2]: https://img-blog.csdnimg.cn/20200524112130719.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 3]: https://img-blog.csdnimg.cn/20200524112200482.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 4]: https://img-blog.csdnimg.cn/20200524113056170.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 5]: https://img-blog.csdnimg.cn/20200524113121891.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 6]: https://img-blog.csdnimg.cn/20200524113145376.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 7]: https://img-blog.csdnimg.cn/2020052515261711.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 8]: https://img-blog.csdnimg.cn/20200525152639841.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2_size_16_color_FFFFFF_t_70 9]: https://img-blog.csdnimg.cn/20200525152700775.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM5OTM4NjY2,size_16,color_FFFFFF,t_70