PyTorch DataLoader 报错 “DataLoader worker exited unexpectedly“ 的解决方案

╰半橙微兮° 2024-03-24 19:33 90阅读 0赞

> 注意：博主没有重写d2l的源代码文件，而是创建了一个新的python文件，并重写了该方法。

## 一、代码运行日志 ##

C:\Users\Administrator\anaconda3\envs\limu\python.exe G:/PyCharmProjects/limu-d2l/ch03/softmax_regression.py
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 116, in spawn_main
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 116, in spawn_main
        exitcode = _main(fd, parent_sentinel)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 125, in _main
        Traceback (most recent call last):
    exitcode = _main(fd, parent_sentinel)
      File "<string>", line 1, in <module>
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 125, in _main
    Traceback (most recent call last):
          File "<string>", line 1, in <module>
    prepare(preparation_data)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 236, in prepare
        prepare(preparation_data)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 236, in prepare
        _fixup_main_from_path(data['init_main_from_path'])
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
        _fixup_main_from_path(data['init_main_from_path'])
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
        main_content = runpy.run_path(main_path,
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 265, in run_path
        main_content = runpy.run_path(main_path,
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 265, in run_path
        return _run_module_code(code, init_globals, run_name,
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 97, in _run_module_code
        return _run_module_code(code, init_globals, run_name,
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 97, in _run_module_code
        _run_code(code, mod_globals, init_globals,
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 87, in _run_code
        _run_code(code, mod_globals, init_globals,
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 87, in _run_code
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 116, in spawn_main
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 116, in spawn_main
        exec(code, run_globals)
          File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 97, in <module>
    exec(code, run_globals)
      File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 97, in <module>
        train_ch03(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
              File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 81, in train_ch03
    exitcode = _main(fd, parent_sentinel)
    train_ch03(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 125, in _main
      File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 81, in train_ch03
        exitcode = _main(fd, parent_sentinel)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 125, in _main
            train_loss, train_acc = train_epoch_ch03(net, train_iter, loss, updater)train_loss, train_acc = train_epoch_ch03(net, train_iter, loss, updater)
    
      File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 64, in train_epoch_ch03
      File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 64, in train_epoch_ch03
        prepare(preparation_data)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 236, in prepare
        prepare(preparation_data)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 236, in prepare
            for X, y in train_iter:for X, y in train_iter:
    
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 441, in __iter__
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 441, in __iter__
            _fixup_main_from_path(data['init_main_from_path'])_fixup_main_from_path(data['init_main_from_path'])
    
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
            main_content = runpy.run_path(main_path,main_content = runpy.run_path(main_path,
    
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 265, in run_path
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 265, in run_path
        return self._get_iterator()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 388, in _get_iterator
        return self._get_iterator()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 388, in _get_iterator
            return _run_module_code(code, init_globals, run_name,return _run_module_code(code, init_globals, run_name,
    
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 97, in _run_module_code
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 97, in _run_module_code
        return _MultiProcessingDataLoaderIter(self)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 1042, in __init__
        return _MultiProcessingDataLoaderIter(self)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 1042, in __init__
        _run_code(code, mod_globals, init_globals,    
    _run_code(code, mod_globals, init_globals,
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 87, in _run_code
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\runpy.py", line 87, in _run_code
        exec(code, run_globals)    
    exec(code, run_globals)
      File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 97, in <module>
      File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 97, in <module>
        train_ch03(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
          File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 81, in train_ch03
    train_ch03(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
      File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 81, in train_ch03
        w.start()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\process.py", line 121, in start
        w.start()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\process.py", line 121, in start
            train_loss, train_acc = train_epoch_ch03(net, train_iter, loss, updater)train_loss, train_acc = train_epoch_ch03(net, train_iter, loss, updater)
    
      File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 64, in train_epoch_ch03
      File "G:\PyCharmProjects\limu-d2l\ch03\softmax_regression.py", line 64, in train_epoch_ch03
        self._popen = self._Popen(self)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\context.py", line 224, in _Popen
            for X, y in train_iter:for X, y in train_iter:
    
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 441, in __iter__
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 441, in __iter__
        self._popen = self._Popen(self)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\context.py", line 224, in _Popen
        return _default_context.get_context().Process._Popen(process_obj)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\context.py", line 327, in _Popen
        return _default_context.get_context().Process._Popen(process_obj)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\context.py", line 327, in _Popen
            return self._get_iterator()return self._get_iterator()
    
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 388, in _get_iterator
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 388, in _get_iterator
        return Popen(process_obj)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\popen_spawn_win32.py", line 45, in __init__
        return Popen(process_obj)            return _MultiProcessingDataLoaderIter(self)return _MultiProcessingDataLoaderIter(self)prep_data = spawn.get_preparation_data(process_obj._name)
    
    
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 1042, in __init__
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 1042, in __init__
    
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 154, in get_preparation_data
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\popen_spawn_win32.py", line 45, in __init__
        prep_data = spawn.get_preparation_data(process_obj._name)
          File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 154, in get_preparation_data
    _check_not_importing_main()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 134, in _check_not_importing_main
        _check_not_importing_main()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 134, in _check_not_importing_main
        raise RuntimeError('''
    RuntimeError: 
            An attempt has been made to start a new process before the
            current process has finished its bootstrapping phase.
    
            This probably means that you are not using fork to start your
            child processes and you have forgotten to use the proper idiom
            in the main module:
    
                if __name__ == '__main__':
                    freeze_support()
                    ...
    
            The "freeze_support()" line can be omitted if the program
            is not going to be frozen to produce an executable.
        w.start()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\process.py", line 121, in start
        w.start()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\process.py", line 121, in start
        raise RuntimeError('''
    RuntimeError: 
            An attempt has been made to start a new process before the
            current process has finished its bootstrapping phase.
    
            This probably means that you are not using fork to start your
            child processes and you have forgotten to use the proper idiom
            in the main module:
    
                if __name__ == '__main__':
                    freeze_support()
                    ...
    
            The "freeze_support()" line can be omitted if the program
            is not going to be frozen to produce an executable.
        self._popen = self._Popen(self)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\context.py", line 224, in _Popen
        self._popen = self._Popen(self)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\context.py", line 224, in _Popen
        return _default_context.get_context().Process._Popen(process_obj)
          File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\context.py", line 327, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\context.py", line 327, in _Popen
        return Popen(process_obj)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\popen_spawn_win32.py", line 45, in __init__
        return Popen(process_obj)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\popen_spawn_win32.py", line 45, in __init__
        prep_data = spawn.get_preparation_data(process_obj._name)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 154, in get_preparation_data
        prep_data = spawn.get_preparation_data(process_obj._name)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 154, in get_preparation_data
        _check_not_importing_main()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 134, in _check_not_importing_main
        _check_not_importing_main()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\spawn.py", line 134, in _check_not_importing_main
        raise RuntimeError('''
        raise RuntimeError('''
    RuntimeError: RuntimeError
            An attempt has been made to start a new process before the
            current process has finished its bootstrapping phase.
    
            This probably means that you are not using fork to start your
            child processes and you have forgotten to use the proper idiom
            in the main module:
    
                if __name__ == '__main__':
                    freeze_support()
                    ...
    
            The "freeze_support()" line can be omitted if the program
            is not going to be frozen to produce an executable.: 
    
            An attempt has been made to start a new process before the
            current process has finished its bootstrapping phase.
    
            This probably means that you are not using fork to start your
            child processes and you have forgotten to use the proper idiom
            in the main module:
    
                if __name__ == '__main__':
                    freeze_support()
                    ...
    
            The "freeze_support()" line can be omitted if the program
            is not going to be frozen to produce an executable.
    Traceback (most recent call last):
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 1132, in _try_get_data
        data = self._data_queue.get(timeout=timeout)
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\multiprocessing\queues.py", line 108, in get
        raise Empty
    _queue.Empty
    
    The above exception was the direct cause of the following exception:
    
    Traceback (most recent call last):
      File "G:/PyCharmProjects/limu-d2l/ch03/softmax_regression.py", line 97, in <module>
        train_ch03(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
      File "G:/PyCharmProjects/limu-d2l/ch03/softmax_regression.py", line 81, in train_ch03
        train_loss, train_acc = train_epoch_ch03(net, train_iter, loss, updater)
      File "G:/PyCharmProjects/limu-d2l/ch03/softmax_regression.py", line 64, in train_epoch_ch03
        for X, y in train_iter:
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 633, in __next__
        data = self._next_data()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 1328, in _next_data
        idx, data = self._get_data()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 1294, in _get_data
        success, data = self._try_get_data()
      File "C:\Users\Administrator\anaconda3\envs\limu\lib\site-packages\torch\utils\data\dataloader.py", line 1145, in _try_get_data
        raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str)) from e
    RuntimeError: DataLoader worker (pid(s) 14032, 23312, 21048, 1952) exited unexpectedly
    
    Process finished with exit code 1

## 二、问题分析 ##

这个错误是由于在使用多进程 DataLoader 时出现的问题，通常与 Windows 操作系统相关。在 Windows 上，使用多进程的 DataLoader 可能会导致一些问题，这与 Windows 的进程模型不太兼容。

## 三、解决方案（使用单进程 DataLoader） ##

在 Windows 上，将 DataLoader 的 `num_workers` 参数设置为 0，以使用单进程 DataLoader。这会禁用多进程加载数据，虽然可能会导致数据加载速度变慢，但通常可以解决与多进程 DataLoader 相关的问题。

#### d2l.load\_data\_fashion\_mnist(batch\_size)源代码 ####

def get_dataloader_workers():
        """Use 4 processes to read the data.
    
        Defined in :numref:`sec_utils`"""
        return 4
    
    def load_data_fashion_mnist(batch_size, resize=None):
        """Download the Fashion-MNIST dataset and then load it into memory.
    
        Defined in :numref:`sec_utils`"""
        trans = [transforms.ToTensor()]
        if resize:
            trans.insert(0, transforms.Resize(resize))
        trans = transforms.Compose(trans)
        mnist_train = torchvision.datasets.FashionMNIST(
            root="../data", train=True, transform=trans, download=True)
        mnist_test = torchvision.datasets.FashionMNIST(
            root="../data", train=False, transform=trans, download=True)
        return (torch.utils.data.DataLoader(mnist_train, batch_size, shuffle=True,
                                            num_workers=get_dataloader_workers()),
                torch.utils.data.DataLoader(mnist_test, batch_size, shuffle=False,
                                            num_workers=get_dataloader_workers()))

#### 在代码中使用修改后的load\_data\_fashion\_mnist函数 ####

def load_data_fashion_mnist(batch_size, resize=None, num_workers=4):
        """下载Fashion-MNIST数据集，然后将其加载到内存中"""
        trans = [transforms.ToTensor()]
        if resize:
            trans.index(0, transforms.Resize(resize))
        trans = transforms.Compose(trans)
    
        mnist_train = torchvision.datasets.FashionMNIST(root='../data', train=True, transform=trans, download=True)
        mnist_test = torchvision.datasets.FashionMNIST(root='../data', train=False, transform=trans, download=True)
        return (data.DataLoader(mnist_train, batch_size, shuffle=True, num_workers=num_workers),
                data.DataLoader(mnist_test, batch_size, shuffle=False, num_workers=num_workers))

train_iter, test_iter = fashion_mnist.load_data_fashion_mnist(batch_size, num_workers=0)

![在这里插入图片描述][8164d59690dc460d98af0a46bccfcf76.jpeg_pic_center]

## 四、为什么将 DataLoader 的 num\_workers 参数设置为 0，是使用的单进程，而不是零进程呢？ ##

在 PyTorch 的 DataLoader 中，`num_workers` 参数控制了用于加载数据的子进程数量。当 `num_workers` 被设置为 0 时，实际上是表示不使用任何子进程来加载数据，即单进程加载数据。

为什么不是零进程？这是因为 DataLoader 需要至少一个进程来加载数据，这个进程被称为主进程。主进程负责数据加载和分发给训练的进程。当 `num_workers` 设置为 0 时，只有主进程用于加载和处理数据，没有额外的子进程。这是一种单进程的数据加载方式。

如果将 `num_workers` 设置为 1，则会有一个额外的子进程来加载数据，总共会有两个进程：一个主进程和一个数据加载子进程。这种设置可以在某些情况下提高数据加载的效率，特别是当数据加载耗时较长时，子进程可以并行地加载数据，从而加速训练过程。

## 五、完整训练代码 ##

import torch
    from d2l import torch as d2l
    import fashion_mnist
    
    batch_size = 256
    train_iter, test_iter = fashion_mnist.load_data_fashion_mnist(batch_size, num_workers=0)
    
    # 初始化模型参数
    num_inputs = 784  # 每个输入图像的通道数为1, 高度和宽度均为28像素
    num_outputs = 10
    
    W = torch.normal(0, 0.01, size=(num_inputs, num_outputs), requires_grad=True)
    b = torch.zeros(num_outputs, requires_grad=True)
    
    
    # 定义softmax操作
    def softmax(X):
        """
        矩阵中的非常大或非常小的元素可能造成数值上溢或者下溢
        解决方案: P84 3.7.2 重新审视softmax的实现
        """
        X_exp = torch.exp(X)
        partition = X_exp.sum(1, keepdim=True)
        return X_exp / partition
    
    
    # 定义模型
    def net(X):
        return softmax(torch.matmul(X.reshape((-1, W.shape[0])), W) + b)
    
    
    # 定义损失函数
    def cross_entropy(y_hat, y):
        return - torch.log(y_hat[range(len(y_hat)), y])
    
    
    # 分类精度
    def accuracy(y_hat, y):
        """计算预测正确的数量"""
        if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:
            y_hat = y_hat.argmax(axis=1)
        cmp = y_hat.type(y.dtype) == y
        return float(cmp.type(y.dtype).sum())
    
    
    def evaluate_accuracy(net, data_iter):
        """计算在制定数据集上模型的精度"""
        if isinstance(net, torch.nn.Module):
            net.eval()
        metric = d2l.Accumulator(2)
        with torch.no_grad():
            for X, y in data_iter:
                metric.add(accuracy(net(X), y), y.numel())
        return metric[0] / metric[1]
    
    
    # 训练
    def train_epoch_ch03(net, train_iter, loss, updater):
        if isinstance(net, torch.nn.Module):
            net.train()
        # 训练损失总和, 训练准确度总和, 样本数
        metric = d2l.Accumulator(3)
        for X, y in train_iter:
            y_hat = net(X)
            l = loss(y_hat, y)
            if isinstance(updater, torch.optim.Optimizer):
                updater.zero_grad()
                l.mean().backward()
                updater.step()
            else:
                l.sum().backward()
                updater(X.shape[0])
            metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())
        # 返回训练损失和训练精度
        return metric[0] / metric[2], metric[1] / metric[2]
    
    
    def train_ch03(net, train_iter, test_iter, loss, num_epochs, updater):
        for epoch in range(num_epochs):
            train_loss, train_acc = train_epoch_ch03(net, train_iter, loss, updater)
            test_acc = evaluate_accuracy(net, test_iter)
            print(f'epoch {
          epoch + 1}, train_loss {
          train_loss:f}, train_acc {
          train_acc:f}, test_acc {
          test_acc:f}')
        assert train_loss < 0.5, train_loss
        assert train_acc <= 1 and train_acc > 0.7, train_acc
        assert test_acc <= 1 and test_acc > 0.7, test_acc
    
    
    lr = 0.1
    
    
    def updater(batch_size):
        return d2l.sgd([W, b], lr, batch_size)
    
    
    num_epochs = 10
    train_ch03(net, train_iter, test_iter, cross_entropy, num_epochs, updater)

[8164d59690dc460d98af0a46bccfcf76.jpeg_pic_center]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/13/31c544904a0345a58ab734e2902da6ec.png