# -*- encoding: utf-8 -*-"""@File : DP.py@Time : 2021/5/19 3:03 下午@Author : Johnsonhttps://www.aiuai.cn/aifarm1764.html"""import torchimport torch.nn as nnfrom torch.utils.data import DataLoader,Dataset#parameters and Dataloadersinput_size = 5output_size = 2batch_size = 30data_size = 100device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")class RandomDataset(Dataset): def __init__(self,size,length): self.len = length self.data = torch.randn(length,size) def __getitem__(self, item): return self.data[item] def __len__(self): return self.lenrand_loader = DataLoader(dataset=RandomDataset(input_size,data_size), batch_size=batch_size,shuffle=True )class Model(nn.Module): def __init__(self,input_size,output_size): super(Model, self).__init__() self.fc = nn.Linear(input_size,output_size) def forward(self,input): output = self.fc(input) return outputmodel = Model(input_size,output_size)if torch.cuda.device_count()>1: print(torch.cuda.device_count()) model = nn.DataParallel(model) #并行model.to(device)#训练for data in rand_loader: input = data.to(device) output = model(input) print("outside:input size",input.size(),"output_size",output.size())
#
# -*- encoding: utf-8 -*-"""@File : DP.py@Time : 2021/5/19 3:03 下午@Author : Johnsonhttps://www.aiuai.cn/aifarm1764.htmlhttps://mp.weixin.qq.com/s/N8jlsrDy1mho1HsNH5GBjA"""## 单机单卡import torchimport torchimport torch.nn as nnimport torchvisionimport torchvision.transforms as transformsBATCH_SIZE = 256EPOCHS = 5if __name__ == "__main__": # 1. define network device = "cuda" net = torchvision.models.resnet18(num_classes=10) net = net.to(device=device) # 2. define dataloader trainset = torchvision.datasets.CIFAR10( root="./data", train=True, download=True, transform=transforms.Compose( [ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010) ), ] ), ) train_loader = torch.utils.data.DataLoader( trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, pin_memory=True, ) # 3. define loss and optimizer criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD( net.parameters(), lr=0.01, momentum=0.9, weight_decay=0.0001, nesterov=True, ) print(" ======= Training ======= \n") # 4. start to train net.train() for ep in range(1, EPOCHS + 1): train_loss = correct = total = 0 for idx, (inputs, targets) in enumerate(train_loader): inputs, targets = inputs.to(device), targets.to(device) outputs = net(inputs) loss = criterion(outputs, targets) optimizer.zero_grad() loss.backward() optimizer.step() train_loss += loss.item() total += targets.size(0) correct += torch.eq(outputs.argmax(dim=1), targets).sum().item() if (idx + 1) % 50 == 0 or (idx + 1) == len(train_loader): print( " == step: [{:3}/{}] [{}/{}] | loss: {:.3f} | acc: {:6.3f}%".format( idx + 1, len(train_loader), ep, EPOCHS, train_loss / (idx + 1), 100.0 * correct / total, ) ) print("\n ======= Training Finished ======= \n")## 单机多卡DPimport torchimport torch.nn as nnimport torchvisionimport torchvision.transforms as transformsBATCH_SIZE = 256EPOCHS = 5if __name__ == "__main__": # 1. define network device = "cuda" net = torchvision.models.resnet18(pretrained=False, num_classes=10) net = net.to(device=device) # Use single-machine multi-GPU DataParallel, # you would like to speed up training with the minimum code change. net = nn.DataParallel(net) # 2. define dataloader trainset = torchvision.datasets.CIFAR10( root="./data", train=True, download=True, transform=transforms.Compose( [ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010) ), ] ), ) train_loader = torch.utils.data.DataLoader( trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, pin_memory=True, ) # 3. define loss and optimizer criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD( net.parameters(), lr=0.01, momentum=0.9, weight_decay=0.0001, nesterov=True, ) print(" ======= Training ======= \n") # 4. start to train net.train() for ep in range(1, EPOCHS + 1): train_loss = correct = total = 0 for idx, (inputs, targets) in enumerate(train_loader): inputs, targets = inputs.to(device), targets.to(device) outputs = net(inputs) loss = criterion(outputs, targets) optimizer.zero_grad() loss.backward() optimizer.step() train_loss += loss.item() total += targets.size(0) correct += torch.eq(outputs.argmax(dim=1), targets).sum().item() if (idx + 1) % 50 == 0 or (idx + 1) == len(train_loader): print( " == step: [{:3}/{}] [{}/{}] | loss: {:.3f} | acc: {:6.3f}%".format( idx + 1, len(train_loader), ep, EPOCHS, train_loss / (idx + 1), 100.0 * correct / total, ) ) print("\n ======= Training Finished ======= \n")## 多机多卡DDP'''进程组的概念：group：进程组，大部分情况下DDP的各个进程是在同一个进程组下world size：总的进程数量（原则上一个process占用一个GPU是最优的）rank：当前进程的序号，用于进程间通讯，rank=0的主机master节点local_rank：当前进程对应的GPU号举个栗子 ：4台机器(每台机器8张卡)进行分布式训练， 通过 init_process_group() 对进程组进行初始化， 初始化后 可以通过 get_world_size() 获取到 world size，在该例中为32， 即有32个进程，其编号为0-31, 通过 get_rank() 函数可以进行获取 在每台机器上，local rank均为0-8，这是 local rank 与 rank 的区别， local rank 会对应到实际的GPU ID上 (单机多任务的情况下注意CUDA_VISIBLE_DEVICES的使用，控制不同程序可见的GPU device)。'''import osimport torchimport torch.distributed as distimport torch.nn as nnimport torchvisionimport torchvision.transforms as transformsfrom torch.nn.parallel import DistributedDataParallel as DDPfrom torch.utils.data.distributed import DistributedSamplerBATCH_SIZE = 256EPOCHS = 5if __name__ == "__main__": # 0. set up distributed device rank = int(os.environ["RANK"]) local_rank = int(os.environ["LOCAL_RANK"]) torch.cuda.set_device(rank % torch.cuda.device_count()) dist.init_process_group(backend="nccl") device = torch.device("cuda", local_rank) print(f"[init] == local rank: {local_rank}, global rank: {rank} ==") # 1. define network net = torchvision.models.resnet18(pretrained=False, num_classes=10) net = net.to(device) # DistributedDataParallel net = DDP(net, device_ids=[local_rank], output_device=local_rank) # 2. define dataloader trainset = torchvision.datasets.CIFAR10( root="./data", train=True, download=False, transform=transforms.Compose( [ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010) ), ] ), ) # DistributedSampler # we test single Machine with 2 GPUs # so the [batch size] for each process is 256 / 2 = 128 train_sampler = torch.utils.data.distributed.DistributedSampler( trainset, shuffle=True, ) train_loader = torch.utils.data.DataLoader( trainset, batch_size=BATCH_SIZE, num_workers=4, pin_memory=True, sampler=train_sampler, ) # 3. define loss and optimizer criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD( net.parameters(), lr=0.01 * 2, momentum=0.9, weight_decay=0.0001, nesterov=True, ) if rank == 0: print(" ======= Training ======= \n") # 4. start to train net.train() for ep in range(1, EPOCHS + 1): train_loss = correct = total = 0 # set sampler train_loader.sampler.set_epoch(ep) for idx, (inputs, targets) in enumerate(train_loader): inputs, targets = inputs.to(device), targets.to(device) outputs = net(inputs) loss = criterion(outputs, targets) optimizer.zero_grad() loss.backward() optimizer.step() train_loss += loss.item() total += targets.size(0) correct += torch.eq(outputs.argmax(dim=1), targets).sum().item() if rank == 0 and ((idx + 1) % 25 == 0 or (idx + 1) == len(train_loader)): print( " == step: [{:3}/{}] [{}/{}] | loss: {:.3f} | acc: {:6.3f}%".format( idx + 1, len(train_loader), ep, EPOCHS, train_loss / (idx + 1), 100.0 * correct / total, ) ) if rank == 0: print("\n ======= Training Finished ======= \n")
女爷i
「爱情、让人受尽委屈。」
不念不忘少年蓝@
我就是我
傷城~
还没有评论，来说两句吧...