-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdataset.py
75 lines (59 loc) · 3.45 KB
/
dataset.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
'''
function for loading datasets
contains:
CIFAR-10
CIFAR-100
'''
import os
import numpy as np
from torchvision import transforms
from torch.utils.data import DataLoader, Subset
from torchvision.datasets import CIFAR10, CIFAR100
__all__ = ['cifar10_dataloaders', 'cifar100_dataloaders']
def cifar10_dataloaders(batch_size=128, data_dir='datasets/cifar10', num_workers=2, holdout=0):
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
test_transform = transforms.Compose([
transforms.ToTensor(),
])
print('Dataset information: CIFAR-10\t 45000 images for training \t 500 images for validation\t')
print('10000 images for testing\t no normalize applied in data_transform')
print('Data augmentation = randomcrop(32,4) + randomhorizontalflip')
if holdout==0:
train_set = Subset(CIFAR10(data_dir, train=True, transform=train_transform, download=True), list(range(45000)))
holdout_set = None
else:
train_set = Subset(CIFAR10(data_dir, train=True, transform=train_transform, download=True), list(range(int(45000*(1-holdout)))))
holdout_set = Subset(CIFAR10(data_dir, train=True, transform=test_transform, download=True), list(range(int(45000*(1-holdout)),45000)))
val_set = Subset(CIFAR10(data_dir, train=True, transform=test_transform, download=True), list(range(45000, 50000)))
test_set = CIFAR10(data_dir, train=False, transform=test_transform, download=True)
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True)
val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)
test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)
if holdout_set==None:
holdout_loader=None
else:
holdout_loader = DataLoader(holdout_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)
return train_loader, val_loader, test_loader, holdout_loader
def cifar100_dataloaders(batch_size=128, data_dir='datasets/cifar100', num_workers=2):
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
test_transform = transforms.Compose([
transforms.ToTensor(),
])
print('Dataset information: CIFAR-100\t 45000 images for training \t 500 images for validation\t')
print('10000 images for testing\t no normalize applied in data_transform')
print('Data augmentation = randomcrop(32,4) + randomhorizontalflip')
train_set = Subset(CIFAR100(data_dir, train=True, transform=train_transform, download=True), list(range(45000)))
val_set = Subset(CIFAR100(data_dir, train=True, transform=test_transform, download=True), list(range(45000, 50000)))
test_set = CIFAR100(data_dir, train=False, transform=test_transform, download=True)
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True)
val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)
test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)
return train_loader, val_loader, test_loader