最近看论文代码常常遇到关于CIFAR10数据集的官方配套函数，这里就专门开一篇文章作为对这个函数理解的笔记。下面贴出源码并附带我个人的注释。
class CIFAR10(VisionDataset):
#这里继承的VisionDataset母类要求继承之后必须对其中的__getitem__和__len__方法进行重写
"""`CIFAR10 <https://www.cs.toronto.edu/~kriz/cifar.html>`_ Dataset.
Args:
root (string): Root directory of dataset where directory
``cifar-10-batches-py`` exists or will be saved to if download is set to True.
train (bool, optional): If True, creates dataset from training set, otherwise
creates from test set.
transform (callable, optional): A function/transform that takes in an PIL image
and returns a transformed version. E.g, ``transforms.RandomCrop``
target_transform (callable, optional): A function/transform that takes in the
target and transforms it.
download (bool, optional): If true, downloads the dataset from the internet and
puts it in root directory. If dataset is already downloaded, it is not
downloaded again.
"""
base_folder = 'cifar-10-batches-py'
url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
filename = "cifar-10-python.tar.gz"
tgz_md5 = 'c58f30108f718f92721af3b95e74349a'
train_list = [
['data_batch_1', 'c99cafc152244af753f735de768cd75f'],
['data_batch_2', 'd4bba439e000b95fd0a9bffe97cbabec'],
['data_batch_3', '54ebc095f3ab1f0389bbae665268c751'],
['data_batch_4', '634d18415352ddfa80567beed471001a'],
['data_batch_5', '482c414d41f54cd18b22e5b47cb7c3cb'],
]
test_list = [
['test_batch', '40351d587109b95175f43aff81a1287e'],
]
#下面的meta{}字典储存关于cifar10数据集的元数据。元数据（Metadata），又称中介数据、中继数据，为描述数据的数据（data about data），主要是描述数据属性（property）的信息，用来支持如指示存储位置、历史数据、资源查找、文件记录等功能。
meta = {
'filename': 'batches.meta',
'key': 'label_names',
'md5': '5ff9c542aee3614f3951f8cda6e48888',
}
def __init__(
self,
root: str,
train: bool = True,
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
download: bool = False,
) -> None:
super(CIFAR10, self).__init__(root, transform=transform,
target_transform=target_transform)
self.train = train # training set or test set
if download:
self.download()
if not self._check_integrity():
raise RuntimeError('Dataset not found or corrupted.' +
' You can use download=True to download it')
if self.train:
downloaded_list = self.train_list
else:
downloaded_list = self.test_list
self.data: Any = [] #这里的冒号:时python3.6引入的对变量的注释符号，self.data: Any = []相当于self.data = [],这里冒号：后面的any没有实际意义，只是对变量data进行类型的注释说明
self.targets = []
# now load the picked numpy arrays
for file_name, checksum in downloaded_list:
file_path = os.path.join(self.root, self.base_folder, file_name)#这里生成cifar10数据集的每一部分的地址
with open(file_path, 'rb') as f:
entry = pickle.load(f, encoding='latin1')#这里将cifar10数据集的文件进行解析提取
self.data.append(entry['data'])#将提取出的数据集中的data数据部分写入到上面定义好的空白列表中
if 'labels' in entry:
self.targets.extend(entry['labels'])
else:
self.targets.extend(entry['fine_labels'])
self.data = np.vstack(self.data).reshape(-1, 3, 32, 32)#这里np.vstack()函数表示对括号中的矩阵进行按行堆叠，reshape（）函数中的-1表示这个地方的维度大小不提前设定具体值而是由实际运算中确定该维度的大小
self.data = self.data.transpose((0, 2, 3, 1)) # convert to HWC，表示将原来的（图像数量，每张图像的通道数，图像高，图像宽）改为（图像数量，图像高，图像宽，每张图像的通道数）的形式，以满足后面函数对图像数据处理的格式要求
self._load_meta()#这里调用在本类中下面定义的类函数 _load_meta()
def _load_meta(self) -> None:#这个函数作用是通过读取cifar10数据集的元数据（上面的名叫meta的字典就储存着cifar10数据集的元数据）来读取cifar10数据集中定义好的类别名称信息，并新增一个名为self.class_to_idx的类属性，该类属性是一个字典，以键值对的形式储存cifar10数据集所有数据的类别名称以及其对应的id序号
path = os.path.join(self.root, self.base_folder, self.meta['filename'])
if not check_integrity(path, self.meta['md5']):
raise RuntimeError('Dataset metadata file not found or corrupted.' +
' You can use download=True to download it')
with open(path, 'rb') as infile:
data = pickle.load(infile, encoding='latin1')
self.classes = data[self.meta['key']]
self.class_to_idx = {_class: i for i, _class in enumerate(self.classes)}
def __getitem__(self, index: int) -> Tuple[Any, Any]:
"""
这里是重新定义了类CIFAR10所继承的母类VisionDataset中的魔法函数__getitem__()
例如有类CIFAR10的实例cifar_project，对该实例传入int类型的参数如8：
则cifar_project(8)运行结果是返回上面提取的CIFAR10数据集的图片列表self.data与标签列
表self.target的对应8序号的数据，如果在实例化cifar_project时还传入了定义对图片和标签
进行变化方式的transform和target_transform则将要返回的图片和标签数据进行对应处理后的
结果
"""
"""
Args:
index (int): Index
Returns:
tuple: (image, target) where target is index of the target class.
"""
img, target = self.data[index], self.targets[index]
# doing this so that it is consistent with all other datasets
# to return a PIL Image
img = Image.fromarray(img)
if self.transform is not None:
img = self.transform(img)
if self.target_transform is not None:
target = self.target_transform(target)
return img, target
def __len__(self) -> int:
return len(self.data)
def _check_integrity(self) -> bool:
root = self.root
for fentry in (self.train_list + self.test_list):
filename, md5 = fentry[0], fentry[1]
fpath = os.path.join(root, self.base_folder, filename)
if not check_integrity(fpath, md5):
return False
return True
def download(self) -> None:
if self._check_integrity():
print('Files already downloaded and verified')
return
download_and_extract_archive(self.url, self.root, filename=self.filename, md5=self.tgz_md5)
def extra_repr(self) -> str:
return "Split: {}".format("Train" if self.train is True else "Test")