title: HDF5 dataloading very slow. Causes zero % volatility in GPUs
body: <p>I am using a custom PyTorch dataclass to load instances from a H5 dataset I created. However, it appears to be incredibly slow when loading samples. I have followed several bits of advice on dealing with large HDF5 datasets, but I am wondering whether I am doing something that is obviously wrong. I am deploying my code on Linux if that makes a difference. I am running the code on 4 GPUs with nn.dataparallel in place for my model. As the dataloading is very slow, the GPU Volatility is at 0%.  Here is my dataclass loader:</p>
  
  <pre><code>import h5py
  from torch.utils import data
  
  class Features_Dataset(data.Dataset):
      def __init__(self, archive, phase):
          self.archive = archive
          self.phase = phase
  
      def __getitem__(self, index):
          with h5py.File(self.archive, 'r', libver='latest', swmr=True) as archive:
              datum = archive[str(self.phase) + '_all_arrays'][index]
              label = archive[str(self.phase) + '_labels'][index]
              path = archive[str(self.phase) +  '_img_paths'][index]
              return datum, label, path
  
      def __len__(self):
          with h5py.File(self.archive, 'r', libver='latest', swmr=True) as archive:
              datum = archive[str(self.phase) + '_all_arrays']
              return len(datum)
  
  
  if __name__ == '__main__':
      train_dataset = Features_Dataset(archive= "featuresdata/train.hdf5", phase= 'train')
      trainloader = data.DataLoader(train_dataset, num_workers=8, batch_size=128)
      print(len(trainloader))
      for i, (data, label, path) in enumerate(trainloader):
          print(path)
  
  
  </code></pre>
  
  <p>Am I missing something obvious? Is there a better way of loading instances rapidly? </p>
  
  <p>EDIT:</p>
  
  <p>Here is the updated dataclass however I now get a picling error when trying to use multiprocessing. </p>
  
  <pre><code>
  import h5py
  from torch.utils import data
  import torch.multiprocessing as mp
  mp.set_start_method('spawn')
  
  class Features_Dataset(data.Dataset):
      def __init__(self, archive, phase):
          self.archive = h5py.File(archive, 'r')
          self.labels = self.archive[str(phase) + '_labels']
          self.data = self.archive[str(phase) + '_all_arrays']
          self.img_paths = self.archive[str(phase) + '_img_paths']
  
      def __getitem__(self, index):
          datum = self.data[index]
          label = self.labels[index]
          path = self.img_paths[index]
          return datum, label, path
  
      def __len__(self):
          return len(self.data)
  
      def close(self):
          self.archive.close()
  
  if __name__ == '__main__':
      train_dataset = Features_Dataset(archive= "featuresdata/train.hdf5", phase= 'train')
      trainloader = data.DataLoader(train_dataset, num_workers=2, batch_size=4)
      print(len(trainloader))
      for i, (data, label, path) in enumerate(trainloader):
          print(path)
  
  </code></pre>