def save_batches(file_list, tar_dir, img_size=256, batch_size=256,
                 flag_avg=False, num_sub_batch=1):
    '''
    num_sub_batch is for parallelling using multiple gpus, it should be
    2, 4, or 8,
    where the indexing is reverted binary number
    when 2, the files ends with _0.pkl and _1.pkl
    when 4, with _00.pkl, _10.pkl, _01.pkl and _11.pkl

    '''

    if not os.path.exists(tar_dir):
        os.makedirs(tar_dir)

    img_batch = np.zeros((3, img_size, img_size, batch_size), np.uint8)

    if flag_avg:
        img_sum = np.zeros((3, img_size, img_size))

    batch_count = 0
    count = 0
    for file_name in file_list:
        img_batch[:, :, :, count % batch_size] = \
            get_img(file_name, img_size=img_size, batch_size=batch_size)

        count += 1
        if count % batch_size == 0:
            batch_count += 1

            if flag_avg:
                img_sum += img_batch.mean(axis=3)

            if num_sub_batch == 1:
                save_name = '%04d' % (batch_count - 1) + '.hkl'
                hkl.dump(img_batch, os.path.join(tar_dir, save_name), mode='w')

            elif num_sub_batch == 2:
                half_size = batch_size / 2
                save_name = '%04d' % (batch_count - 1) + '_0.hkl'
                hkl.dump(img_batch[:, :, :, :half_size],
                         os.path.join(tar_dir, save_name), mode='w')

                save_name = '%04d' % (batch_count - 1) + '_1.hkl'
                hkl.dump(img_batch[:, :, :, half_size:],
                         os.path.join(tar_dir, save_name), mode='w')

            elif num_sub_batch == 4:
                q1 = batch_size / 4
                q2 = batch_size / 2
                q3 = batch_size / 4 * 3

                save_name = '%04d' % (batch_count - 1) + '_00.hkl'
                hkl.dump(img_batch[:, :, :, :q1],
                         os.path.join(tar_dir, save_name), mode='w')
                save_name = '%04d' % (batch_count - 1) + '_10.hkl'
                hkl.dump(img_batch[:, :, :, q1:q2],
                         os.path.join(tar_dir, save_name), mode='w')
                save_name = '%04d' % (batch_count - 1) + '_01.hkl'
                hkl.dump(img_batch[:, :, :, q2:q3],
                         os.path.join(tar_dir, save_name), mode='w')
                save_name = '%04d' % (batch_count - 1) + '_11.hkl'
                hkl.dump(img_batch[:, :, :, q3:],
                         os.path.join(tar_dir, save_name), mode='w')
            else:
                NotImplementedError("num_sub_batch has to be 1, 2, or 4")

    return img_sum / batch_count if flag_avg else None
dataset/ETHPy150Open uoguelph-mlrg/theano_alexnet/preprocessing/make_hkl.py/save_batches/original