In this work we compare different batch construction methods for mini-batch training of recurrent neural networks. While popular implementations like TensorFlow and MXNet suggest a bucketing approach to improve the parallelization capabilities of the recurrent training process, we propose a simple ordering strategy that arranges the training sequences in a stochastic alternatingly sorted way. We compare our method to sequence bucketing as well as various other batch construction strategies on the CHiME-4 noisy speech recognition corpus. The experiments show that our alternated sorting approach is able to compete both in training time and recognition performance while being conceptually simpler to implement.
@article{arxiv.1705.02414,
title = {A comprehensive study of batch construction strategies for recurrent neural networks in MXNet},
author = {Patrick Doetsch and Pavel Golik and Hermann Ney},
journal= {arXiv preprint arXiv:1705.02414},
year = {2017}
}