Stacked denoising auto encoders (DAEs) are well known to learn useful deep representations, which can be used to improve supervised training by initializing a deep network. We investigate a training scheme of a deep DAE, where DAE layers are gradually added and keep adapting as additional layers are added. We show that in the regime of mid-sized datasets, this gradual training provides a small but consistent improvement over stacked training in both reconstruction quality and classification error over stacked training on MNIST and CIFAR datasets.
@article{arxiv.1412.6257,
title = {Gradual training of deep denoising auto encoders},
author = {Alexander Kalmanovich and Gal Chechik},
journal= {arXiv preprint arXiv:1412.6257},
year = {2014}
}