Pre-trained language models for code (PLMCs) have gained attention in recent research. These models are pre-trained on large-scale datasets using multi-modal objectives. However, fine-tuning them requires extensive supervision and is limited by the size of the dataset provided. We aim to improve this issue by proposing a simple data augmentation framework. Our framework utilizes knowledge gained during the pre-training and fine-tuning stage to generate pseudo data, which is then used as training data for the next step. We incorporate this framework into the state-of-the-art language models, such as CodeT5, CodeBERT, and UnixCoder. The results show that our framework significantly improves PLMCs' performance in code-related sequence generation tasks, such as code summarization and code generation in the CodeXGLUE benchmark.
@article{arxiv.2304.01228,
title = {Better Language Models of Code through Self-Improvement},
author = {Hung Quoc To and Nghi D. Q. Bui and Jin Guo and Tien N. Nguyen},
journal= {arXiv preprint arXiv:2304.01228},
year = {2023}
}