The role of the face and mouth in speech production as well asnon-verbal communication suggests the use of facial action tocontrol musical sound. Here we document work on theMouthesizer, a system which uses a headworn miniaturecamera and computer vision algorithm to extract shapeparameters from the mouth opening and output these as MIDIcontrol changes. We report our experience with variousgesture-to-sound mappings and musical applications, anddescribe a live performance which used the Mouthesizerinterface.
@article{arxiv.2010.03213,
title = {Designing, Playing, and Performing with a Vision-based Mouth Interface},
author = {Michael J. Lyons and Michael Haehnel and Nobuji Tetsutani},
journal= {arXiv preprint arXiv:2010.03213},
year = {2020}
}
Comments
Proceedings of the International Conference on New Interfaces for Musical Expression, 2003