@article {5460, title = {Many but not all deep neural network audio models capture brain responses and exhibit correspondence between model stages and brain regions}, journal = {PLOS Biology}, volume = {21}, year = {2023}, month = {12/2023}, pages = {e3002366}, abstract = {

Models that predict brain responses to stimuli provide one measure of understanding of a sensory system and have many potential applications in science and engineering. Deep artificial neural networks have emerged as the leading such predictive models of the visual system but are less explored in audition. Prior work provided examples of audio-trained neural networks that produced good predictions of auditory cortical fMRI responses and exhibited correspondence between model stages and brain regions, but left it unclear whether these results generalize to other neural network models and, thus, how to further improve models in this domain. We evaluated model-brain correspondence for publicly available audio neural network models along with in-house models trained on 4 different tasks. Most tested models outpredicted standard spectromporal filter-bank models of auditory cortex and exhibited systematic model-brain correspondence: Middle stages best predicted primary auditory cortex, while deep stages best predicted non-primary cortex. However, some state-of-the-art models produced substantially worse brain predictions. Models trained to recognize speech in background noise produced better brain predictions than models trained to recognize speech in quiet, potentially because hearing in noise imposes constraints on biological auditory representations. The training task influenced the prediction quality for specific cortical tuning properties, with best overall predictions resulting from models trained on multiple tasks. The results generally support the promise of deep neural networks as models of audition, though they also indicate that current models do not explain auditory cortical responses in their entirety.

}, doi = {10.1371/journal.pbio.3002366}, url = {https://dx.plos.org/10.1371/journal.pbio.3002366}, author = {Tuckute, Greta and Feather, Jenelle and Boebinger, Dana and McDermott, Josh H.}, editor = {Poeppel, David} } @article {4969, title = {The neural architecture of language: Integrative modeling converges on predictive processing}, journal = {Proceedings of the National Academy of Sciences}, volume = {118}, year = {2021}, month = {11/2021}, pages = {e2105646118}, abstract = {

Significance

Language is a quintessentially human ability. Research has long probed the functional architecture of language in the mind and brain using diverse neuroimaging, behavioral, and computational modeling approaches. However, adequate neurally-mechanistic accounts of how meaning might be extracted from language are sorely lacking. Here, we report a first step toward addressing this gap by connecting recent artificial neural networks from machine learning to human recordings during language processing. We find that the most powerful models predict neural and behavioral responses across different datasets up to noise levels. Models that perform better at predicting the next word in a sequence also better predict brain measurements{\textemdash}providing computationally explicit evidence that predictive processing fundamentally shapes the language comprehension mechanisms in the brain.

Abstract

The neuroscience of perception has recently been revolutionized with an integrative modeling approach in which computation, brain function, and behavior are linked across many datasets and many computational models. By revealing trends across models, this approach yields novel insights into cognitive and neural mechanisms in the target domain. We here present a systematic study taking this approach to higher-level cognition: human language processing, our species{\textquoteright} signature cognitive skill. We find that the most powerful {\textquotedblleft}transformer{\textquotedblright} models predict nearly 100\% of explainable variance in neural responses to sentences and generalize across different datasets and imaging modalities (functional MRI and electrocorticography). Models{\textquoteright} neural fits ({\textquotedblleft}brain score{\textquotedblright}) and fits to behavioral responses are both strongly correlated with model accuracy on the next-word prediction task (but not other language tasks). Model architecture appears to substantially contribute to neural fit. These results provide computationally explicit evidence that predictive processing fundamentally shapes the language comprehension mechanisms in the human brain.

}, issn = {0027-8424}, doi = {10.1073/pnas.2105646118}, url = {http://www.pnas.org/lookup/doi/10.1073/pnas.2105646118}, author = {Martin Schrimpf and Blank, Idan Asher and Tuckute, Greta and Kauf, Carina and Hosseini, Eghbal A. and Nancy Kanwisher and Joshua B. Tenenbaum and Fedorenko, Evelina} }