@article {5214, title = {Decoding of human identity by computer vision and neuronal vision}, journal = {Scientific Reports}, volume = {13}, year = {2023}, month = {01/2023}, abstract = {

Extracting meaning from a dynamic and variable flow of incoming information is a major goal of both natural and artificial intelligence. Computer vision (CV) guided by deep learning (DL) has made significant strides in recognizing a specific identity despite highly variable attributes. This is the same challenge faced by the nervous system and partially addressed by the concept cells{\textemdash}neurons exhibiting selective firing in response to specific persons/places, described in the human medial temporal lobe (MTL) . Yet, access to neurons representing a particular concept is limited due to these neurons{\textquoteright} sparse coding. It is conceivable, however, that the information required for such decoding is present in relatively small neuronal populations. To evaluate how well neuronal populations encode identity information in natural settings, we recorded neuronal activity from multiple brain regions of nine neurosurgical epilepsy patients implanted with depth electrodes, while the subjects watched an episode of the TV series {\textquotedblleft}24{\textquotedblright}. First, we devised a minimally supervised CV algorithm (with comparable performance against manually-labeled data) to detect the most prevalent characters (above 1\% overall appearance) in each frame. Next, we implemented DL models that used the time- varying population neural data as inputs and decoded the visual presence of the four main characters throughout the episode. This methodology allowed us to compare {\textquotedblleft}computer vision{\textquotedblright} with {\textquotedblleft}neuronal vision{\textquotedblright}{\textemdash}footprints associated with each character present in the activity of a subset of neurons{\textemdash}and identify the brain regions that contributed to this decoding process. We then tested the DL models during a recognition memory task following movie viewing where subjects were asked to recognize clip segments from the presented episode. DL model activations were not only modulated by the presence of the corresponding characters but also by participants{\textquoteright} subjective memory of whether they had seen the clip segment, and by the associative strengths of the characters in the narrative plot. The described approach can offer novel ways to probe the representation of concepts in time-evolving dynamic behavioral tasks. Further, the results suggest that the information required to robustly decode concepts is present in the population activity of only tens of neurons even in brain regions beyond MTL.

}, doi = {10.1038/s41598-022-26946-w}, url = {https://www.nature.com/articles/s41598-022-26946-w}, author = {Zhang, Yipeng and Aghajan, Zahra M. and Ison, Matias and Lu, Qiujing and Tang, Hanlin and Kalender, Guldamla and Monsoor, Tonmoy and Zheng, Jie and Kreiman, Gabriel and Roychowdhury, Vwani and Fried, Itzhak} } @article {5315, title = {Decoding of human identity by computer vision and neuronal visionAbstract}, journal = {Scientific Reports}, volume = {13}, year = {2023}, month = {01/2023}, abstract = {

Extracting meaning from a dynamic and variable flow of incoming information is a major goal of both natural and artificial intelligence. Computer vision (CV) guided by deep learning (DL) has made significant strides in recognizing a specific identity despite highly variable attributes. This is the same challenge faced by the nervous system and partially addressed by the concept cells{\textemdash}neurons exhibiting selective firing in response to specific persons/places, described in the human medial temporal lobe (MTL) . Yet, access to neurons representing a particular concept is limited due to these neurons{\textquoteright} sparse coding. It is conceivable, however, that the information required for such decoding is present in relatively small neuronal populations. To evaluate how well neuronal populations encode identity information in natural settings, we recorded neuronal activity from multiple brain regions of nine neurosurgical epilepsy patients implanted with depth electrodes, while the subjects watched an episode of the TV series {\textquotedblleft}24{\textquotedblright}. First, we devised a minimally supervised CV algorithm (with comparable performance against manually-labeled data) to detect the most prevalent characters (above 1\% overall appearance) in each frame. Next, we implemented DL models that used the time- varying population neural data as inputs and decoded the visual presence of the four main characters throughout the episode. This methodology allowed us to compare {\textquotedblleft}computer vision{\textquotedblright} with {\textquotedblleft}neuronal vision{\textquotedblright}{\textemdash}footprints associated with each character present in the activity of a subset of neurons{\textemdash}and identify the brain regions that contributed to this decoding process. We then tested the DL models during a recognition memory task following movie viewing where subjects were asked to recognize clip segments from the presented episode. DL model activations were not only modulated by the presence of the corresponding characters but also by participants{\textquoteright} subjective memory of whether they had seen the clip segment, and by the associative strengths of the characters in the narrative plot. The described approach can offer novel ways to probe the representation of concepts in time-evolving dynamic behavioral tasks. Further, the results suggest that the information required to robustly decode concepts is present in the population activity of only tens of neurons even in brain regions beyond MTL.

}, doi = {10.1038/s41598-022-26946-w}, url = {https://www.nature.com/articles/s41598-022-26946-w}, author = {Zhang, Yipeng and Aghajan, Zahra M. and Ison, Matias and Lu, Qiujing and Tang, Hanlin and Kalender, Guldamla and Monsoor, Tonmoy and Zheng, Jie and Kreiman, Gabriel and Roychowdhury, Vwani and Fried, Itzhak} } @article {5435, title = {Minute-scale periodicity of neuronal firing in the human entorhinal cortex}, journal = {Cell Reports}, volume = {42}, year = {2023}, month = {11/2023}, pages = {113271}, abstract = {

Grid cells in the entorhinal cortex demonstrate spatially periodic firing, thought to provide a spatial map on behaviorally relevant length scales. Whether such periodicity exists for behaviorally relevant time scales in the human brain remains unclear. We investigate neuronal firing during a temporally continuous experience by presenting 14 neurosurgical patients with a video while recording neuronal activity from multiple brain regions. We report on neurons that modulate their activity in a periodic manner across different time scales{\textemdash}from seconds to many minutes, most prevalently in the entorhinal cortex. These neurons remap their dominant periodicity to shorter time scales during a subsequent recognition memory task. When the video is presented at two different speeds, a significant percentage of these temporally periodic cells (TPCs) maintain their time scales, suggesting a degree of invariance. The TPCs{\textquoteright} temporal periodicity might complement the spatial periodicity of grid cells and together provide scalable spatiotemporal metrics for human experience.

}, issn = {22111247}, doi = {10.1016/j.celrep.2023.113271}, url = {https://linkinghub.elsevier.com/retrieve/pii/S2211124723012834}, author = {M. Aghajan, Zahra and Kreiman, Gabriel and Fried, Itzhak} }