@article {3517, title = {What is changing when: decoding visual information in movies from human intracranial recordings}, journal = {NeuroImage}, volume = {180, Part A}, year = {2018}, month = {10/2018}, pages = {147-159}, abstract = {

The majority of visual recognition studies have focused on the neural responses to repeated presentations of static stimuli with abrupt and well-defined onset and offset times. In contrast, natural vision involves unique renderings of visual inputs that are continuously changing without explicitly defined temporal transitions. Here we considered commercial movies as a coarse proxy to natural vision. We recorded intracranial field potential signals from 1,284 electrodes implanted in 15 patients with epilepsy while the subjects passively viewed commercial movies. We could rapidly detect large changes in the visual inputs within approximately 100 ms of their occurrence, using exclusively field potential signals from ventral visual cortical areas including the inferior temporal gyrus and inferior occipital gyrus. Furthermore, we could decode the content of those visual changes even in a single movie presentation, generalizing across the wide range of transformations present in a movie. These results present a methodological framework for studying cognition during dynamic and natural vision.

}, keywords = {Electrocorticography (ECoG), Movies, Natural vision, neural decoding, object recognition, Ventral pathway}, doi = {10.1016/j.neuroimage.2017.08.027}, url = {https://www.sciencedirect.com/science/article/pii/S1053811917306742}, author = {Leyla Isik and Jedediah Singer and Nancy Kanwisher and Madsen JR and Anderson WS and Gabriel Kreiman} } @article {3451, title = {What is changing when: Decoding visual information in movies from human intracranial recordings}, journal = {Neuroimage}, year = {2017}, abstract = {

The majority of visual recognition studies have focused on the neural responses to repeated presentations of static stimuli with abrupt and well-defined onset and offset times. In contrast,\ natural vision\ involves unique renderings of visual inputs that are continuously changing without explicitly defined temporal transitions. Here we considered commercial movies as a coarse proxy to natural vision. We recorded intracranial\ field potential\ signals from 1,284 electrodes implanted in 15 patients with\ epilepsy\ while the subjects passively viewed commercial movies. We could rapidly detect large changes in the visual inputs within approximately 100\ ms of their occurrence, using exclusively field potential signals from ventral visual cortical areas including the inferior temporal\ gyrus\ and inferior\ occipitalgyrus. Furthermore, we could decode the content of those visual changes even in a single movie presentation, generalizing across the wide range of transformations present in a movie. These results present a methodological framework for studying cognition during dynamic and natural vision.

}, doi = {https://doi.org/10.1016/j.neuroimage.2017.08.027}, url = {https://www.sciencedirect.com/science/article/pii/S1053811917306742}, author = {Leyla Isik and Jedediah Singer and Joseph Madsen and Nancy Kanwisher and Gabriel Kreiman} } @proceedings {2121, title = {A machine learning approach to predict episodic memory formation}, year = {2016}, pages = {539 - 544 }, address = {Princeton, NJ }, doi = {10.1109/CISS.2016.7460560}, url = {http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=7460560\&newsearch=true\&queryText=A\%20machine\%20learning\%20approach\%20to\%20predict\%20episodic\%20memory\%20formation}, author = {Hanlin Tang and Jedediah Singer and Matias J. Ison and Gnel Pivazyan and Melissa Romaine and Elizabeth Meller and Victoria Perron and Marlise Arlellano and Gabriel Kreiman and Melissa Romaine and Adrianna Boulin and Rosa Frias and James Carroll and Sarah Dowcett} } @article {2565, title = {Predicting episodic memory formation for movie events}, journal = {Scientific Reports}, year = {2016}, month = {10/2016}, abstract = {

Episodic memories are long lasting and full of detail, yet imperfect and malleable. We quantitatively evaluated recollection of short audiovisual segments from movies as a proxy to real-life memory formation in 161 subjects at 15 minutes up to a year after encoding. Memories were reproducible within and across individuals, showed the typical decay with time elapsed between encoding and testing, were fallible yet accurate, and were insensitive to low-level stimulus manipulations but sensitive to high-level stimulus properties. Remarkably, memorability was also high for single movie frames, even one year post-encoding. To evaluate what determines the efficacy of long-term memory formation, we developed an extensive set of content annotations that included actions, emotional valence, visual cues and auditory cues. These annotations enabled us to document the content properties that showed a stronger correlation with recognition memory and to build a machine-learning computational model that accounted for episodic memory formation in single events for group averages and individual subjects with an accuracy of up to 80\%. These results provide initial steps towards the development of a quantitative computational theory capable of explaining the subjective filtering steps that lead to how humans learn and consolidate memories.

}, doi = {10.1038/srep30175}, url = {http://www.nature.com/articles/srep30175}, author = {Hanlin Tang and Jedediah Singer and Matias J. Ison and Gnel Pivazyan and Melissa Romaine and Rosa Frias and Elizabeth Meller and Adrianna Boulin and James Carroll and Victoria Perron and Sarah Dowcett and Arellano, Marlise and Gabriel Kreiman} } @article {2885, title = {Predicting episodic memory formation for movie events [code]}, year = {2016}, abstract = {

Episodic memories are long lasting and full of detail, yet imperfect and malleable. We quantitatively\  evaluated recollection of short audiovisual segments from movies as a proxy to real-life memory\  formation in 161 subjects at 15\  minutes up to a year after encoding. Memories were reproducible within\  and across individuals, showed the typical decay with time elapsed between encoding and testing,\  were fallible yet accurate, and were insensitive to low-level stimulus manipulations but sensitive to\  high-level stimulus properties. Remarkably, memorability was also high for single movie frames, even\  one year post-encoding. To evaluate what determines the efficacy of long-term memory formation,\  we developed an extensive set of content annotations that included actions, emotional valence, visual\  cues and auditory cues. These annotations enabled us to document the content properties that showed\  a stronger correlation with recognition memory and to build a machine-learning computational model\  that accounted for episodic memory formation in single events for group averages and individual\  subjects with an accuracy of up to 80\%. These results provide initial steps towards the development of a\  quantitative computational theory capable of explaining the subjective filtering steps that lead to how\  humans learn and consolidate memories.

To view more information and dowload datasets, etc. please visit the project website - http://klab.tch.harvard.edu/resources/Tangetal_episodicmemory_2016.html$\#$sthash.cj1STRah.bumwWxcX.dpbs

The corresponding publication can be found here.

The corresponding code entry can be found here.

}, author = {Hanlin Tang and Jedediah Singer and Matias Ison and Gnel Pivazyan and Melissa Romaine and Rosa Frias and Elizabeth Meller and Adrianna Boulin and James Carroll and Victoria Perron and Sarah Dowcett and Marlise Arlellano and Gabriel Kreiman} } @article {2886, title = {Predicting episodic memory formation for movie events [dataset]}, year = {2016}, abstract = {

Episodic memories are long lasting and full of detail, yet imperfect and malleable. We quantitatively\  evaluated recollection of short audiovisual segments from movies as a proxy to real-life memory\  formation in 161 subjects at 15\  minutes up to a year after encoding. Memories were reproducible within\  and across individuals, showed the typical decay with time elapsed between encoding and testing,\  were fallible yet accurate, and were insensitive to low-level stimulus manipulations but sensitive to\  high-level stimulus properties. Remarkably, memorability was also high for single movie frames, even\  one year post-encoding. To evaluate what determines the efficacy of long-term memory formation,\  we developed an extensive set of content annotations that included actions, emotional valence, visual\  cues and auditory cues. These annotations enabled us to document the content properties that showed\  a stronger correlation with recognition memory and to build a machine-learning computational model\  that accounted for episodic memory formation in single events for group averages and individual\  subjects with an accuracy of up to 80\%. These results provide initial steps towards the development of a\  quantitative computational theory capable of explaining the subjective filtering steps that lead to how\  humans learn and consolidate memories.

To view more information and dowload datasets, etc. please visit the project website - http://klab.tch.harvard.edu/resources/Tangetal_episodicmemory_2016.html$\#$sthash.cj1STRah.bumwWxcX.dpbs

The corresponding publication can be found here.

The corresponding code entry can be found here.

}, author = {Hanlin Tang and Jedediah Singer and Matias Ison and Gnel Pivazyan and Melissa Romaine and Rosa Frias and Elizabeth Meller and Adrianna Boulin and James Carroll and Victoria Perron and Sarah Dowcett and Marlise Arlellano and Gabriel Kreiman} } @article {1154, title = {Sensitivity to timing and order in human visual cortex}, journal = {Journal of Neurophysiology}, volume = {113}, year = {2015}, month = {Jan-03-2015}, pages = {1656 - 1669}, issn = {0022-3077}, doi = {10.1152/jn.00556.2014}, url = {http://jn.physiology.org/lookup/doi/10.1152/jn.00556.2014}, author = {Jedediah Singer and Joseph Madsen and WS Anderson and Gabriel Kreiman} } @article {440, title = {Sensitivity to Timing and Order in Human Visual Cortex.}, number = {005}, year = {2014}, month = {04/2014}, abstract = {

Visual recognition takes a small fraction of a second and relies on the cascade of signals along the ventral visual stream. Given the rapid path through multiple processing steps between photoreceptors and higher visual areas, information must progress from stage to stage very quickly. This rapid progression of information suggests that fine temporal details of the neural response may be important to the how the brain encodes visual signals. We investigated how changes in the relative timing of incoming visual stimulation affect the representation of object information by recording intracranial field potentials along the human ventral visual stream while subjects recognized objects whose parts were presented with varying asynchrony. Visual responses along the ventral stream were sensitive to timing differences between parts as small as 17 ms. In particular, there was a strong dependency on the temporal order of stimulus presentation, even at short asynchronies. This sensitivity to the order of stimulus presentation provides evidence that the brain may use differences in relative timing as a means of representing information.

}, keywords = {Circuits for Intelligence, Pattern recognition, Visual}, author = {Jedediah Singer and Joseph Madsen and WS Anderson and Gabriel Kreiman} } @article {2260, title = {Short temporal asynchrony disrupts visual object recognition}, year = {2014}, publisher = {Journal of Vision}, abstract = {

Intracranial field potential recordings, images and code from Liu et al. (2009). The data shows rapid responses along the ventral visual stream in the human brain with selectivity to faces and objects, tolerance to object transformations than can be decoded in single trials

Used in conjunction with this mirrored CBMM Dataset entry

}, url = {http://klab.tch.harvard.edu/resources/singer_asynchrony.html}, author = {Jedediah Singer and Gabriel Kreiman} } @article {2266, title = {Short temporal asynchrony disrupts visual object recognition}, year = {2014}, publisher = {Journal of Vision}, abstract = {

Intracranial field potential recordings, images and code from Liu et al. (2009). The data shows rapid responses along the ventral visual stream in the human brain with selectivity to faces and objects, tolerance to object transformations than can be decoded in single trials.

Used in conjunction with this mirrored CBMM Code entry

}, url = {http://klab.tch.harvard.edu/resources/singer_asynchrony.html}, author = {Jedediah Singer and Gabriel Kreiman} } @article {1133, title = {Short temporal asynchrony disrupts visual object recognition}, journal = {Journal of Vision}, volume = {12}, year = {2014}, author = {Jedediah Singer and Gabriel Kreiman} } @article {1156, title = {Short temporal asynchrony disrupts visual object recognition.}, journal = {J Vis}, volume = {14}, year = {2014}, month = {2014}, pages = {7}, abstract = {

Humans can recognize objects and scenes in a small fraction of a second. The cascade of signals underlying rapid recognition might be disrupted by temporally jittering different parts of complex objects. Here we investigated the time course over which shape information can be integrated to allow for recognition of complex objects. We presented fragments of object images in an asynchronous fashion and behaviorally evaluated categorization performance. We observed that visual recognition was significantly disrupted by asynchronies of approximately 30 ms, suggesting that spatiotemporal integration begins to break down with even small deviations from simultaneity. However, moderate temporal asynchrony did not completely obliterate recognition; in fact, integration of visual shape information persisted even with an asynchrony of 100 ms. We describe the data with a concise model based on the dynamic reduction of uncertainty about what image was presented. These results emphasize the importance of timing in visual processing and provide strong constraints for the development of dynamical models of visual shape recognition.

}, keywords = {Adult, Female, Form Perception, Humans, Male, Pattern Recognition, Visual, Psychophysics, Time Factors, Vision, Ocular, Visual Pathways, Young Adult}, issn = {1534-7362}, doi = {10.1167/14.5.7}, author = {Jedediah Singer and Gabriel Kreiman} }