@article {5614, title = {The Indoor-Training Effect: unexpected gains from distribution shifts in the transition function}, year = {2025}, month = {01/2025}, abstract = {

Is it better to perform tennis training in a pristine indoor environment or a noisy outdoor one? To model this problem, here we investigate whether shifts in the transition probabilities between the training and testing environments in reinforcement learning problems can lead to better performance under certain conditions. We generate new Markov Decision Processes (MDPs) starting from a given MDP, by adding quantifiable, parametric noise into the transition function. We refer to this process as Noise Injection and the resulting environments as δenvironments. This process allows us to create variations of the same environment with quantitative control over noise serving as a metric of distance between environments. Conventional wisdom suggests that training and testing on the same MDP should yield the best results. In stark contrast, we observe that agents can perform better when trained on the noise-free environment and tested on the noisy δ-environments, compared to training and testing on the same δ-environments. We confirm that this finding extends beyond noise variations: it is possible to showcase the same phenomenon in ATARI game variations including varying Ghost behaviour in PacMan, and Paddle behaviour in Pong. We demonstrate this intriguing behaviour across 60 different variations of ATARI games, including PacMan, Pong, and Breakout. We refer to this phenomenon as the Indoor-Training Effect. Code to reproduce our experiments and to implement Noise Injection can be found at https://bit.ly/3X6CTYk.

}, url = {https://arxiv.org/abs/2401.15856}, author = {Serena Bono and Spandan Madan and Ishaan Grover and Mao Yasueda and Cynthia Breazeal and Hanspeter Pfister and Gabriel Kreiman} } @conference {5606, title = {Benchmarking Out-of-Distribution Generalization Capabilities of DNN-based Encoding Models for the Ventral Visual Cortex}, booktitle = {NeurIPS 2024}, year = {2024}, month = {06/2024}, abstract = {

We characterized the generalization capabilities of DNN-based encoding models when predicting neuronal responses from the visual cortex. We collected Macaque- ITBench, a large-scale dataset of neural population responses from the macaque inferior temporal (IT) cortex to over 300, 000 images, comprising 8, 233 unique natural images presented to seven monkeys over 109 sessions. Using Macaque- ITBench, we investigated the impact of distribution shifts on models predicting neural activity by dividing the images into Out-Of-Distribution (OOD) train and test splits. The OOD splits included several different image-computable types including image contrast, hue, intensity, temperature, and saturation. Compared to the performance on in-distribution test images{\textemdash}the conventional way these models have been evaluated{\textemdash}models performed worse at predicting neuronal responses to out-of-distribution images, retaining as little as 20\% of the performance on in- distribution test images. The generalization performance under OOD shifts can be well accounted by a simple image similarity metric{\textemdash}the cosine distance between image representations extracted from a pre-trained object recognition model is a strong predictor of neural predictivity under different distribution shifts. The dataset of images, neuronal firing rate recordings, and computational benchmarks are hosted publicly at: MacaqueITBench Link.

}, author = {Spandan Madan and Will Xiao and Mingran Cao and Hanspeter Pfister and Margaret S. Livingstone and Gabriel Kreiman} } @article {5011, title = {Three approaches to facilitate DNN generalization to objects in out-of-distribution orientations and illuminations}, number = {119}, year = {2022}, month = {01/2022}, abstract = {

The training data distribution is often biased towards objects in certain orientations and illumination conditions. While humans have a remarkable capability of recognizing objects in out-of-distribution (OoD) orientations and illuminations, Deep Neural Networks (DNNs) severely suffer in this case, even when large amounts of training examples are available. In this paper, we investigate three different approaches to improve DNNs in recognizing objects in OoD orientations and illuminations. Namely, these are (i) training much longer after convergence of the in-distribution (InD) validation accuracy, i.e., late-stopping, (ii) tuning the momentum parameter of the batch normalization layers, and (iii) enforcing invariance of the neural activity in an intermediate layer to orientation and illumination conditions. Each of these approaches substantially improves the DNN{\textquoteright}s OoD accuracy (more than 20\% in some cases). We report results in four datasets: two datasets are modified from the MNIST and iLab datasets, and the other two are novel (one of 3D rendered cars and another of objects taken from various controlled orientations and illumination conditions). These datasets allow to study the effects of different amounts of bias and are challenging as DNNs perform poorly in OoD conditions. Finally, we demonstrate that even though the three approaches focus on different aspects of DNNs, they all tend to lead to the same underlying neural mechanism to enable OoD accuracy gains {\textendash} individual neurons in the intermediate layers become more selective to a category and also invariant to OoD orientations and illuminations. We anticipate this study to be a basis for further improvement of deep neural networks{\textquoteright} OoD generalization performance, which is highly demanded to achieve safe and fair AI applications.

}, author = {Akira Sakai and Taro Sunagawa and Spandan Madan and Kanata Suzuki and Takashi Katoh and Hiromichi Kobashi and Hanspeter Pfister and Pawan Sinha and Xavier Boix and Tomotake Sasaki} } @proceedings {5042, title = {When Pigs Fly: Contextual Reasoning in Synthetic and Natural Scenes}, year = {2021}, month = {08/2021}, abstract = {

Context is of fundamental importance to both human and machine vision; e.g., an object in the air is more likely to be an airplane than a pig. The rich notion of context incorporates several aspects including physics rules, statistical co-occurrences, and relative object sizes, among others. While previous work has focused on crowd-sourced out-of-context photographs from the web to study scene context, controlling the nature and extent of contextual violations has been a daunting task. Here we introduce a diverse, synthetic Out-of-Context Dataset (OCD) with fine-grained control over scene context. By leveraging a 3D simulation engine, we systematically control the gravity, object co-occurrences and relative sizes across 36 object categories in a virtual household environment. We conducted a series of experiments to gain insights into the impact of contextual cues on both human and machine vision using OCD. We conducted psychophysics experiments to establish a human benchmark for out-of-context recognition, and then compared it with state-of-the-art computer vision models to quantify the gap between the two. We propose a context-aware recognition transformer model, fusing object and contextual information via multi-head attention. Our model captures useful information for contextual reasoning, enabling human-level performance and better robustness in out-of-context conditions compared to baseline models across OCD and other out-of-context datasets. All source code and data are publicly available at https://github.com/kreimanlab/ WhenPigsFlyContext

}, doi = {10.1109/iccv48922.2021.00032}, author = {Philipp Bomatter and Mengmi Zhang and Dimitar Karev and Spandan Madan and Claire Tseng and Gabriel Kreiman} } @article {4583, title = {On the Capability of Neural Networks to Generalize to Unseen Category-Pose Combinations}, year = {2020}, month = {07/2020}, abstract = {

Recognizing an object{\textquoteright}s category and pose lies at the heart of visual understanding. Recent works suggest that deep neural networks (DNNs) often fail to generalize to category-pose combinations not seen during training. However, it is unclear when and how such generalization may be possible. Does the number of combinations seen during training impact generalization? Is it better to learn category and pose in separate networks, or in a single shared network? Furthermore, what are the neural mechanisms that drive the network{\textquoteright}s generalization? In this paper, we answer these questions by analyzing state-of-the-art DNNs trained to recognize both object category and pose (position, scale, and 3D viewpoint) with quantitative control over the number of category-pose combinations seen during training. We also investigate the emergence of two types of specialized neurons that can explain generalization to unseen combinations{\textemdash}neurons selective to category and invariant to pose, and vice versa. We perform experiments on MNIST extended with position or scale, the iLab dataset with vehicles at different viewpoints, and a challenging new dataset for car model recognition and viewpoint estimation that we introduce in this paper, the Biased-Cars dataset. Our results demonstrate that as the number of combinations seen during training increases, networks generalize better to unseen category-pose combinations, facilitated by an increase in the selectivity and invariance of individual neurons. We find that learning category and pose in separate networks compared to a shared one leads to an increase in such selectivity and invariance, as separate networks are not forced to preserve information about both category and pose. This enables separate networks to significantly outperform shared ones at predicting unseen category-pose combinations.

}, author = {Spandan Madan and Timothy Henry and Jamell Dozier and Helen Ho and Nishchal Bhandari and Tomotake Sasaki and Fredo Durand and Hanspeter Pfister and Xavier Boix} }