@article {5217,
	title = {Dynamics in Deep Classifiers trained with the Square Loss: normalization, low rank, neural collapse and generalization bounds},
	journal = {Research},
	year = {2023},
	month = {01/2023},
	abstract = {<p>We overview several properties {\textendash} old and new {\textendash} of training overparametrized deep networks under the square loss. We first consider a model of the dynamics of gradient flow under the square loss in deep homogeneous ReLU networks. We study the convergence to a solution with the absolute minimum ρ, which is the product of the Frobenius norms of each layer weight matrix, when normalization by Lagrange multipliers (LM) is used together with Weight Decay (WD) under different forms of gradient descent. A main property of the minimizers that bounds their expected error for a specific network architecture is ρ. In particular, we derive novel norm- based bounds for convolutional layers that are orders of magnitude better than classical bounds for dense networks. Next we prove that quasi-interpolating solutions obtained by Stochastic Gradient Descent (SGD) in the presence of WD have a bias towards low rank weight matrices {\textendash} that should improve generalization. The same analysis predicts the existence of an inherent SGD noise for deep networks. In both cases, we verify our predictions experimentally. We then predict Neural Collapse and its properties without any specific assumption {\textendash} unlike other published proofs. Our analysis supports the idea that the advantage of deep networks relative to other classifiers is greater for problems that are appropriate for sparse deep architectures such as CNNs. The reason is that compositionally sparse target functions can be approximated well by {\textquotedblleft}sparse{\textquotedblright} deep networks without incurring in the curse of dimensionality.</p>
},
	doi = {10.34133/research.0024},
	url = {https://spj.science.org/doi/10.34133/research.0024},
	author = {Xu, Mengjia and Rangamani, Akshay and Liao, Qianli and Galanti, Tomer and Tomaso Poggio}
}
@book {5289,
	title = {Lecture Notes in Computer ScienceComputer Vision {\textendash} ECCV 2022Image2Point: 3D Point-Cloud Understanding with~2D Image Pretrained Models},
	volume = {13697},
	year = {2022},
	month = {10/2022},
	pages = {638 - 656},
	publisher = {Springer Nature Switzerland},
	organization = {Springer Nature Switzerland},
	address = {Cham},
	abstract = {<p>3D point-clouds and 2D images are different visual representations of the physical world. While human vision can understand both representations, computer vision models designed for 2D image and 3D point-cloud understanding are quite different. Our paper explores the potential of transferring 2D model architectures and weights to understand 3D point-clouds, by empirically investigating the feasibility of the transfer, the benefits of the transfer, and shedding light on why the transfer works. We discover that we can indeed use the same architecture and pretrained weights of a neural net model to understand both images and point-clouds. Specifically, we transfer the image-pretrained model to a point-cloud model by copying or inflating the weights. We find that finetuning the transformed image-pretrained models (FIP) with minimal efforts{\textemdash}only on input, output, and normalization layers{\textemdash}can achieve competitive performance on 3D point-cloud classification, beating a wide range of point-cloud models that adopt task-specific architectures and use a variety of tricks. When finetuning the whole model, the performance gets further improved. Meanwhile, FIP improves data efficiency, reaching up to 10.0 top-1 accuracy percent on few-shot classification. It also speeds up the training of point-cloud models by up to 11.1x for a target accuracy (e.g., 90\% accuracy). Lastly, we provide an explanation of the image to point-cloud transfer from the aspect of neural collapse. The code is available at: https://github.com/chenfengxu714/image2point.</p>
},
	isbn = {978-3-031-19835-9},
	issn = {0302-9743},
	doi = {10.1007/978-3-031-19836-610.1007/978-3-031-19836-6_36},
	url = {https://link.springer.com/10.1007/978-3-031-19836-6},
	author = {Xu, Chenfeng and Yang, Shijia and Galanti, Tomer and Wu, Bichen and Yue, Xiangyu and Zhai, Bohan and Zhan, Wei and Vajda, Peter and Keutzer, Kurt and Tomizuka, Masayoshi},
	editor = {Avidan, Shai and Brostow, Gabriel and Ciss{\'e}, Moustapha and Farinella, Giovanni Maria and Hassner, Tal}
}
