From 2d1e65981fa34835140b1901ef70b46b2418b350 Mon Sep 17 00:00:00 2001 From: Martin Schrimpf Date: Sun, 9 Aug 2020 11:23:20 -0400 Subject: [PATCH] update tests/examples to MajajHong identifier; update model references; rename pytorch_model->torchvision_model (#63) * update unit tests to use MajajHong identifier * sneak in fixes to model references * rename pytorch_model to torchvision_model * update example with MajajHong identifier --- candidate_models/base_models/__init__.py | 18 +++++++++--------- candidate_models/base_models/models.csv | 18 ++++++++++-------- examples/score-model.ipynb | 11 +---------- tests/test___init__.py | 18 +++++++++--------- 4 files changed, 29 insertions(+), 36 deletions(-) diff --git a/candidate_models/base_models/__init__.py b/candidate_models/base_models/__init__.py index 7effa38..07fa693 100644 --- a/candidate_models/base_models/__init__.py +++ b/candidate_models/base_models/__init__.py @@ -15,12 +15,12 @@ _logger = logging.getLogger(__name__) -def pytorch_model(function, image_size): +def torchvision_model(identifier, image_size): module = import_module(f'torchvision.models') - model_ctr = getattr(module, function) + model_ctr = getattr(module, identifier) from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=image_size) - wrapper = PytorchWrapper(identifier=function, model=model_ctr(pretrained=True), preprocessing=preprocessing) + wrapper = PytorchWrapper(identifier=identifier, model=model_ctr(pretrained=True), preprocessing=preprocessing) wrapper.image_size = image_size return wrapper @@ -349,12 +349,12 @@ def __init__(self): super(BaseModelPool, self).__init__(reload=True) _key_functions = { - 'alexnet': lambda: pytorch_model('alexnet', image_size=224), - 'squeezenet1_0': lambda: pytorch_model('squeezenet1_0', image_size=224), - 'squeezenet1_1': lambda: pytorch_model('squeezenet1_1', image_size=224), - 'resnet-18': lambda: pytorch_model('resnet18', image_size=224), - 'resnet-34': lambda: pytorch_model('resnet34', image_size=224), - 'resnet-50-pytorch': lambda: pytorch_model('resnet50', image_size=224), + 'alexnet': lambda: torchvision_model('alexnet', image_size=224), + 'squeezenet1_0': lambda: torchvision_model('squeezenet1_0', image_size=224), + 'squeezenet1_1': lambda: torchvision_model('squeezenet1_1', image_size=224), + 'resnet-18': lambda: torchvision_model('resnet18', image_size=224), + 'resnet-34': lambda: torchvision_model('resnet34', image_size=224), + 'resnet-50-pytorch': lambda: torchvision_model('resnet50', image_size=224), 'resnet-50-robust': lambda: robust_model('resnet50', image_size=224), 'vgg-16': lambda: keras_model('vgg16', 'VGG16', image_size=224), diff --git a/candidate_models/base_models/models.csv b/candidate_models/base_models/models.csv index a0b8c07..b35747e 100644 --- a/candidate_models/base_models/models.csv +++ b/candidate_models/base_models/models.csv @@ -10,7 +10,7 @@ publisher = {Curran Associates, Inc.}, url = {http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf} } ",0.577, -CORnet-S,https://www.biorxiv.org/content/early/2018/09/04/408385,"@incollection{Kubilius2018CORnet, +CORnet-S,https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns,"@incollection{Kubilius2018CORnet, title = {CORnet: Modeling the Neural and Behavioral Mechanisms of Core Object Recognition}, author = {Jonas Kubilius and Martin Schrimpf and Aran Nayebi and Daniel Bear and Daniel L. K. Yamins and James J. DiCarlo}, year = {2018}, @@ -1376,11 +1376,13 @@ booktitle = {The European Conference on Computer Vision (ECCV)}, month = {sep}, year = {2018} } ",0.854,0.976 -fixres_resnext101_32x48d_wsl,http://openaccess.thecvf.com/content_ECCV_2018/html/Dhruv_Mahajan_Exploring_the_Limits_ECCV_2018_paper.html,"@article{touvron2019fixres, - author = {Hugo, Touvron and Vedaldi, Andrea and Douze, Matthijs and J{\'e}gou, Herv{\'e}}, - title = ""{Fixing the train-test resolution discrepancy}"", - journal = {arXiv preprint}, - url={https://arxiv.org/abs/1906.06423}, - year = {2019}, - month = {jun}, +fixres_resnext101_32x48d_wsl,http://papers.nips.cc/paper/9035-fixing-the-train-test-resolution-discrepancy,"@incollection{NIPS2019_9035, +title = {Fixing the train-test resolution discrepancy}, +author = {Touvron, Hugo and Vedaldi, Andrea and Douze, Matthijs and Jegou, Herve}, +booktitle = {Advances in Neural Information Processing Systems 32}, +editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett}, +pages = {8252--8262}, +year = {2019}, +publisher = {Curran Associates, Inc.}, +url = {http://papers.nips.cc/paper/9035-fixing-the-train-test-resolution-discrepancy.pdf} }",0.863, diff --git a/examples/score-model.ipynb b/examples/score-model.ipynb index 276a6d7..108031f 100644 --- a/examples/score-model.ipynb +++ b/examples/score-model.ipynb @@ -285,7 +285,7 @@ "name": "stderr", "text": [ "alexnet is accessed again and reloaded\n", - "cross-validation: 100%|██████████| 10/10 [00:21<00:00, 2.15s/it]\u001b[A\n", + "cross-validation: 100%|██████████| 10/10 [00:21<00:00, 2.15s/it]\u001B[A\n", "layers: 100%|██████████| 7/7 [02:43<00:00, 23.40s/it]\n", "activations: 100%|██████████| 3200/3200 [00:24<00:00, 133.19it/s]\n", "layer packaging: 100%|██████████| 1/1 [00:00<00:00, 16.89it/s]\n", @@ -336,15 +336,6 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.8" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "source": [], - "metadata": { - "collapsed": false - } - } } }, "nbformat": 4, diff --git a/tests/test___init__.py b/tests/test___init__.py index 06ae71c..568e97a 100644 --- a/tests/test___init__.py +++ b/tests/test___init__.py @@ -33,21 +33,21 @@ def load(model_name=model_name, layer=layer, region=region, pca_components=pca_c def test_alexnet_conv2_V4(self): model = self.layer_candidate('alexnet', layer='features.5', region='V4', pca_components=1000) score = score_model(model_identifier='alexnet-f5-pca_1000', model=model, - benchmark_identifier='dicarlo.Majaj2015.V4-pls') + benchmark_identifier='dicarlo.MajajHong2015.V4-pls') assert score.raw.sel(aggregation='center').max() == approx(0.633703, abs=0.005) @pytest.mark.memory_intense def test_alexnet_conv5_V4(self): model = self.layer_candidate('alexnet', layer='features.12', region='V4', pca_components=1000) score = score_model(model_identifier='alexnet-f12-pca_1000', model=model, - benchmark_identifier='dicarlo.Majaj2015.V4-pls') + benchmark_identifier='dicarlo.MajajHong2015.V4-pls') assert score.raw.sel(aggregation='center') == approx(0.490769, abs=0.005) @pytest.mark.memory_intense def test_alexnet_conv5_IT(self): model = self.layer_candidate('alexnet', layer='features.12', region='IT', pca_components=1000) score = score_model(model_identifier='alexnet-f12-pca_1000', model=model, - benchmark_identifier='dicarlo.Majaj2015.IT-pls') + benchmark_identifier='dicarlo.MajajHong2015.IT-pls') assert score.raw.sel(aggregation='center') == approx(0.590345, abs=0.005) @pytest.mark.memory_intense @@ -55,16 +55,16 @@ def test_alexnet_conv3_IT_mask(self): model = self.layer_candidate('alexnet', layer='features.6', region='IT', pca_components=None) np.random.seed(123) score = score_model(model_identifier='alexnet-f6', model=model, - benchmark_identifier='dicarlo.Majaj2015.IT-mask') + benchmark_identifier='dicarlo.MajajHong2015.IT-mask') assert score.raw.sel(aggregation='center') == approx(0.607037, abs=0.005) @pytest.mark.memory_intense def test_repeat_same_result(self): model = self.layer_candidate('alexnet', layer='features.12', region='IT', pca_components=1000) score1 = score_model(model_identifier='alexnet-f12-pca_1000', model=model, - benchmark_identifier='dicarlo.Majaj2015.IT-pls') + benchmark_identifier='dicarlo.MajajHong2015.IT-pls') score2 = score_model(model_identifier='alexnet-f12-pca_1000', model=model, - benchmark_identifier='dicarlo.Majaj2015.IT-pls') + benchmark_identifier='dicarlo.MajajHong2015.IT-pls') assert (score1 == score2).all() def test_newmodel_pytorch(self): @@ -104,7 +104,7 @@ def forward(self, x): candidate = TemporalIgnore(candidate) ceiled_score = score_model(model_identifier=model_id, model=candidate, - benchmark_identifier='dicarlo.Majaj2015.IT-pls') + benchmark_identifier='dicarlo.MajajHong2015.IT-pls') score = ceiled_score.raw assert score.sel(aggregation='center') == approx(.0820823, abs=.01) @@ -132,14 +132,14 @@ class TestBrainTranslated: ('alexnet', .59033, True), ('CORnet-S', .600, False), ]) - def test_Majaj2015ITpls(self, model_identifier, expected_score, attach_hook): + def test_MajajHong2015ITpls(self, model_identifier, expected_score, attach_hook): model = brain_translated_pool[model_identifier] if attach_hook: activations_model = model.layer_model._layer_model.activations_model LayerPCA.hook(activations_model, n_components=1000) identifier = activations_model.identifier + "-pca_1000" activations_model.identifier = identifier - score = score_model(model_identifier, 'dicarlo.Majaj2015.IT-pls', model=model) + score = score_model(model_identifier, 'dicarlo.MajajHong2015.IT-pls', model=model) assert score.raw.sel(aggregation='center') == approx(expected_score, abs=0.005) @pytest.mark.parametrize(['model_identifier', 'expected_score'], [