-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathConvMixer.py
executable file
·74 lines (62 loc) · 2.17 KB
/
ConvMixer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
"""
ConvMixer.py - TensorFlow implementation of
ICLR 2022 submission "Patches Are All You Need?"
Dan Mezhiborsky - @dmezh
Theo Jaquenoud - @thjaquenoud
See:
https://github.com/dmezh/convmixer-tf
https://github.com/tmp-iclr/convmixer
Our final layer uses softmax activation.
"""
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow import keras
class Residual(keras.layers.Layer):
def __init__(self, fn):
super().__init__()
self.fn = fn
def call(self, x):
return self.fn(x) + x
def GELU():
return keras.layers.Activation(tf.keras.activations.gelu)
def ConvMixer(dim, depth, kernel_size=9, patch_size=7, n_classes=10):
return keras.Sequential(
[
keras.layers.Conv2D(
dim,
kernel_size=(patch_size, patch_size),
strides=(patch_size, patch_size),
input_shape=(32, 32, 3),
),
GELU(),
keras.layers.BatchNormalization(),
*[
keras.Sequential(
[
Residual(
keras.Sequential(
[
keras.layers.Conv2D(
dim,
kernel_size=(kernel_size, kernel_size),
groups=dim,
padding="same",
),
GELU(),
keras.layers.BatchNormalization(),
]
)
),
keras.layers.Conv2D(dim, kernel_size=(1, 1)),
GELU(),
keras.layers.BatchNormalization(),
]
)
for i in range(depth)
],
tfa.layers.AdaptiveAveragePooling2D((1, 1)),
keras.layers.Flatten(),
keras.layers.Activation(tf.keras.activations.linear),
keras.layers.Dense(n_classes, activation="softmax"),
]
)