Init

.gitignore

@@ -0,0 +1,9 @@
+__pycache__
+.eggs
+build
+dist
+*.egg-info
+data
+.idea
+*.h5
+*.pth

LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2019 Qiang Wang
+Copyright (c) 2020 Licht Takeuchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

setup.cfg

@@ -0,0 +1,26 @@
+[metadata]
+name = tf-siammask
+version = attr: siammask.VERSION
+description = SiamMask implementation by Tensorflow 2
+long_description = file: LICENSE
+keywords = yolov4, tensorflow
+license = MIT
+classifiers =
+    Framework :: Tensorflow
+    License :: OSI Approved :: MIT License
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.7
+
+[options]
+pythonrequire = >=3.7,4.0>
+zip_safe = False
+include_package_data = True
+packages = find:
+install_requires =
+    pillow
+    tensorflow
+    numpy
+
+[options.packages.find]
+exclude =
+    tests

setup.py

@@ -0,0 +1,2 @@
+import setuptools
+setuptools.setup()

siammask/__init__.py

@@ -0,0 +1,24 @@
+"""
+MIT License
+
+Copyright (c) 2020 Licht Takeuchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+from .siammask import SiamMask
+
+VERSION = '0.0.1'

siammask/model/__init__.py

@@ -0,0 +1,22 @@
+"""
+MIT License
+
+Copyright (c) 2020 Licht Takeuchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+from .prediction import Prediction

siammask/model/adjuster.py

@@ -0,0 +1,34 @@
+"""
+MIT License
+
+Copyright (c) 2020 Licht Takeuchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+import tensorflow as tf
+
+from .convolution import Conv2DWithBatchNorm
+
+
+class Adjuster(tf.keras.Model):
+    def __init__(self):
+        super(Adjuster, self).__init__()
+
+        self.conv = Conv2DWithBatchNorm(256, 3)
+
+    def call(self, inputs, training=None, mask=None):
+        return tf.keras.activations.relu(self.conv(inputs))

siammask/model/convolution.py

@@ -0,0 +1,41 @@
+"""
+MIT License
+
+Copyright (c) 2020 Licht Takeuchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+import tensorflow as tf
+
+
+class Conv2DWithBatchNorm(tf.keras.layers.Conv2D):
+    def __init__(self,
+                 filters: int,
+                 kernel_size: int,
+                 strides: int = 1,
+                 padding: str = 'VALID',
+                 dilation_rate: int = 1
+                 ):
+        super(Conv2DWithBatchNorm, self).__init__(
+            filters=filters, kernel_size=kernel_size, strides=strides,
+            padding=padding, dilation_rate=dilation_rate, use_bias=False,
+            kernel_initializer='he_normal'
+        )
+        self.bn = tf.keras.layers.BatchNormalization()
+
+    def call(self, x, **kwargs):
+        return self.bn(super(Conv2DWithBatchNorm, self).call(x))

siammask/model/mask_refinement.py

@@ -0,0 +1,38 @@
+"""
+MIT License
+
+Copyright (c) 2020 Licht Takeuchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+import tensorflow as tf
+
+
+class MaskRefinementBlock(tf.keras.Model):
+    def __init__(self):
+        super(MaskRefinementBlock, self).__init__()
+
+    def call(self, inputs, training=None, mask=None):
+        pass
+
+
+class MaskRefinementNetwork(tf.keras.Model):
+    def __init__(self):
+        super(MaskRefinementNetwork, self).__init__()
+
+    def call(self, inputs, training=None, mask=None):
+        pass

siammask/model/prediction.py

@@ -0,0 +1,67 @@
+"""
+MIT License
+
+Copyright (c) 2020 Licht Takeuchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+import tensorflow as tf
+
+from .resnet import CustomResNet50
+from .proposal_network import ProposalNetwork
+
+
+class Prediction(tf.keras.Model):
+    def __init__(self, num_anchors, kernel_cut_off_size):
+        super(Prediction, self).__init__()
+
+        self.num_anchors = num_anchors
+        self.cut_off = kernel_cut_off_size // 2
+
+        self.resnet = CustomResNet50()
+
+        self.score_proposal = ProposalNetwork(2 * self.num_anchors)
+        self.box_proposal = ProposalNetwork(4 * self.num_anchors)
+        self.mask_proposal = ProposalNetwork(63 * 63)
+
+    def call(self, inputs, training=None, mask=None):
+        exampler, search = inputs
+
+        exampler_features, res3, res2, res1 = self.resnet(exampler)
+        exampler_features = exampler_features[:, self.cut_off:-self.cut_off, self.cut_off:-self.cut_off, :]
+        search_features, _, _, _ = self.resnet(search)
+
+        scores, _ = self.score_proposal([exampler_features, search_features])
+        boxes, _ = self.box_proposal([exampler_features, search_features])
+        masks, mask_features = self.mask_proposal([exampler_features, search_features])
+
+        batch_size = tf.shape(scores)[0]
+        h = tf.shape(scores)[1]
+        w = tf.shape(scores)[2]
+        scores = tf.reshape(scores, (batch_size, h, w, self.num_anchors, 2))
+
+        scores = tf.keras.activations.softmax(scores)[..., 1]
+
+        batch_size = tf.shape(boxes)[0]
+        h = tf.shape(boxes)[1]
+        w = tf.shape(boxes)[2]
+        boxes = tf.reshape(boxes, (batch_size, h, w, self.num_anchors, 4))
+
+        xy = boxes[..., :2]
+        wh = tf.keras.backend.exp(boxes[..., 2:])
+
+        return [scores, tf.concat([xy, wh], -1), masks]

siammask/model/proposal_network.py

@@ -0,0 +1,72 @@
+"""
+MIT License
+
+Copyright (c) 2020 Licht Takeuchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+import tensorflow as tf
+
+from .adjuster import Adjuster
+from .convolution import Conv2DWithBatchNorm
+
+
+class ProposalNetwork(tf.keras.Model):
+    def __init__(self, output_channels):
+        super(ProposalNetwork, self).__init__()
+
+        self.exampler_adjuster = Adjuster()
+        self.search_adjuster = Adjuster()
+
+        self.conv1 = Conv2DWithBatchNorm(256, 1)
+        self.conv2 = tf.keras.layers.Conv2D(output_channels, 1)
+
+    def call(self, inputs, training=None, mask=None):
+        exampler_features, search_features = inputs
+
+        kernel = self.exampler_adjuster(exampler_features)
+        search_features = self.search_adjuster(search_features)
+
+        # TODO: ???
+        batch_size = tf.shape(kernel)[0]
+        kernel_h = 5
+        kernel_w = 5
+        channel_size = 256
+
+        image_patches = tf.image.extract_patches(
+            search_features,
+            [1, kernel_h, kernel_w, 1],
+            [1, 1, 1, 1], [1, 1, 1, 1],
+            padding='VALID'
+        )
+
+        correlation_h = tf.shape(image_patches)[1]
+        correlation_w = tf.shape(image_patches)[2]
+
+        kernel = tf.reshape(kernel, (batch_size, 1, 1, kernel_h * kernel_w * channel_size))
+
+        correlation = tf.reduce_sum(
+            tf.reshape(
+                tf.multiply(image_patches, kernel),
+                (batch_size, correlation_h, correlation_w, -1, channel_size)
+            ),
+            axis=-2
+        )
+
+        output = tf.keras.activations.relu(self.conv1(correlation))
+
+        return [self.conv2(output), correlation]