add points associations renderer

src/main/kotlin/solve/constants/ResourcesPaths.kt

@@ -41,3 +41,5 @@ const val ShadersLineLandmarkVertexPath = "engine/shaders/landmark/line/line.ver
 const val ShadersLineLandmarkFragmentPath = "engine/shaders/landmark/line/line.frag"
 const val ShadersPlaneLandmarkVertexPath = "engine/shaders/landmark/plane/plane.vert"
 const val ShadersPlaneLandmarkFragmentPath = "engine/shaders/landmark/plane/plane.frag"
+const val ShadersPointAssociationVertexPath = "engine/shaders/association/point/point.vert"
+const val ShadersPointAssociationFragmentPath = "engine/shaders/association/point/point.frag"

src/main/kotlin/solve/rendering/canvas/SceneCanvas.kt

@@ -12,6 +12,7 @@ import solve.rendering.engine.core.input.MouseInputHandler
 import solve.rendering.engine.core.renderers.FramesRenderer
 import solve.rendering.engine.core.renderers.LinesLayerRenderer
 import solve.rendering.engine.core.renderers.PlanesLayerRenderer
+import solve.rendering.engine.core.renderers.PointAssociationsRenderer
 import solve.rendering.engine.core.renderers.PointsLayerRenderer
 import solve.rendering.engine.core.renderers.Renderer
 import solve.rendering.engine.utils.minus
@@ -24,6 +25,7 @@ import solve.scene.model.Landmark
 import solve.scene.model.Layer
 import solve.scene.model.Scene
 import solve.scene.model.VisualizationFrame
+import solve.scene.view.association.AssociationManager
 import solve.utils.ServiceLocator
 import solve.utils.action
 import solve.utils.ceilToInt
@@ -43,7 +45,6 @@ class SceneCanvas : OpenGLCanvas() {
     private var leftUpperCornerCameraPosition = Vector2f()
     private var rightLowerCornerCameraPosition = Vector2f()
 
-    private var lastLayersWithCommonSettings = listOf<Layer>()
     private var lastFramesSelection = listOf<VisualizationFrame>()
     private var framesSelectionSize = 0
     private var framesSize = Vector2i()
@@ -60,6 +61,8 @@ class SceneCanvas : OpenGLCanvas() {
 
     private var contextMenuInvokeFrame: VisualizationFrame? = null
 
+    private val associationManager = AssociationManager()
+
     private val contextMenu = buildContextMenu(canvas)
 
     init {
@@ -69,8 +72,7 @@ class SceneCanvas : OpenGLCanvas() {
     fun setNewScene(scene: Scene) {
         this.scene = scene
         this.framesSize = Vector2i(scene.frameSize.width.toInt(), scene.frameSize.height.toInt())
-        engineScene?.framesRenderer?.setNewSceneFrames(scene.frames, framesSize.toFloatVector())
-        engineScene?.landmarkRenderers?.forEach { it.setNewSceneFrames(scene.frames, framesSize.toFloatVector()) }
+        engineScene?.setNewScene(scene)
         isFirstFramesSelection = true
 
         needToReinitializeRenderers = true
@@ -85,23 +87,17 @@ class SceneCanvas : OpenGLCanvas() {
         recalculateCameraCornersPositions()
         window.camera.position = leftUpperCornerCameraPosition
 
-        engineScene?.framesRenderer?.setFramesSelection(framesSelection)
-        engineScene?.landmarkRenderers?.forEach { it.setFramesSelection(framesSelection) }
-        engineScene?.landmarkRenderers?.forEach { renderer ->
-            val rendererLayerSettings =
-                engineScene?.landmarkLayerRendererLayers?.get(renderer)?.settings ?: return@forEach
-            lastLayersWithCommonSettings =
-                scene?.getLayersWithCommonSettings(rendererLayerSettings, framesSelection) ?: return@forEach
-            renderer.setFramesSelectionLayers(lastLayersWithCommonSettings)
-        }
+        engineScene?.setFramesSelection(framesSelection)
+        associationManager.setFramesSelection(framesSelection)
         framesSelectionSize = framesSelection.count()
         lastFramesSelection = framesSelection
+
+        associationManager.associate(0, 12)
     }
 
     fun setColumnsNumber(columnsNumber: Int) {
-        engineScene?.framesRenderer?.setNewGridWidth(columnsNumber)
-        engineScene?.landmarkRenderers?.forEach { it.setNewGridWidth(columnsNumber) }
         this.columnsNumber = columnsNumber
+        engineScene?.setColumnsNumber(columnsNumber)
     }
 
     fun dragTo(toScreenPoint: Vector2i) {
@@ -160,8 +156,13 @@ class SceneCanvas : OpenGLCanvas() {
         super.onInit()
         val controller = ServiceLocator.getService<SceneController>() ?: return
         sceneController = controller
+        val framesRenderer = FramesRenderer(window)
+        val pointAssociationsRenderer = PointAssociationsRenderer(window, associationManager)
 
-        engineScene = EngineScene(FramesRenderer(window))
+        engineScene = EngineScene(
+            framesRenderer,
+            pointAssociationsRenderer
+        ) { scene }
     }
 
     override fun onDraw(deltaTime: Float) {

src/main/kotlin/solve/rendering/engine/core/renderers/LandmarkLayerRenderer.kt

@@ -97,21 +97,4 @@ abstract class LandmarkLayerRenderer(
         this._visibleLayers = newVisibleLayers
         _visibleLayersSelectionIndices = newVisibleLayersSelectionIndices
     }
-
-    protected fun getFrameTopLeftShaderPosition(frameIndex: Int): Vector2f {
-        val frameXIndex = frameIndex % gridWidth
-        val frameYIndex = frameIndex / gridWidth
-
-        return Vector2f(
-            frameXIndex.toFloat() * framesRatio + frameXIndex * FramesSpacing,
-            frameYIndex.toFloat() + frameYIndex * FramesSpacing
-        )
-    }
-
-    protected fun getFramePixelShaderPosition(frameIndex: Int, framePixelPosition: Vector2f): Vector2f {
-        val frameRelativePosition = Vector2f(framePixelPosition) / framesSize.y
-        val frameTopLeftPosition = getFrameTopLeftShaderPosition(frameIndex)
-
-        return frameTopLeftPosition + frameRelativePosition
-    }
 }

src/main/kotlin/solve/rendering/engine/core/renderers/LinesLayerRenderer.kt

@@ -38,7 +38,6 @@ class LinesLayerRenderer(
     override fun createNewBatch(zIndex: Int): RenderBatch {
         val shaderAttributesTypes = listOf(
             ShaderAttributeType.FLOAT2,
-            ShaderAttributeType.FLOAT,
             ShaderAttributeType.FLOAT3
         )
 
@@ -72,7 +71,7 @@ class LinesLayerRenderer(
         val linesWidth = getLinesWidth()
 
         visibleLineLayersLandmarks.forEachIndexed { visibleLayerIndex, linesLayerLandmarks ->
-            linesLayerLandmarks.forEachIndexed { lineLandmarkIndex, lineLandmark ->
+            linesLayerLandmarks.forEach { lineLandmark ->
                 val selectionLayerIndex = visibleLayersSelectionIndices[visibleLayerIndex]
                 val batch = getAvailableBatch(null, 0)
 
@@ -119,10 +118,8 @@ class LinesLayerRenderer(
                     val firstVertexPosition = if (sideIndex == 0) upperVertexPosition else bottomVertexPosition
                     val secondVertexPosition = if (sideIndex == 0) bottomVertexPosition else upperVertexPosition
                     batch.pushVector2f(firstVertexPosition)
-                    batch.pushFloat(lineLandmarkIndex.toFloat())
                     batch.pushVector3f(lineColorVector)
                     batch.pushVector2f(secondVertexPosition)
-                    batch.pushFloat(lineLandmarkIndex.toFloat())
                     batch.pushVector3f(lineColorVector)
                 }
             }

src/main/kotlin/solve/rendering/engine/core/renderers/PointAssociationsRenderer.kt

@@ -0,0 +1,114 @@
+package solve.rendering.engine.core.renderers
+
+import org.joml.Vector2f
+import org.joml.Vector3f
+import solve.constants.ShadersPointAssociationFragmentPath
+import solve.constants.ShadersPointAssociationVertexPath
+import solve.rendering.engine.Window
+import solve.rendering.engine.core.batch.PrimitiveType
+import solve.rendering.engine.core.batch.RenderBatch
+import solve.rendering.engine.shader.ShaderAttributeType
+import solve.rendering.engine.shader.ShaderProgram
+import solve.rendering.engine.shader.ShaderType
+import solve.rendering.engine.utils.minus
+import solve.rendering.engine.utils.plus
+import solve.rendering.engine.utils.times
+import solve.rendering.engine.utils.toFloatVector
+import solve.rendering.engine.utils.toVector2i
+import solve.scene.view.association.AssociationManager
+
+class PointAssociationsRenderer(
+    window: Window,
+    private val associationManager: AssociationManager
+) : Renderer(window) {
+    override val maxBatchSize = 1000
+
+    init {
+        renderPriority = 1
+    }
+
+    override fun createShaderProgram(): ShaderProgram {
+        val shaderProgram = ShaderProgram()
+        shaderProgram.addShader(ShadersPointAssociationVertexPath, ShaderType.VERTEX)
+        shaderProgram.addShader(ShadersPointAssociationFragmentPath, ShaderType.FRAGMENT)
+        shaderProgram.link()
+
+        return shaderProgram
+    }
+
+    override fun createNewBatch(zIndex: Int): RenderBatch {
+        val shaderAttributesTypes = listOf(
+            ShaderAttributeType.FLOAT2,
+            ShaderAttributeType.FLOAT3
+        )
+
+        return RenderBatch(
+            maxBatchSize,
+            zIndex,
+            PrimitiveType.Quad,
+            shaderAttributesTypes
+        )
+    }
+
+    override fun uploadUniforms(shaderProgram: ShaderProgram) {
+        shaderProgram.uploadMatrix4f(ProjectionUniformName, window.calculateProjectionMatrix())
+    }
+
+    override fun updateBatchesData() {
+        val associationConnections = associationManager.associationConnections
+
+        associationConnections.forEach { associationConnection ->
+            associationConnection.associationLines.forEach { associationLine ->
+                val batch = getAvailableBatch(null, 0)
+
+                val lineStartShaderPosition = getFramePixelShaderPosition(
+                    associationLine.firstKeypointFrameIndex,
+                    associationLine.firstKeypoint.coordinate.toVector2i().toFloatVector()
+                )
+                val lineFinishShaderPosition = getFramePixelShaderPosition(
+                    associationLine.secondKeypointFrameIndex,
+                    associationLine.secondKeypoint.coordinate.toVector2i().toFloatVector()
+                )
+
+                println("line start shader pos: ${lineStartShaderPosition}")
+                println("line finish shader pos: ${lineFinishShaderPosition}")
+
+                val lineVector = lineFinishShaderPosition - lineStartShaderPosition
+                val normalVector = Vector2f(-lineVector.y, lineVector.x).normalize()
+                val linePoints = listOf(lineStartShaderPosition, lineFinishShaderPosition)
+
+                val firstKeypointColor =
+                    associationLine.firstKeypoint.layerSettings.getColor(associationLine.firstKeypoint)
+                val secondKeypointColor =
+                    associationLine.secondKeypoint.layerSettings.getColor(associationLine.secondKeypoint)
+                val lineColor = firstKeypointColor.interpolate(secondKeypointColor, 0.5)
+
+                val lineColorVector = Vector3f(
+                    lineColor.red.toFloat(),
+                    lineColor.green.toFloat(),
+                    lineColor.blue.toFloat()
+                )
+
+                linePoints.forEachIndexed { sideIndex, linePoint ->
+                    val pointToVertexVector = Vector2f(normalVector) * AssociationLineWidth / window.camera.zoom /
+                            DefaultLocalVerticesPositionsDivider
+
+                    val upperVertexPosition = linePoint + pointToVertexVector
+                    val bottomVertexPosition = linePoint - pointToVertexVector
+                    val firstVertexPosition = if (sideIndex == 0) upperVertexPosition else bottomVertexPosition
+                    val secondVertexPosition = if (sideIndex == 0) bottomVertexPosition else upperVertexPosition
+                    batch.pushVector2f(firstVertexPosition)
+                    batch.pushVector3f(lineColorVector)
+                    batch.pushVector2f(secondVertexPosition)
+                    batch.pushVector3f(lineColorVector)
+                }
+            }
+        }
+    }
+
+    companion object {
+        private const val DefaultLocalVerticesPositionsDivider = 800f
+
+        private const val AssociationLineWidth = 3.0f
+    }
+}

src/main/kotlin/solve/rendering/engine/core/renderers/Renderer.kt

@@ -8,6 +8,7 @@ import solve.rendering.engine.Window
 import solve.rendering.engine.core.batch.RenderBatch
 import solve.rendering.engine.core.texture.Texture
 import solve.rendering.engine.shader.ShaderProgram
+import solve.rendering.engine.utils.plus
 import solve.rendering.engine.utils.toIntVector
 import solve.scene.model.VisualizationFrame
 import solve.utils.ceilToInt
@@ -168,6 +169,24 @@ abstract class Renderer(protected val window: Window) : Comparable<Renderer> {
         batches.forEach { it.rebuffer() }
     }
 
+    protected fun getFrameTopLeftShaderPosition(frameIndex: Int): Vector2f {
+        val frameXIndex = frameIndex % gridWidth
+        val frameYIndex = frameIndex / gridWidth
+        val framesRatio = framesSize.x / framesSize.y
+
+        return Vector2f(
+            frameXIndex.toFloat() * framesRatio + frameXIndex * FramesSpacing,
+            frameYIndex.toFloat() + frameYIndex * FramesSpacing
+        )
+    }
+
+    protected fun getFramePixelShaderPosition(frameIndex: Int, framePixelPosition: Vector2f): Vector2f {
+        val frameRelativePosition = Vector2f(framePixelPosition) / framesSize.y
+        val frameTopLeftPosition = getFrameTopLeftShaderPosition(frameIndex)
+
+        return frameTopLeftPosition + frameRelativePosition
+    }
+
     companion object {
         const val ProjectionUniformName = "uProjection"
         const val GridWidthUniformName = "uGridWidth"

src/main/kotlin/solve/rendering/engine/scene/Scene.kt

@@ -4,21 +4,60 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
+import org.joml.Vector2i
 import solve.rendering.engine.core.renderers.FramesRenderer
 import solve.rendering.engine.core.renderers.LandmarkLayerRenderer
+import solve.rendering.engine.core.renderers.PointAssociationsRenderer
+import solve.rendering.engine.utils.toFloatVector
 import solve.scene.model.Layer
+import solve.scene.model.Scene
+import solve.scene.model.VisualizationFrame
 import java.util.concurrent.CopyOnWriteArrayList
 
-class Scene(val framesRenderer: FramesRenderer) {
+class Scene(
+    val framesRenderer: FramesRenderer,
+    val pointAssociationsRenderer: PointAssociationsRenderer,
+    val getProjectScene: () -> Scene?
+) {
     private val _landmarkRenderers = CopyOnWriteArrayList<LandmarkLayerRenderer>()
     private val _landmarkRenderersLayers = mutableMapOf<LandmarkLayerRenderer, Layer>()
+
     val landmarkRenderers: List<LandmarkLayerRenderer>
         get() = _landmarkRenderers
     val landmarkLayerRendererLayers: Map<LandmarkLayerRenderer, Layer>
         get() = _landmarkRenderersLayers
 
     var isFramesRendererInitializing = false
 
+    fun setNewScene(scene: Scene) {
+        val framesSize = Vector2i(scene.frameSize.width.toInt(), scene.frameSize.height.toInt())
+
+        framesRenderer.setNewSceneFrames(scene.frames, framesSize.toFloatVector())
+        pointAssociationsRenderer.setNewSceneFrames(scene.frames, framesSize.toFloatVector())
+        landmarkRenderers.forEach { it.setNewSceneFrames(scene.frames, framesSize.toFloatVector()) }
+    }
+
+    fun setFramesSelection(selection: List<VisualizationFrame>) {
+        framesRenderer.setFramesSelection(selection)
+        pointAssociationsRenderer.setFramesSelection(selection)
+        _landmarkRenderers.forEach {
+            it.setFramesSelection(selection)
+        }
+        landmarkRenderers.forEach { renderer ->
+            val rendererLayerSettings =
+                landmarkLayerRendererLayers[renderer]?.settings ?: return@forEach
+            val lastLayersWithCommonSettings =
+                getProjectScene()?.getLayersWithCommonSettings(rendererLayerSettings, selection) ?: return@forEach
+            renderer.setFramesSelectionLayers(lastLayersWithCommonSettings)
+        }
+    }
+
+    fun setColumnsNumber(columnsNumber: Int) {
+        framesRenderer.setNewGridWidth(columnsNumber)
+        pointAssociationsRenderer.setNewGridWidth(columnsNumber)
+        landmarkRenderers.forEach { it.setNewGridWidth(columnsNumber) }
+    }
+
     fun update() {
         render()
     }
@@ -48,6 +87,7 @@ class Scene(val framesRenderer: FramesRenderer) {
             return
         }
 
+        pointAssociationsRenderer.render()
         _landmarkRenderers.sort()
         _landmarkRenderers.forEach { it.render() }
     }

src/main/kotlin/solve/scene/view/SceneView.kt

@@ -10,7 +10,6 @@ import solve.scene.SceneFacade
 import solve.scene.controller.SceneController
 import solve.scene.model.Landmark
 import solve.scene.model.VisualizationFrame
-import solve.scene.view.association.AssociationsManager
 import tornadofx.View
 import tornadofx.onChange
 import javafx.scene.input.MouseButton as JavaFXMouseButton
@@ -27,9 +26,6 @@ class SceneView : View() {
 
     private var wasMouseDragging = false
 
-    var currentAssociationsManager: AssociationsManager<VisualizationFrame, Landmark.Keypoint>? = null
-        private set
-
     override val root = canvas.canvas
 
     private val sceneChangedEventHandler = InvalidationListener {

src/main/kotlin/solve/scene/view/association/AssociationConnection.kt

@@ -0,0 +1,8 @@
+package solve.scene.view.association
+
+data class AssociationConnection(
+    val firstFrameIndex: Int,
+    val secondFrameIndex: Int,
+    val keypointLayerIndex: Int,
+    val associationLines: List<AssociationLine>
+)