Skip to content

Commit

Permalink
add planes selection
Browse files Browse the repository at this point in the history
  • Loading branch information
@mi-sts
mi-sts committed May 18, 2024
1 parent e364889 commit 19d035e
Show file tree
Hide file tree
Showing 15 changed files with 212 additions and 89 deletions.
3 changes: 1 addition & 2 deletions src/main/kotlin/solve/parsers/factories/PlaneFactory.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ object PlaneFactory : LandmarkFactory<StoragePlane, ScenePlane, PlaneLayerSettin
): ScenePlane = ScenePlane(
storageFormatLandmark.uid,
containingLayerSettings,
containingLayerState,
storageFormatLandmark.points
containingLayerState
)
}
80 changes: 48 additions & 32 deletions src/main/kotlin/solve/parsers/planes/ImagePlanesParser.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,37 +28,6 @@ object ImagePlanesParser : Parser<Plane> {
const val ColorBitsNumber = 8
const val BackgroundColor = 0

private fun convertSeparateToWholeRGB(r: UByte, g: UByte, b: UByte): Int =
(r.toInt() shl ColorBitsNumber * 2) + (g.toInt() shl ColorBitsNumber) + b.toInt()

private fun getPlanePixelColor(
pixelIndex: Int,
imageByteDataArray: ByteArray,
segmentsType: ColorSegmentsType
): Int {
val segmentsByteOffset = segmentsType.segmentsByteOffset

return convertSeparateToWholeRGB(
imageByteDataArray[pixelIndex + segmentsByteOffset + 2].toUByte(),
imageByteDataArray[pixelIndex + segmentsByteOffset + 1].toUByte(),
imageByteDataArray[pixelIndex + segmentsByteOffset].toUByte()
)
}

private fun getImageByteDataArray(image: BufferedImage) = (image.data.dataBuffer as DataBufferByte).data

private inline fun BufferedImage.forEachPixelColor(action: (index: Int, color: Int) -> Unit) {
val imageByteDataArray = getImageByteDataArray(this)
val colorSegmentsType: ColorSegmentsType =
ColorSegmentsType.getColorSegmentsType(colorModel.numComponents) ?: return
val colorComponentsNumber = colorSegmentsType.colorComponentsNumber

for (i in imageByteDataArray.indices step colorComponentsNumber) {
val pixelColor = getPlanePixelColor(i, imageByteDataArray, colorSegmentsType)
action(i / colorComponentsNumber, pixelColor)
}
}

override fun parse(filePath: String): List<Plane> {
val bufferedImage = loadBufferedImage(filePath) ?: return emptyList()

Expand All @@ -75,7 +44,7 @@ object ImagePlanesParser : Parser<Plane> {
}
}

return planePoints.keys.map { Plane(it, planePoints[it] ?: emptyList()) }
return planePoints.keys.map { Plane(it) }
}

override fun extractUIDs(filePath: String): List<Long> {
Expand All @@ -92,4 +61,51 @@ object ImagePlanesParser : Parser<Plane> {

return uids.toList()
}

fun getPixelColor(filePath: String, pixelPosition: Point) : Int? {
val bufferedImage = loadBufferedImage(filePath) ?: return null

return bufferedImage.getPixelColor(pixelPosition)
}

private fun convertSeparateToWholeRGB(r: UByte, g: UByte, b: UByte): Int =
(r.toInt() shl ColorBitsNumber * 2) + (g.toInt() shl ColorBitsNumber) + b.toInt()

private fun getImageByteDataPixelColor(
pixelIndex: Int,
imageByteDataArray: ByteArray,
segmentsType: ColorSegmentsType
): Int {
val segmentsByteOffset = segmentsType.segmentsByteOffset

return convertSeparateToWholeRGB(
imageByteDataArray[pixelIndex + segmentsByteOffset + 2].toUByte(),
imageByteDataArray[pixelIndex + segmentsByteOffset + 1].toUByte(),
imageByteDataArray[pixelIndex + segmentsByteOffset].toUByte()
)
}

private fun getImageByteDataArray(image: BufferedImage) = (image.data.dataBuffer as DataBufferByte).data

private inline fun BufferedImage.forEachPixelColor(action: (index: Int, color: Int) -> Unit) {
val imageByteDataArray = getImageByteDataArray(this)
val colorSegmentsType: ColorSegmentsType =
ColorSegmentsType.getColorSegmentsType(colorModel.numComponents) ?: return
val colorComponentsNumber = colorSegmentsType.colorComponentsNumber

for (i in imageByteDataArray.indices step colorComponentsNumber) {
val pixelColor = getImageByteDataPixelColor(i, imageByteDataArray, colorSegmentsType)
action(i / colorComponentsNumber, pixelColor)
}
}

private fun BufferedImage.getPixelColor(pixelPosition: Point) : Int? {
val imageByteDataArray = getImageByteDataArray(this)
val colorSegmentsType: ColorSegmentsType =
ColorSegmentsType.getColorSegmentsType(colorModel.numComponents) ?: return null
val colorComponentsNumber = colorSegmentsType.colorComponentsNumber
val pixelFirstComponentIndex = (pixelPosition.y * this.width + pixelPosition.x) * colorComponentsNumber

return getImageByteDataPixelColor(pixelFirstComponentIndex, imageByteDataArray, colorSegmentsType)
}
}
4 changes: 1 addition & 3 deletions src/main/kotlin/solve/parsers/structures/Plane.kt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
package solve.parsers.structures

import solve.scene.model.Point

data class Plane(val uid: Long, val points: List<Point>)
data class Plane(val uid: Long)
41 changes: 31 additions & 10 deletions src/main/kotlin/solve/rendering/canvas/SceneCanvas.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import javafx.scene.input.Clipboard
import javafx.scene.input.ClipboardContent
import org.joml.Vector2f
import org.joml.Vector2i
import solve.parsers.planes.ImagePlanesParser
import solve.rendering.engine.Window
import solve.rendering.engine.core.input.MouseButton
import solve.rendering.engine.core.input.MouseInputHandler
Expand All @@ -21,8 +22,9 @@ import solve.rendering.engine.utils.times
import solve.rendering.engine.utils.toFloatVector
import solve.rendering.engine.utils.toIntVector
import solve.scene.controller.SceneController
import solve.scene.model.Landmark
import solve.scene.model.Layer
import solve.scene.model.LayerState
import solve.scene.model.Point
import solve.scene.model.Scene
import solve.scene.model.VisualizationFrame
import solve.scene.view.association.AssociationAdorner
Expand Down Expand Up @@ -198,7 +200,8 @@ class SceneCanvas : OpenGLCanvas() {
}

val interactingVisualizationFrame = lastFramesSelection[frameIndex]
var clickedLandmark: Landmark? = null
var clickedLandmarkLayerState: LayerState? = null
var clickedLandmarkUID = 0L

interactingVisualizationFrame.layers.forEach { layer ->
when (layer) {
Expand All @@ -215,7 +218,9 @@ class SceneCanvas : OpenGLCanvas() {
return@forEach
}

clickedLandmark = pointLandmarks[landmarkIndex]
val clickedLandmark = pointLandmarks[landmarkIndex]
clickedLandmarkLayerState = clickedLandmark.layerState
clickedLandmarkUID = clickedLandmark.uid
}
is Layer.LineLayer -> {
val lineLandmarks = layer.getLandmarks()
Expand All @@ -230,17 +235,33 @@ class SceneCanvas : OpenGLCanvas() {
return@forEach
}

clickedLandmark = lineLandmarks[landmarkIndex]
val clickedLandmark = lineLandmarks[landmarkIndex]
clickedLandmarkLayerState = clickedLandmark.layerState
clickedLandmarkUID = clickedLandmark.uid
}
is Layer.PlanesLayer -> {
val framePlaneUIDs = ImagePlanesParser.extractUIDs(layer.filePath.toString())
val clickedPixelIntegerColor = ImagePlanesParser.getPixelColor(
layer.filePath.toString(),
Point(frameInteractionPixel.x.toShort(), frameInteractionPixel.y.toShort())
)?.toLong() ?: return@forEach

// The integer pixel color is equal to UID of the corresponding plane.
if (!framePlaneUIDs.contains(clickedPixelIntegerColor))
return@forEach

clickedLandmarkLayerState = layer.layerState
clickedLandmarkUID = clickedPixelIntegerColor
}
is Layer.PlaneLayer -> {}
}
}

val selectedLandmark = clickedLandmark ?: return
if (selectedLandmark.layerState.selectedLandmarksUIDs.contains(selectedLandmark.uid)) {
selectedLandmark.layerState.deselectLandmark(selectedLandmark.uid)
val selectedLandmarkLayerState = clickedLandmarkLayerState ?: return

if (selectedLandmarkLayerState.selectedLandmarksUIDs.contains(clickedLandmarkUID)) {
selectedLandmarkLayerState.deselectLandmark(clickedLandmarkUID)
} else {
selectedLandmark.layerState.selectLandmark(selectedLandmark.uid)
selectedLandmarkLayerState.selectLandmark(clickedLandmarkUID)
}
}

Expand All @@ -267,7 +288,7 @@ class SceneCanvas : OpenGLCanvas() {
val addingRenderer = when (layer) {
is Layer.PointLayer -> PointsLayerRenderer(window) { sceneController?.scene }
is Layer.LineLayer -> LinesLayerRenderer(window) { sceneController?.scene }
is Layer.PlaneLayer -> PlanesLayerRenderer(window) { sceneController?.scene }
is Layer.PlanesLayer -> PlanesLayerRenderer(window) { sceneController?.scene }
}
addingRenderer.setNewSceneFrames(scene.frames, framesSize.toFloatVector())
engineScene?.addLandmarkRenderer(addingRenderer, layer)
Expand Down
18 changes: 18 additions & 0 deletions src/main/kotlin/solve/rendering/engine/core/input/MouseInputHandler.kt
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
package solve.rendering.engine.core.input

import javafx.scene.robot.Robot
import org.joml.Vector2i
import solve.rendering.engine.utils.minus
import solve.rendering.engine.utils.pointToSegmentDistance
import solve.rendering.engine.utils.toFloatVector
import solve.rendering.engine.utils.toVector2i
import solve.scene.model.Landmark
import java.awt.Color
import java.awt.MouseInfo

object MouseInputHandler {
private const val LineHandledDistanceMultiplier = 4f
private const val PointHandledDistanceMultiplier = 1.5f

private const val ColorSegmentMaxValue = 255

fun indexOfClickedLineLandmark(
lineLandmarks: List<Landmark.Line>,
clickedPixelCoordinate: Vector2i,
Expand Down Expand Up @@ -57,4 +62,17 @@ object MouseInputHandler {

return minKeypointDistanceIndex
}

fun getClickedPixelColor() : Color {
val pointerLocation = MouseInfo.getPointerInfo().location
val robot = Robot()

val javaFXColor = robot.getPixelColor(pointerLocation.x.toDouble(), pointerLocation.y.toDouble())

return Color(
(javaFXColor.red * ColorSegmentMaxValue).toInt(),
(javaFXColor.green * ColorSegmentMaxValue).toInt(),
(javaFXColor.blue * ColorSegmentMaxValue).toInt()
)
}
}
65 changes: 45 additions & 20 deletions src/main/kotlin/solve/rendering/engine/core/renderers/PlanesLayerRenderer.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,21 @@ import solve.rendering.engine.shader.ShaderAttributeType
import solve.rendering.engine.shader.ShaderProgram
import solve.rendering.engine.shader.ShaderType
import solve.rendering.engine.utils.plus
import solve.scene.model.Layer.PlaneLayer
import solve.scene.model.Layer.PlanesLayer
import solve.scene.model.LayerState
import solve.scene.model.Scene

class PlanesLayerRenderer(
window: Window,
getScene: () -> Scene?
) : LandmarkLayerRenderer(window, getScene) {
private var visiblePlaneLayers = emptyList<PlaneLayer>()
private var visiblePlaneLayersToTexturesMap = mutableMapOf<PlaneLayer, Texture2D>()
private var visiblePlanesLayers = emptyList<PlanesLayer>()
private var visiblePlanesLayersToTexturesMap = mutableMapOf<PlanesLayer, Texture2D>()
private var visiblePlaneLayersTextures = listOf<Texture2D>()

private val planeLayersState: LayerState?
get() = layers.filterIsInstance<PlanesLayer>().firstOrNull()?.layerState

override val maxBatchSize = 1000

private var needToInitializePlaneTextures = false
Expand Down Expand Up @@ -55,43 +59,61 @@ class PlanesLayerRenderer(
override fun uploadUniforms(shaderProgram: ShaderProgram) {
shaderProgram.uploadMatrix4f(ProjectionUniformName, window.calculateProjectionMatrix())
shaderProgram.uploadIntArray(TexturesUniformName, texturesIndices)

val layersState = planeLayersState
var interactingPlanesNumber = 0
if (layersState != null) {
val interactingPlanesUIDs = layersState.selectedLandmarksUIDs.union(
layersState.hoveredLandmarkUIDs
).distinct()
shaderProgram.uploadIntArray(
InteractingPlanesUIDsUniformName,
interactingPlanesUIDs.map { it.toInt() }.toIntArray()
)
val planesOpacity = interactingPlanesUIDs.map {
1f - layersState.getLandmarkHighlightingProgress(it)
}.toFloatArray()
shaderProgram.uploadFloatArray(InteractingPlanesOpacityUniformName, planesOpacity)
interactingPlanesNumber = interactingPlanesUIDs.count()
}
shaderProgram.uploadInt(InteractingPlanesNumberUniformName, interactingPlanesNumber)
}

override fun delete() {
visiblePlaneLayersToTexturesMap.values.forEach { it.delete() }
visiblePlaneLayersToTexturesMap.clear()
visiblePlanesLayersToTexturesMap.values.forEach { it.delete() }
visiblePlanesLayersToTexturesMap.clear()
super.delete()
}

override fun beforeRender() {
super.beforeRender()
visiblePlaneLayers = visibleLayers.filterIsInstance<PlaneLayer>()
visiblePlanesLayers = visibleLayers.filterIsInstance<PlanesLayer>()

val hiddenVisiblePlaneLayers = hiddenVisibleLayersInCurrentFrame.filterIsInstance<PlaneLayer>()
val newVisiblePlaneLayers = newVisibleLayersInCurrentFrame.filterIsInstance<PlaneLayer>()
val hiddenVisiblePlanesLayers = hiddenVisibleLayersInCurrentFrame.filterIsInstance<PlanesLayer>()
val newVisiblePlanesLayers = newVisibleLayersInCurrentFrame.filterIsInstance<PlanesLayer>()

hiddenVisiblePlaneLayers.forEach {
visiblePlaneLayersToTexturesMap[it]?.delete()
visiblePlaneLayersToTexturesMap.remove(it)
hiddenVisiblePlanesLayers.forEach {
visiblePlanesLayersToTexturesMap[it]?.delete()
visiblePlanesLayersToTexturesMap.remove(it)
}
newVisiblePlaneLayers.forEach {
visiblePlaneLayersToTexturesMap[it] = Texture2D(it.filePath.toString(), TextureFilterType.PixelPerfect)
newVisiblePlanesLayers.forEach {
visiblePlanesLayersToTexturesMap[it] = Texture2D(it.filePath.toString(), TextureFilterType.PixelPerfect)
}
visiblePlaneLayersTextures = visiblePlaneLayersToTexturesMap.keys.sortedBy {
visiblePlaneLayers.indexOf(it)
}.mapNotNull { visiblePlaneLayersToTexturesMap[it] }
visiblePlaneLayersTextures = visiblePlanesLayersToTexturesMap.keys.sortedBy {
visiblePlanesLayers.indexOf(it)
}.mapNotNull { visiblePlanesLayersToTexturesMap[it] }

val firstPlaneLayer = layers.filterIsInstance<PlaneLayer>().firstOrNull() ?: return
renderPriority = getScene()?.indexOf(firstPlaneLayer.settings) ?: return
val firstPlanesLayer = layers.filterIsInstance<PlanesLayer>().firstOrNull() ?: return
renderPriority = getScene()?.indexOf(firstPlanesLayer.settings) ?: return
}

override fun updateBatchesData() {
val firstLayer = visiblePlaneLayers.firstOrNull() ?: return
val firstLayer = visiblePlanesLayers.firstOrNull() ?: return
if (!firstLayer.settings.enabled) {
return
}

visiblePlaneLayers.forEachIndexed { visibleLayerIndex, _ ->
visiblePlanesLayers.forEachIndexed { visibleLayerIndex, _ ->
val selectionLayerIndex = visibleLayersSelectionIndices[visibleLayerIndex]
val planeLayerTexture = visiblePlaneLayersTextures[visibleLayerIndex]
val batch = getAvailableBatch(planeLayerTexture, 0)
Expand All @@ -108,6 +130,9 @@ class PlanesLayerRenderer(
}
companion object {
private const val TexturesUniformName = "uTextures"
private const val InteractingPlanesUIDsUniformName = "uInteractingPlanesUIDs"
private const val InteractingPlanesOpacityUniformName = "uInteractingPlanesOpacity"
private const val InteractingPlanesNumberUniformName = "uInteractingPlanesNumber"

private val texturesIndices = intArrayOf(0, 1, 2, 3, 4, 5, 6, 7)

Expand Down
9 changes: 6 additions & 3 deletions src/main/kotlin/solve/scene/SceneFacade.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package solve.scene

import javafx.beans.property.SimpleObjectProperty
import solve.parsers.factories.LineFactory
import solve.parsers.factories.PlaneFactory
import solve.parsers.factories.PointFactory
import solve.parsers.lines.CSVLinesParser
import solve.parsers.points.CSVPointsParser
Expand All @@ -11,6 +12,7 @@ import solve.project.model.ProjectFrame
import solve.project.model.ProjectLayer
import solve.scene.controller.SceneController
import solve.scene.model.ColorManager
import solve.scene.model.Landmark
import solve.scene.model.Layer
import solve.scene.model.LayerSettings
import solve.scene.model.LayerState
Expand Down Expand Up @@ -57,7 +59,7 @@ object SceneFacade {

val layerStates = layers.map { projectLayer -> LayerState(projectLayer.name) }
val visualizationFrames = frames.map { projectFrame -> projectFrame.toVisualizationFrame(layerStates) }
val scene = Scene(visualizationFrames, layersSettings)
val scene = Scene(visualizationFrames, layersSettings, layerStates)
controller.setScene(scene, keepSettings)
}

Expand Down Expand Up @@ -100,10 +102,11 @@ object SceneFacade {
}
}

LayerKind.Plane -> Layer.PlaneLayer(
LayerKind.Plane -> Layer.PlanesLayer(
file.projectLayer.name,
layerSettings as LayerSettings.PlaneLayerSettings,
file.path
file.path,
layerState
)
}
}
Expand Down
Loading

0 comments on commit 19d035e

Please sign in to comment.