feat: Add functions for populating in-memory triple store from sparql…

…-results/json

pom.xml

@@ -132,6 +132,12 @@
             <artifactId>kotlinx-serialization-json-jvm</artifactId>
             <version>1.6.3</version>
         </dependency>
+        <dependency>
+            <groupId>io.kotest</groupId>
+            <artifactId>kotest-property</artifactId>
+            <version>5.8.1</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

src/main/ExternalTripleStore.kt

@@ -1,9 +1,10 @@
 package com.github.statnett.loadflowservice
 
 import com.powsybl.triplestore.api.QueryCatalog
+import com.powsybl.triplestore.impl.rdf4j.TripleStoreRDF4J
 
 data class ParsedSparqlQuery(
-    val prefixes: Map<String, String>,
+    val prefixes: MutableMap<String, String>,
     val predicates: Set<String>,
 )
 
@@ -22,14 +23,14 @@ fun parseQuery(sparqlQueryResource: String): ParsedSparqlQuery {
 fun extractPredicates(catalog: QueryCatalog): Set<String> {
     val regexpPattern = Regex("([a-zA-Z0-9.]*:[a-zA-Z0-9.]+)")
     val matches = catalog.values.map { query -> regexpPattern.findAll(query) }.asSequence().flatten()
-    val extraPredicates = setOf("a")
+    val extraPredicates = setOf("rdf:type")
     return matches.map { match -> match.value }.toSet().union(extraPredicates)
 }
 
 fun extractPrefixes(catalog: QueryCatalog): Map<String, String> {
     val regex = Regex("PREFIX|prefix ([0-9a-zA-Z]+):.*(<[0-9a-zA-Z:/#.-]+>)")
     val matches = catalog.values.map { query -> regex.findAll(query) }.asSequence().flatten()
-    return matches.map { match -> Pair(match.groupValues[0], match.groupValues[1]) }.toMap()
+    return matches.map { match -> Pair(match.groupValues[1], match.groupValues[2]) }.toMap()
 }
 
 fun createExtractionQuery(query: ParsedSparqlQuery): String {
@@ -38,3 +39,34 @@ fun createExtractionQuery(query: ParsedSparqlQuery): String {
     val select = "SELECT ?graph ?s ?p ?o {\nVALUES ?p { $predicates }\nGRAPH ?graph {?s ?p ?o}}"
     return "$prefix\n$select"
 }
+
+fun populateInMemTripleStore(sparqlResult: SparqlResultJson): TripleStoreRDF4J {
+    val store = TripleStoreRDF4J()
+    store.update(insertQuery(sparqlResult.result.bindings))
+    return store
+}
+
+fun insertTriple(result: Map<String, SparqlItem>): String? {
+    val graph = result["graph"]
+    val subject = result["s"]
+    val predicate = result["p"]
+    val obj = result["o"]
+
+    if ((graph == null) || (subject == null) || (predicate == null) || (obj == null)) {
+        return null
+    }
+    var insertStatement = "GRAPH <${graph.value}> {<${subject.value}> <${predicate.value}> "
+    insertStatement +=
+        if (obj.type == SparqlTypes.URI) {
+            "<${obj.value}>"
+        } else {
+            "\"${obj.value}\""
+        }
+    insertStatement += "}"
+    return insertStatement
+}
+
+fun insertQuery(results: List<Map<String, SparqlItem>>): String {
+    val triples = results.mapNotNull { insertTriple(it) }.joinToString(" .\n")
+    return "INSERT DATA {$triples}"
+}

src/main/SparqlResultJson.kt

@@ -0,0 +1,34 @@
+package com.github.statnett.loadflowservice
+
+import kotlinx.serialization.Serializable
+
+class SparqlTypes {
+    companion object {
+        const val URI = "uri"
+        const val LITERAL = "literal"
+    }
+}
+
+@Serializable
+data class SparqlResultJson(
+    val head: SparqlVars,
+    val result: SparqlResult,
+    val link: List<String> = listOf(),
+)
+
+@Serializable
+data class SparqlVars(
+    val vars: List<String>,
+)
+
+@Serializable
+data class SparqlResult(
+    val bindings: List<Map<String, SparqlItem>>,
+)
+
+@Serializable
+data class SparqlItem(
+    val type: String,
+    val value: String,
+    val dataType: String? = null,
+)

src/test/TestExternalTripleStore.kt

@@ -1,15 +1,28 @@
 import com.github.statnett.loadflowservice.ParsedSparqlQuery
+import com.github.statnett.loadflowservice.SparqlItem
+import com.github.statnett.loadflowservice.SparqlTypes
 import com.github.statnett.loadflowservice.createExtractionQuery
+import com.github.statnett.loadflowservice.insertTriple
 import com.github.statnett.loadflowservice.parseQuery
+import com.github.statnett.loadflowservice.populateInMemTripleStore
+import com.powsybl.triplestore.impl.rdf4j.TripleStoreRDF4J
+import io.kotest.common.ExperimentalKotest
+import io.kotest.property.PropTestConfig
+import io.kotest.property.forAll
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.DynamicTest
 import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.TestFactory
+import testDataFactory.sparqlResultArb
 import kotlin.test.assertEquals
+import kotlin.test.assertNull
 import kotlin.test.assertTrue
 
 class TestExternalTripleStore {
     @Test
     fun `test predicate extraction`() {
         val queryContent = parseQuery("CIM16.sparql")
-        val expect = setOf("a", "cim:ConnectivityNode.TopologicalNode", "cim:IdentifiedObject.name")
+        val expect = setOf("rdf:type", "cim:ConnectivityNode.TopologicalNode", "cim:IdentifiedObject.name")
         assertTrue { queryContent.predicates.containsAll(expect) }
         assertTrue { queryContent.prefixes.isNotEmpty() }
     }
@@ -18,7 +31,7 @@ class TestExternalTripleStore {
     fun `test create extraction query`() {
         val parsedQuery =
             ParsedSparqlQuery(
-                mapOf("md" to "<http://md>", "cim" to "<http://cim>"),
+                mutableMapOf("md" to "<http://md>", "cim" to "<http://cim>"),
                 setOf("cim:a", "md:b"),
             )
 
@@ -28,4 +41,92 @@ class TestExternalTripleStore {
         )
         assertEquals(createExtractionQuery(parsedQuery), expectedQuery)
     }
+
+    @Test
+    fun `test insert query literal`() {
+        val result =
+            mapOf(
+                "graph" to SparqlItem(SparqlTypes.URI, "g"),
+                "s" to SparqlItem(SparqlTypes.URI, "s"),
+                "p" to SparqlItem(SparqlTypes.URI, "p"),
+                "o" to SparqlItem(SparqlTypes.LITERAL, "0.0"),
+            )
+        val query = insertTriple(result)
+        val expect = "GRAPH <g> {<s> <p> \"0.0\"}"
+        assertEquals(expect, query)
+    }
+
+    @Test
+    fun `test insert query object`() {
+        val result =
+            mapOf(
+                "graph" to SparqlItem(SparqlTypes.URI, "g"),
+                "s" to SparqlItem(SparqlTypes.URI, "s"),
+                "p" to SparqlItem(SparqlTypes.URI, "p"),
+                "o" to SparqlItem(SparqlTypes.URI, "o"),
+            )
+        val query = insertTriple(result)
+        val expect = "GRAPH <g> {<s> <p> <o>}"
+        assertEquals(expect, query)
+    }
+
+    @TestFactory
+    fun `test insert query null when any null`() =
+        listOf(
+            listOf("graph", "s", "p"),
+            listOf("graph", "s", "o"),
+            listOf("graph", "p", "o"),
+            listOf("s", "p", "o"),
+            listOf(),
+        ).map {
+            DynamicTest.dynamicTest("Test null") {
+                val result = it.associateWith { field -> SparqlItem(SparqlTypes.URI, field) }
+                assertNull(insertTriple(result))
+            }
+        }
+
+    @OptIn(ExperimentalKotest::class)
+    @Test
+    fun `test populate triple store`() =
+        runBlocking {
+            val result =
+                forAll(PropTestConfig(iterations = 100), sparqlResultArb) {
+                    val store = populateInMemTripleStore(it)
+                    val cnt = store.query("SELECT (COUNT(*) as ?cnt) WHERE {graph ?g {?s ?p ?o}}")[0]["cnt"]!!.toInt()
+                    cnt == it.result.bindings.size
+                }
+        }
+
+    @Test
+    fun `test extraction query works as expected`() {
+        val store = TripleStoreRDF4J()
+        val query =
+            """
+            PREFIX cim: <http://iec.ch/TC57/2013/CIM-schema-cim16#>
+            
+            INSERT DATA {
+                GRAPH <http://g> {
+                    _:b0 cim:ConnectivityNode.TopologicalNode _:b1 .
+                    _:b0 cim:IdentifiedObject.name "Connectivity node" .
+                    _:b1 cim:IdentifiedObject.name "Topological node" .
+                    _:b1 cim:SomeRandomNonExistentPredicate.name "random predicate"
+                }
+            }
+            """.trimIndent()
+        store.update(query)
+
+        // Confirm 4 triples were inserted
+        val cnt = store.query("SELECT (count(*) as ?cnt) where {graph ?g {?s ?p ?o}}")[0]["cnt"]!!.toInt()
+        assertEquals(4, cnt)
+
+        val parsedQuery = parseQuery("CIM16.sparql")
+
+        // Update with a few prefixes not found in the CIM16.sparql file
+        parsedQuery.prefixes["cim"] = "<http://iec.ch/TC57/2013/CIM-schema-cim16#>"
+        parsedQuery.prefixes["entsoe"] = "<http://entsoe>"
+        val result = store.query(createExtractionQuery(parsedQuery))
+
+        // Expect three records to be extracted
+        assertEquals(3, result.size)
+    }
 }

src/test/testDataFactory/SparqlResultFactory.kt

@@ -0,0 +1,86 @@
+package testDataFactory
+
+import com.github.statnett.loadflowservice.SparqlItem
+import com.github.statnett.loadflowservice.SparqlResult
+import com.github.statnett.loadflowservice.SparqlResultJson
+import com.github.statnett.loadflowservice.SparqlTypes
+import com.github.statnett.loadflowservice.SparqlVars
+import io.kotest.property.Arb
+import io.kotest.property.arbitrary.arbitrary
+import io.kotest.property.arbitrary.choice
+import io.kotest.property.arbitrary.constant
+import io.kotest.property.arbitrary.distinct
+import io.kotest.property.arbitrary.float
+import io.kotest.property.arbitrary.list
+import io.kotest.property.arbitrary.orNull
+import io.kotest.property.arbitrary.string
+import io.kotest.property.arbitrary.stringPattern
+import io.kotest.property.arbitrary.uuid
+
+val sparqlResultArb =
+    arbitrary {
+        val vars = listOf("graph", "s", "p", "o")
+        val items =
+            Arb.list(
+                arbitrary {
+                    mapOf(
+                        "graph" to sparqlGraphArb.bind(),
+                        "s" to sparqlUuidArb.bind(),
+                        "p" to sparqlPredicateArb.bind(),
+                        "o" to sparqlObjectArb.bind(),
+                    )
+                },
+            )
+        SparqlResultJson(
+            SparqlVars(vars),
+            result = SparqlResult(items.bind().distinct()),
+        )
+    }
+
+val sparqlItemArb =
+    arbitrary {
+        val type = Arb.string().bind()
+        val value = Arb.string().bind()
+        val dataType = Arb.string().orNull().bind()
+        SparqlItem(type, value, dataType)
+    }
+
+val sparqlUuidArb =
+    arbitrary {
+        SparqlItem(SparqlTypes.URI, "urn:uuid:${Arb.uuid().bind()}")
+    }
+
+val sparqlGraphArb =
+    arbitrary {
+        val graphs =
+            listOf(
+                Arb.constant("http://sv"),
+                Arb.constant("http://ssh"),
+                Arb.constant("http://tp"),
+                Arb.constant("http://eq"),
+            )
+        SparqlItem(
+            SparqlTypes.URI,
+            Arb.choice(graphs).bind(),
+        )
+    }
+val sparqlPredicateArb =
+    arbitrary {
+        val name = Arb.stringPattern("[a-z]+").bind()
+        SparqlItem(
+            "uri",
+            "http://predicate.com#$name",
+        )
+    }
+
+val sparqlObjectArb =
+    arbitrary {
+        val options =
+            listOf(
+                Arb.constant(Pair(SparqlTypes.URI, "urn:uuid:${Arb.uuid().bind()}")),
+                Arb.constant(Pair(SparqlTypes.LITERAL, Arb.stringPattern("[a-zA-Z0-9-_]+").bind())),
+                Arb.constant(Pair(SparqlTypes.LITERAL, Arb.float().bind().toString())),
+            )
+        val chosen = Arb.choice(options).bind()
+        SparqlItem(chosen.first, chosen.second)
+    }