From 02627ef42410eefde72eae66881f3049f4f9a545 Mon Sep 17 00:00:00 2001 From: David Gautier Date: Wed, 18 Oct 2023 10:49:33 +0200 Subject: [PATCH 1/9] fix : log error about source file when getting InvalidPathException --- .../convert/confluence/ReferenceProvider.kt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt index 7a372ce0..7cbc43fa 100644 --- a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt +++ b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt @@ -1,6 +1,8 @@ package com.github.zeldigas.text2confl.convert.confluence import com.github.zeldigas.text2confl.convert.PageHeader +import io.github.oshai.kotlinlogging.KotlinLogging +import java.nio.file.InvalidPathException import java.nio.file.Path import kotlin.io.path.relativeTo @@ -40,12 +42,14 @@ class ReferenceProviderImpl(private val basePath: Path, documents: Map path.relativeTo(basePath).normalize() to header }.toMap() override fun resolveReference(source: Path, refTo: String): Reference? { + if (URI_DETECTOR.matches(refTo)) return null if (refTo.startsWith("#")) return Anchor(refTo.substring(1)) @@ -53,10 +57,15 @@ class ReferenceProviderImpl(private val basePath: Path, documents: Map Date: Wed, 18 Oct 2023 11:46:39 +0200 Subject: [PATCH 2/9] fix : error when loading templates from windows file system --- .../text2confl/convert/asciidoc/AsciidocParser.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/asciidoc/AsciidocParser.kt b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/asciidoc/AsciidocParser.kt index ec38be72..13388cc4 100644 --- a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/asciidoc/AsciidocParser.kt +++ b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/asciidoc/AsciidocParser.kt @@ -5,6 +5,7 @@ import com.vladsch.flexmark.util.sequence.Escaping.unescapeHtml import org.asciidoctor.* import org.asciidoctor.ast.Document import java.nio.file.Path +import java.nio.file.Paths import kotlin.io.path.Path import kotlin.io.path.div @@ -14,7 +15,7 @@ class AsciidocParser( ) { companion object { - private val TEMPLATES_LOCATION = "/com/github/zeldigas/text2confl/asciidoc" + private val TEMPLATES_LOCATION = "com/github/zeldigas/text2confl/asciidoc" } private val ADOC: Asciidoctor by lazy { @@ -27,9 +28,10 @@ class AsciidocParser( } private val templatesLocation: Path by lazy { - val templateResources = AsciidocParser::class.java.getResource(TEMPLATES_LOCATION)!!.toURI() + val templateResources = AsciidocParser::class.java.classLoader.getResource(TEMPLATES_LOCATION)!!.toURI() if (templateResources.scheme == "file") { - Path(templateResources.path) + val mainPath: String = Paths.get(templateResources).toString() + Path(mainPath) } else { val dest = config.workdir / "templates" From b2a0fe331fabed8b61caff820579a0f2a8821d6c Mon Sep 17 00:00:00 2001 From: David Date: Wed, 18 Oct 2023 14:04:50 +0200 Subject: [PATCH 3/9] fix : Force Locale to English to avoid error message translation --- .../com/github/zeldigas/text2confl/convert/PageContentTest.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/convert/src/test/kotlin/com/github/zeldigas/text2confl/convert/PageContentTest.kt b/convert/src/test/kotlin/com/github/zeldigas/text2confl/convert/PageContentTest.kt index 0fa84fcc..6a5c7a55 100644 --- a/convert/src/test/kotlin/com/github/zeldigas/text2confl/convert/PageContentTest.kt +++ b/convert/src/test/kotlin/com/github/zeldigas/text2confl/convert/PageContentTest.kt @@ -11,6 +11,7 @@ import org.junit.jupiter.api.io.TempDir import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.CsvSource import java.nio.file.Path +import java.util.* import kotlin.io.path.Path import kotlin.io.path.writeBytes @@ -70,6 +71,7 @@ class PageContentTest { @Test internal fun `Invalid result for unbalanced xml`() { + Locale.setDefault(Locale.ENGLISH); val sampleXml = """

hello world

From ef2de2e51ab577b49fa5dc29e26bf6709a8768df Mon Sep 17 00:00:00 2001 From: David Date: Wed, 18 Oct 2023 14:05:52 +0200 Subject: [PATCH 4/9] refactor : use isNotEmpty instead of !isEmpty --- .../convert/markdown/ConfluenceStorageFormatConverters.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/markdown/ConfluenceStorageFormatConverters.kt b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/markdown/ConfluenceStorageFormatConverters.kt index 6845027f..f2332005 100644 --- a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/markdown/ConfluenceStorageFormatConverters.kt +++ b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/markdown/ConfluenceStorageFormatConverters.kt @@ -197,7 +197,7 @@ class ConfluenceNodeRenderer(options: DataHolder) : PhasedNodeRenderer, Attribut val resolvedLink = context.resolveLink(LinkType.IMAGE, node.url.unescape(), null, null) var linkUrl: String = resolvedLink.url - if (!node.urlContent.isEmpty) { + if (node.urlContent.isNotEmpty) { // reverse URL encoding of =, & val content = Escaping.percentEncodeUrl(node.urlContent).replace("+", "%2B").replace("%3D", "=") .replace("%26", "&") From 9e0bf7e9bd12f990cd9ec9c4b68c881cf47f17a0 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 18 Oct 2023 14:06:31 +0200 Subject: [PATCH 5/9] fix : URI_DETECTOR regex --- .../text2confl/core/upload/ContentUploader.kt | 106 +++++++++++++++--- 1 file changed, 93 insertions(+), 13 deletions(-) diff --git a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt index 800a05b6..211d65b7 100644 --- a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt +++ b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt @@ -1,16 +1,16 @@ package com.github.zeldigas.text2confl.core.upload import com.github.zeldigas.confclient.ConfluenceClient +import com.github.zeldigas.confclient.PageNotCreatedException +import com.github.zeldigas.confclient.PageNotUpdatedException import com.github.zeldigas.confclient.model.ConfluencePage import com.github.zeldigas.text2confl.convert.EditorVersion import com.github.zeldigas.text2confl.convert.Page import com.github.zeldigas.text2confl.convert.PageHeader import com.github.zeldigas.text2confl.core.config.Cleanup import io.github.oshai.kotlinlogging.KotlinLogging -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.launch +import io.ktor.client.network.sockets.* +import kotlinx.coroutines.* class ContentUploader( @@ -46,29 +46,109 @@ class ContentUploader( private val logger = KotlinLogging.logger {} } + fun run(pages: List, space: String, parentPageId: String) = runBlocking { + try { + withContext(Dispatchers.Default) { + uploadPages(pages , space, parentPageId) + } + } catch (ex: Exception) { + tryHandleException(ex) + } + } + + fun runBlocking(pages: List, space: String, parentPageId: String) = runBlocking { + try { + withContext(Dispatchers.Default) { + uploadPagesBlocking(pages , space, parentPageId) + } + } catch (ex: Exception) { + tryHandleException(ex) + } + } + + private fun tryHandleException(ex: Exception) { + logger.error { "Exception caught in tryHandleException" } + when (ex) { + is PageNotCreatedException -> { + logger.error { ex.message } + } + is PageNotUpdatedException -> { + logger.error { ex.message } + } + else -> { + throw ex + } + } + } + + suspend fun uploadPagesBlocking(pages: List, space: String, parentPageId: String) { + val uploadedPages = uploadPagesRecursiveBlocking(pages, space, parentPageId) + val uploadedPagesByParent = buildOrphanedRemovalRegistry(uploadedPages) + deleteOrphans(uploadedPagesByParent) + } + suspend fun uploadPages(pages: List, space: String, parentPageId: String) { val uploadedPages = uploadPagesRecursive(pages, space, parentPageId) val uploadedPagesByParent = buildOrphanedRemovalRegistry(uploadedPages) deleteOrphans(uploadedPagesByParent) } - private suspend fun uploadPagesRecursive( + private suspend fun uploadPagesRecursiveBlocking( pages: List, space: String, parentPageId: String ): List { - return coroutineScope { - pages.map { page -> - async { - val result = uploadPage(page, space, parentPageId) + + val uploadedPages = ArrayList() + pages.forEach { + try { + val result = uploadPage(it, space, parentPageId) + uploadedPages.addAll( buildList { add(result) - if (page.children.isNotEmpty()) { - addAll(uploadPagesRecursive(page.children, space, result.page.id)) + if (it.children.isNotEmpty()) { + addAll(uploadPagesRecursiveBlocking(it.children, space, result.page.id)) + } + }) + } catch (e: PageNotCreatedException){ + logger.error { e.message } + return emptyList() + } catch (e: PageNotUpdatedException){ + logger.error { e.message } + return emptyList() + } catch (e: ConnectTimeoutException){ + logger.error { e.message } + return emptyList() + } + } + return uploadedPages + } + + private suspend fun uploadPagesRecursive( + pages: List, + space: String, + parentPageId: String + ): List { + return try { + supervisorScope { + pages.map { page -> + async { + val result = uploadPage(page, space, parentPageId) + buildList { + add(result) + if (page.children.isNotEmpty()) { + addAll(uploadPagesRecursive(page.children, space, result.page.id)) + } } } - } - }.awaitAll().flatten() + }.awaitAll().flatten() + } + } catch (e: PageNotCreatedException){ + logger.error { e.message } + return emptyList() + } catch (e: PageNotUpdatedException){ + logger.error { e.message } + return emptyList() } } From cf1fe23b2ea90af2183afc64d3ad2b2f1b45853f Mon Sep 17 00:00:00 2001 From: David Date: Wed, 18 Oct 2023 14:13:01 +0200 Subject: [PATCH 6/9] Revert "fix : URI_DETECTOR regex" This reverts commit 9e0bf7e9bd12f990cd9ec9c4b68c881cf47f17a0. --- .../text2confl/core/upload/ContentUploader.kt | 106 +++--------------- 1 file changed, 13 insertions(+), 93 deletions(-) diff --git a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt index 211d65b7..800a05b6 100644 --- a/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt +++ b/core/src/main/kotlin/com/github/zeldigas/text2confl/core/upload/ContentUploader.kt @@ -1,16 +1,16 @@ package com.github.zeldigas.text2confl.core.upload import com.github.zeldigas.confclient.ConfluenceClient -import com.github.zeldigas.confclient.PageNotCreatedException -import com.github.zeldigas.confclient.PageNotUpdatedException import com.github.zeldigas.confclient.model.ConfluencePage import com.github.zeldigas.text2confl.convert.EditorVersion import com.github.zeldigas.text2confl.convert.Page import com.github.zeldigas.text2confl.convert.PageHeader import com.github.zeldigas.text2confl.core.config.Cleanup import io.github.oshai.kotlinlogging.KotlinLogging -import io.ktor.client.network.sockets.* -import kotlinx.coroutines.* +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch class ContentUploader( @@ -46,109 +46,29 @@ class ContentUploader( private val logger = KotlinLogging.logger {} } - fun run(pages: List, space: String, parentPageId: String) = runBlocking { - try { - withContext(Dispatchers.Default) { - uploadPages(pages , space, parentPageId) - } - } catch (ex: Exception) { - tryHandleException(ex) - } - } - - fun runBlocking(pages: List, space: String, parentPageId: String) = runBlocking { - try { - withContext(Dispatchers.Default) { - uploadPagesBlocking(pages , space, parentPageId) - } - } catch (ex: Exception) { - tryHandleException(ex) - } - } - - private fun tryHandleException(ex: Exception) { - logger.error { "Exception caught in tryHandleException" } - when (ex) { - is PageNotCreatedException -> { - logger.error { ex.message } - } - is PageNotUpdatedException -> { - logger.error { ex.message } - } - else -> { - throw ex - } - } - } - - suspend fun uploadPagesBlocking(pages: List, space: String, parentPageId: String) { - val uploadedPages = uploadPagesRecursiveBlocking(pages, space, parentPageId) - val uploadedPagesByParent = buildOrphanedRemovalRegistry(uploadedPages) - deleteOrphans(uploadedPagesByParent) - } - suspend fun uploadPages(pages: List, space: String, parentPageId: String) { val uploadedPages = uploadPagesRecursive(pages, space, parentPageId) val uploadedPagesByParent = buildOrphanedRemovalRegistry(uploadedPages) deleteOrphans(uploadedPagesByParent) } - private suspend fun uploadPagesRecursiveBlocking( + private suspend fun uploadPagesRecursive( pages: List, space: String, parentPageId: String ): List { - - val uploadedPages = ArrayList() - pages.forEach { - try { - val result = uploadPage(it, space, parentPageId) - uploadedPages.addAll( + return coroutineScope { + pages.map { page -> + async { + val result = uploadPage(page, space, parentPageId) buildList { add(result) - if (it.children.isNotEmpty()) { - addAll(uploadPagesRecursiveBlocking(it.children, space, result.page.id)) - } - }) - } catch (e: PageNotCreatedException){ - logger.error { e.message } - return emptyList() - } catch (e: PageNotUpdatedException){ - logger.error { e.message } - return emptyList() - } catch (e: ConnectTimeoutException){ - logger.error { e.message } - return emptyList() - } - } - return uploadedPages - } - - private suspend fun uploadPagesRecursive( - pages: List, - space: String, - parentPageId: String - ): List { - return try { - supervisorScope { - pages.map { page -> - async { - val result = uploadPage(page, space, parentPageId) - buildList { - add(result) - if (page.children.isNotEmpty()) { - addAll(uploadPagesRecursive(page.children, space, result.page.id)) - } + if (page.children.isNotEmpty()) { + addAll(uploadPagesRecursive(page.children, space, result.page.id)) } } - }.awaitAll().flatten() - } - } catch (e: PageNotCreatedException){ - logger.error { e.message } - return emptyList() - } catch (e: PageNotUpdatedException){ - logger.error { e.message } - return emptyList() + } + }.awaitAll().flatten() } } From 56e4d0c45cc43566c31a187fa8a177ec0ce0da0e Mon Sep 17 00:00:00 2001 From: David Date: Wed, 18 Oct 2023 14:13:49 +0200 Subject: [PATCH 7/9] fix : URI_DETECTOR regex --- .../text2confl/convert/confluence/ReferenceProvider.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt index 7cbc43fa..1bb5e314 100644 --- a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt +++ b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt @@ -41,7 +41,7 @@ class ReferenceProviderImpl(private val basePath: Path, documents: Map Date: Wed, 18 Oct 2023 15:15:22 +0200 Subject: [PATCH 8/9] fix : URI_DETECTOR regex --- .../convert/confluence/ReferenceProvider.kt | 16 ++++++++++--- .../confluence/ReferenceProviderImplTest.kt | 23 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt index 1bb5e314..80e92b87 100644 --- a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt +++ b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt @@ -2,8 +2,11 @@ package com.github.zeldigas.text2confl.convert.confluence import com.github.zeldigas.text2confl.convert.PageHeader import io.github.oshai.kotlinlogging.KotlinLogging +import java.net.URL import java.nio.file.InvalidPathException import java.nio.file.Path +import java.util.regex.Matcher +import java.util.regex.Pattern import kotlin.io.path.relativeTo interface ReferenceProvider { @@ -41,16 +44,23 @@ class ReferenceProviderImpl(private val basePath: Path, documents: Map path.relativeTo(basePath).normalize() to header }.toMap() override fun resolveReference(source: Path, refTo: String): Reference? { - if (URI_DETECTOR.matches(refTo)) return null + if (refTo.startsWith(MAILTO_DETECTOR)) return null + if (isValid(refTo)) return null if (refTo.startsWith("#")) return Anchor(refTo.substring(1)) val parts = refTo.split("#", limit = 2) @@ -62,7 +72,7 @@ class ReferenceProviderImpl(private val basePath: Path, documents: Map Date: Wed, 18 Oct 2023 20:26:52 +0200 Subject: [PATCH 9/9] fix : URI_DETECTOR regex, added custom detector localhost and mailto --- .../convert/confluence/ReferenceProvider.kt | 5 +++- .../confluence/ReferenceProviderImplTest.kt | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt index 80e92b87..9b34908e 100644 --- a/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt +++ b/convert/src/main/kotlin/com/github/zeldigas/text2confl/convert/confluence/ReferenceProvider.kt @@ -44,8 +44,10 @@ class ReferenceProviderImpl(private val basePath: Path, documents: Map