From f0ff987b7948fa68ee4358334988d6ee24c447e5 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Fri, 15 Aug 2025 17:53:51 +0200 Subject: [PATCH 01/28] ftp initial --- server/build.gradle | 2 + .../gokb/PackageSourceUpdateService.groovy | 58 ++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/server/build.gradle b/server/build.gradle index b02e305a2..60d34adbc 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -115,6 +115,8 @@ dependencies { implementation 'org.apache.tika:tika-core:2.9.4' implementation 'dk.glasius:external-config:4.0.0' + implementation 'commons-net:commons-net:3.12.0' + implementation "io.micronaut:micronaut-http-client" // implementation 'io.micronaut.xml:micronaut-jackson-xml:4.4.0' implementation 'com.github.albfernandez:juniversalchardet:2.5.0' diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index b7dc638ae..c352775ba 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -4,10 +4,13 @@ import com.k_int.ConcurrencyManagerService.Job import grails.converters.JSON import groovy.util.logging.Slf4j +import org.apache.commons.net.ftp.FTPClient +import org.apache.commons.net.ftp.FTPClientConfig import java.net.http.* import java.net.http.HttpResponse.BodyHandlers import java.net.http.HttpRequest.BodyPublishers +import java.nio.charset.StandardCharsets import java.security.MessageDigest import java.time.Duration import java.time.LocalDate @@ -16,6 +19,7 @@ import java.util.regex.Pattern import org.gokb.cred.* import org.mozilla.universalchardet.UniversalDetector +import org.apache.commons.net.* @Slf4j class PackageSourceUpdateService { @@ -323,7 +327,59 @@ class PackageSourceUpdateService { return result } } - // else if (src_url.getProtocol() in ['ftp', 'sftp']) { + else if (src_url.getProtocol() in ['ftp', 'sftp']) { + FTPClient ftp = new FTPClient() + FTPClientConfig config = new FTPClientConfig() + + def hostname = "ftp.epnet.com" + def username = "jake" + def password = "gijaq3eV" + def directory = "/kbart" + def filename = "8gh-kbart2.txt" + + try { + ftp.connect(hostname) + ftp.enterLocalPassiveMode() + log.debug("1111 : " + ftp.getReplyString() ) + def loggedIn = ftp.login(username, password) + log.debug("+++ EINGELOGGT... : " + loggedIn ) + + + if (ftp.isConnected()) { + //log.debug(" #### Directories: " + ftp.listDirectories()) + + ftp.changeWorkingDirectory(directory) + log.debug("2222 : " + ftp.getReplyString() ) + log.debug("##### Files: " + ftp.listFiles()) + log.debug("3333 : " + ftp.getReplyString() ) + + InputStream is = ftp.retrieveFileStream(filename) + + ByteArrayOutputStream tmp_result = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + for (int length; (length = is.read(buffer)) != -1; ) { + tmp_result.write(buffer, 0, length); + } + + log.debug("###########################################################################################") + log.debug(tmp_result.toString(StandardCharsets.UTF_8.name())) + log.debug("###########################################################################################") + + + //def file_result = TSVIngestionService.analyseFile(is) + + + ftp.logout() + ftp.disconnect() + } + + } catch (Exception e) { + e.printStackTrace() + } + + return + + } else { result.result = 'ERROR' result.messageCode = 'kbart.errors.url.protocol' From 39977e605fbec84be6047cfd3911014d491da1a9 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Tue, 19 Aug 2025 19:10:52 +0200 Subject: [PATCH 02/28] add GSP for creating endpoints in legacy UI --- server/grails-app/views/apptemplates/_webEndpoint.gsp | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 server/grails-app/views/apptemplates/_webEndpoint.gsp diff --git a/server/grails-app/views/apptemplates/_webEndpoint.gsp b/server/grails-app/views/apptemplates/_webEndpoint.gsp new file mode 100644 index 000000000..e69de29bb From fb8044e099ea33ef752741307668819840545cce Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Wed, 20 Aug 2025 16:51:13 +0200 Subject: [PATCH 03/28] apply create and search functionality for WebHookEndpoint to old UI --- server/grails-app/conf/application.groovy | 24 +++++++++++++++++++ .../org/gokb/cred/WebHookEndpoint.groovy | 12 +++++++--- .../org/gokb/DisplayTemplateService.groovy | 1 + .../views/apptemplates/_webEndpoint.gsp | 0 .../views/apptemplates/_web_hook_endpoint.gsp | 16 +++++++++++++ 5 files changed, 50 insertions(+), 3 deletions(-) delete mode 100644 server/grails-app/views/apptemplates/_webEndpoint.gsp create mode 100644 server/grails-app/views/apptemplates/_web_hook_endpoint.gsp diff --git a/server/grails-app/conf/application.groovy b/server/grails-app/conf/application.groovy index e99123d00..1e001188c 100644 --- a/server/grails-app/conf/application.groovy +++ b/server/grails-app/conf/application.groovy @@ -1247,6 +1247,30 @@ globalSearchTemplates = [ ] ] ], + 'WebHookEndpoints':[ + baseclass:'org.gokb.cred.WebHookEndpoint', + title:'Web Endpoints', + group:'Secondary', + defaultSort:'id', + defaultOrder:'desc', + qbeConfig:[ + qbeForm:[ + [ + prompt:'Name', + qparam:'qp_name', + placeholder:'Name of Web Endpoint', + contextTree:['ctxtp':'qry', 'comparator' : 'ilike', 'prop':'name','wildcard':'R'] + ], + ], + qbeGlobals:[ + ], + qbeResults:[ + [heading:'Name', property:'name', link:[controller:'resource', action:'show', id:'x.r.uuid'] ], + [heading:'URL', property:'url'], + [heading:'Methode', property:'supplyMethod'], + ] + ] + ], ] diff --git a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy index e9aa3b548..bd33d75ac 100644 --- a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy +++ b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy @@ -7,10 +7,13 @@ import groovy.util.logging.* class WebHookEndpoint { String name String url - Long authmethod - String principal - String credentials + Long authmethod //legacy + RefdataValue supplyMethod + String principal //legacy + String credentials //legacy User owner + String ba_username + String ba_password static mapping = { url column:'ep_url' @@ -25,6 +28,9 @@ class WebHookEndpoint { authmethod(nullable:true, blank:true) principal(nullable:true, blank:true) credentials(nullable:true, blank:true) + supplyMethod(nullable:true, blank:true) + ba_username(nullable:true, blank:true) + ba_password(nullable:true, blank:true) } static def refdataFind(params) { diff --git a/server/grails-app/services/org/gokb/DisplayTemplateService.groovy b/server/grails-app/services/org/gokb/DisplayTemplateService.groovy index d505598c0..f13fd4b34 100644 --- a/server/grails-app/services/org/gokb/DisplayTemplateService.groovy +++ b/server/grails-app/services/org/gokb/DisplayTemplateService.groovy @@ -41,6 +41,7 @@ public class DisplayTemplateService { globalDisplayTemplates.put('org.gokb.cred.Folder',[ type:'staticgsp', rendername:'folder' ]); globalDisplayTemplates.put('org.gokb.cred.Work',[ type:'staticgsp', rendername:'work' ]); globalDisplayTemplates.put('org.gokb.cred.BulkImportListConfig',[ type:'staticgsp', rendername:'bulkconfig', noCreate:true ]); + globalDisplayTemplates.put('org.gokb.cred.WebHookEndpoint',[ type:'staticgsp', rendername:'web_hook_endpoint' ]); } public Map getTemplateInfo(String type) { diff --git a/server/grails-app/views/apptemplates/_webEndpoint.gsp b/server/grails-app/views/apptemplates/_webEndpoint.gsp deleted file mode 100644 index e69de29bb..000000000 diff --git a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp new file mode 100644 index 000000000..0011f8ee9 --- /dev/null +++ b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp @@ -0,0 +1,16 @@ +
+
Name
+
+ +
Base URL
+
+ +
Methode
+
+ +
Benutzername
+
+ +
Passwort
+
+
\ No newline at end of file From 2bb2e5bdb5826899a66eaae22d77cc3dfb5e33eb Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Fri, 22 Aug 2025 17:40:42 +0200 Subject: [PATCH 04/28] controller --- .../controllers/org/gokb/UrlMappings.groovy | 2 + .../gokb/rest/WebEndpointController.groovy | 58 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy diff --git a/server/grails-app/controllers/org/gokb/UrlMappings.groovy b/server/grails-app/controllers/org/gokb/UrlMappings.groovy index 22e082692..67a4d5f8c 100644 --- a/server/grails-app/controllers/org/gokb/UrlMappings.groovy +++ b/server/grails-app/controllers/org/gokb/UrlMappings.groovy @@ -188,6 +188,8 @@ class UrlMappings { get "/jobs/$id" (controller: 'jobs', namespace: 'rest', action: 'show') patch "/jobs/$id/cancel" (controller: 'jobs', namespace: 'rest', action: 'cancel') delete "/jobs/$id" (controller: 'jobs', namespace: 'rest', action: 'delete') + + get "/web-endpoint"(controller: 'webEndpoint', namespace: 'rest', action: 'show') } "/$controller/$action?/$id?" { constraints { diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy new file mode 100644 index 000000000..f7b85a2dd --- /dev/null +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -0,0 +1,58 @@ +package org.gokb.rest + +import grails.converters.JSON +import grails.plugin.springsecurity.annotation.Secured +import org.gokb.cred.User +import org.gokb.cred.WebHookEndpoint + +import java.time.Duration +import java.time.LocalDateTime + +class WebEndpointController { + + def componentLookupService + def springSecurityService + + @Secured(['IS_AUTHENTICATED_ANONYMOUSLY']) + def index() { + def result = [:] + def base = grailsApplication.config.getProperty('grails.serverURL') + "/rest" + User user = null + + if (springSecurityService.isLoggedIn()) { + user = User.get(springSecurityService.principal?.id) + } + def start_db = LocalDateTime.now() + + + params['_embed'] = params['_embed'] ?: 'identifiedComponents' + + result = componentLookupService.restLookup(user, WebHookEndpoint, params) + //log.debug("DB duration: ${Duration.between(start_db, LocalDateTime.now()).toMillis();}") + log.debug("#### " + result) + + render result as JSON + } + + def show() { + def result = [:] + def base = grailsApplication.config.getProperty('grails.serverURL') + "/rest" + User user = null + + if (springSecurityService.isLoggedIn()) { + user = User.get(springSecurityService.principal?.id) + } + def start_db = LocalDateTime.now() + + + params['_embed'] = params['_embed'] ?: 'identifiedComponents' + + result = componentLookupService.restLookup(user, WebHookEndpoint, params) + //log.debug("DB duration: ${Duration.between(start_db, LocalDateTime.now()).toMillis();}") + log.debug("#### " + result) + + render result as JSON + } + + +} From 3c4d8de65b9d5a0e093b86143039872481594a9c Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Wed, 27 Aug 2025 17:46:48 +0200 Subject: [PATCH 05/28] add RDV for source data transfer methods --- server/grails-app/controllers/org/gokb/UrlMappings.groovy | 2 +- .../controllers/org/gokb/rest/WebEndpointController.groovy | 2 ++ server/grails-app/init/org/gokb/BootStrap.groovy | 3 +++ server/grails-app/views/apptemplates/_web_hook_endpoint.gsp | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/UrlMappings.groovy b/server/grails-app/controllers/org/gokb/UrlMappings.groovy index 67a4d5f8c..ed8fbe2e5 100644 --- a/server/grails-app/controllers/org/gokb/UrlMappings.groovy +++ b/server/grails-app/controllers/org/gokb/UrlMappings.groovy @@ -189,7 +189,7 @@ class UrlMappings { patch "/jobs/$id/cancel" (controller: 'jobs', namespace: 'rest', action: 'cancel') delete "/jobs/$id" (controller: 'jobs', namespace: 'rest', action: 'delete') - get "/web-endpoint"(controller: 'webEndpoint', namespace: 'rest', action: 'show') + get "/web-endpoint"(controller: 'webEndpoint', namespace: 'rest', action: 'index') } "/$controller/$action?/$id?" { constraints { diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index f7b85a2dd..00c5e305a 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -10,6 +10,8 @@ import java.time.LocalDateTime class WebEndpointController { + static namespace = 'rest' + def componentLookupService def springSecurityService diff --git a/server/grails-app/init/org/gokb/BootStrap.groovy b/server/grails-app/init/org/gokb/BootStrap.groovy index 8ae46657e..4a4090ea7 100644 --- a/server/grails-app/init/org/gokb/BootStrap.groovy +++ b/server/grails-app/init/org/gokb/BootStrap.groovy @@ -1115,6 +1115,9 @@ class BootStrap { RefdataCategory.lookupOrCreate('Source.Frequency', 'Quarterly', '090').save(flush: true, failOnError: true) RefdataCategory.lookupOrCreate('Source.Frequency', 'Yearly', '365').save(flush: true, failOnError: true) + RefdataCategory.lookupOrCreate('Source.TransferMethod', 'HTTP').save(flush: true, failOnError: true) + RefdataCategory.lookupOrCreate('Source.TransferMethod', 'FTP').save(flush: true, failOnError: true) + RefdataCategory.lookupOrCreate('BulkImportListConfig.Frequency', 'Daily', '001').save(flush: true, failOnError: true) RefdataCategory.lookupOrCreate('BulkImportListConfig.Frequency', 'Weekly', '007').save(flush: true, failOnError: true) RefdataCategory.lookupOrCreate('BulkImportListConfig.Frequency', 'Monthly', '030').save(flush: true, failOnError: true) diff --git a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp index 0011f8ee9..cf02ef4be 100644 --- a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp +++ b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp @@ -6,7 +6,7 @@
Methode
-
+
Benutzername
From 2890c2528ed620c2cd4adb20dd324fdc3781591b Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Fri, 29 Aug 2025 15:50:49 +0200 Subject: [PATCH 06/28] changes --- .../gokb/rest/WebEndpointController.groovy | 21 ++++++++++++++++++- .../domain/org/gokb/cred/Source.groovy | 6 ++++++ .../org/gokb/cred/WebHookEndpoint.groovy | 4 +++- .../views/apptemplates/_web_hook_endpoint.gsp | 4 ++-- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index 00c5e305a..a81eeab20 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -31,7 +31,26 @@ class WebEndpointController { result = componentLookupService.restLookup(user, WebHookEndpoint, params) //log.debug("DB duration: ${Duration.between(start_db, LocalDateTime.now()).toMillis();}") - log.debug("#### " + result) + log.debug("#### " + result.data.getClass().getName()) + log.debug("#### " + result.data) + + if (result.data) { + def resultList = result.data + resultList*.remove('ba_password') + resultList*.remove('ba_username') + + log.debug("+++++ " + resultList) + + if (params['method']) { + resultList = resultList.findAll( x -> x.transferMethod?.name == params['method']) + } + + result.data = resultList + } + + + + render result as JSON } diff --git a/server/grails-app/domain/org/gokb/cred/Source.groovy b/server/grails-app/domain/org/gokb/cred/Source.groovy index 687c19280..4fd9f489a 100644 --- a/server/grails-app/domain/org/gokb/cred/Source.groovy +++ b/server/grails-app/domain/org/gokb/cred/Source.groovy @@ -27,6 +27,9 @@ class Source extends KBComponent { BulkImportListConfig bulkConfig RefdataValue importConfig Boolean ignoreSizeLimit = false + WebHookEndpoint webEndpoint + String ftpUrl + RefdataValue transferMethod static manyByCombo = [ curatoryGroups: CuratoryGroup @@ -58,6 +61,9 @@ class Source extends KBComponent { bulkConfig(nullable: true, blank: false) importConfig(nullable: true, blank: true) ignoreSizeLimit(nullable: true, blank: true) + webEndpoint(nullable: true, blank: true) + ftpUrl(nullable: true, blank: true) + transferMethod(nullable: true, blank: true) } public static final String restPath = "/sources" diff --git a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy index bd33d75ac..4fbdd9d77 100644 --- a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy +++ b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy @@ -8,7 +8,8 @@ class WebHookEndpoint { String name String url Long authmethod //legacy - RefdataValue supplyMethod + RefdataValue supplyMethod //legacy + RefdataValue transferMethod String principal //legacy String credentials //legacy User owner @@ -31,6 +32,7 @@ class WebHookEndpoint { supplyMethod(nullable:true, blank:true) ba_username(nullable:true, blank:true) ba_password(nullable:true, blank:true) + transferMethod(nullable:true, blank:true) } static def refdataFind(params) { diff --git a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp index cf02ef4be..fa06d243e 100644 --- a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp +++ b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp @@ -5,8 +5,8 @@
Base URL
-
Methode
-
+
Methode
+
Benutzername
From 0d5e73918b0798dc08bcfde1eee410138b66d955 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Wed, 1 Oct 2025 23:21:33 +0200 Subject: [PATCH 07/28] changes --- .../gokb/PackageSourceUpdateService.groovy | 292 ++++++++++-------- 1 file changed, 162 insertions(+), 130 deletions(-) diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index c352775ba..2ec9682a2 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -97,7 +97,11 @@ class PackageSourceUpdateService { result.report = wekbIngestionService.startTitleImport(pkgInfo, pkg_source, pkg_plt, pkg_prov, p, job, async, restrictSize) } else { - if (pkg_source?.url) { + def transferMethod = pkg_source?.getTransferMethod() + def rdv_FTP = RefdataCategory.lookup('Source.TransferMethod', 'FTP') + boolean isFtpTransfer = (transferMethod == rdv_FTP) + + if (pkg_source?.url || isFtpTransfer) { URL src_url = null Boolean dynamic_date = false def valid_url_string = validationService.checkUrl(pkg_source?.url, true) @@ -123,7 +127,12 @@ class PackageSourceUpdateService { src_url = new URL(valid_url_string) } - } else { + + } + else if (isFtpTransfer) { + + } + else { log.debug("No source URL!") result.result = 'ERROR' result.messageCode = 'kbart.errors.url.invalid' @@ -133,8 +142,7 @@ class PackageSourceUpdateService { return result } - - if (src_url?.getProtocol() in ['http', 'https']) { + if (src_url?.getProtocol() in ['http', 'https'] || isFtpTransfer) { def deposit_token = java.util.UUID.randomUUID().toString() File tmp_file = TSVIngestionService.handleTempFile(deposit_token) def lastRunLocal = pkg_source.lastRun ? pkg_source.lastRun.toInstant().atZone(ZoneId.systemDefault()).toLocalDate() : null @@ -142,102 +150,111 @@ class PackageSourceUpdateService { pkg_source.lastRun = new Date() pkg_source.save(flush: true) - if (!extracted_date || !lastRunLocal || extracted_date > lastRunLocal) { - log.debug("Request initial URL..") - file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + if ( isFtpTransfer ) { + log.debug("Fetch KBART File from FTP Server...") + log.debug("##### SOURCE ####: " + pkg_source + ", " + pkg_source.getFtpUrl() + ", " + pkg_source.getWebEndpoint().getBa_username()) + return + file_info = fetchKbartFileFromFTPServer(tmp_file) } + else { // start not-FTP - if (file_info.connectError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.connection' - result.message = "There was an error trying to fetch KBART via URL!" - result.exceptionMsg = file_info.exceptionMsg + if (!extracted_date || !lastRunLocal || extracted_date > lastRunLocal) { + log.debug("Request initial URL..") + file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + } - createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + if (file_info.connectError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.connection' + result.message = "There was an error trying to fetch KBART via URL!" + result.exceptionMsg = file_info.exceptionMsg - return result - } + createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - if (file_info.fileSizeError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.fileSize' - result.message = "The attached KBART file is too big! Files bigger than 20 MB have to be authorized manually by an administrator." + return result + } - createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + if (file_info.fileSizeError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.fileSize' + result.message = "The attached KBART file is too big! Files bigger than 20 MB have to be authorized manually by an administrator." - return result - } + createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - if (file_info.accessError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.html' - result.message = "URL returned HTML, indicating provider configuration issues!" + return result + } - createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + if (file_info.accessError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.html' + result.message = "URL returned HTML, indicating provider configuration issues!" - return result - } else if (file_info.mimeTypeError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.mimeType' - result.message = "KBART URL returned a wrong content type!" - log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") + createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + return result + } else if (file_info.mimeTypeError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.mimeType' + result.message = "KBART URL returned a wrong content type!" + log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") - return result - } else if (file_info.status == 403) { - log.debug("URL request failed!") - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.denied' - result.message = "URL request returned 403 ACCESS DENIED, skipping further tries!" + createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + return result + } else if (file_info.status == 403) { + log.debug("URL request failed!") + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.denied' + result.message = "URL request returned 403 ACCESS DENIED, skipping further tries!" - return result - } + createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - if (!file_info.file_name && (dynamic_date || extracted_date)) { - LocalDate active_date = LocalDate.now() - boolean skipLookupByDate = false - src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) - log.debug("Fetching dated URL for today..") - file_info = fetchKbartFile(tmp_file, src_url, restrictSize) - - // Look at first of this month - if (!file_info.file_name) { - sleep(500) - log.debug("Fetching first of the month..") - def som_date_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.withDayOfMonth(1).toString())) - file_info = fetchKbartFile(tmp_file, som_date_url, restrictSize) - } + return result + } - // Check all days of this month - while (!skipLookupByDate && active_date.isAfter(LocalDate.now().minusDays(30)) && !file_info.file_name) { - active_date = active_date.minusDays(1) - src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) - log.debug("Fetching dated URL for date ${active_date}") - sleep(500) - file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + if (!file_info.file_name && (dynamic_date || extracted_date)) { + LocalDate active_date = LocalDate.now() + boolean skipLookupByDate = false + src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) + log.debug("Fetching dated URL for today..") + file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + + // Look at first of this month + if (!file_info.file_name) { + sleep(500) + log.debug("Fetching first of the month..") + def som_date_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.withDayOfMonth(1).toString())) + file_info = fetchKbartFile(tmp_file, som_date_url, restrictSize) + } - if (file_info.mimeTypeError) { - skipLookupByDate = true + // Check all days of this month + while (!skipLookupByDate && active_date.isAfter(LocalDate.now().minusDays(30)) && !file_info.file_name) { + active_date = active_date.minusDays(1) + src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) + log.debug("Fetching dated URL for date ${active_date}") + sleep(500) + file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + + if (file_info.mimeTypeError) { + skipLookupByDate = true + } + } } - } - } - if (file_info.mimeTypeError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.mimeType' - result.message = "KBART URL returned a wrong content type!" - log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") + if (file_info.mimeTypeError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.mimeType' + result.message = "KBART URL returned a wrong content type!" + log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") - createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - return result - } + return result + } - log.debug("Got mime type ${file_info.content_mime_type} for file ${file_info.file_name}") + log.debug("Got mime type ${file_info.content_mime_type} for file ${file_info.file_name}") + } // end not-FTP if (file_info.file_name) { try { MessageDigest md5_digest = MessageDigest.getInstance("MD5") @@ -327,59 +344,6 @@ class PackageSourceUpdateService { return result } } - else if (src_url.getProtocol() in ['ftp', 'sftp']) { - FTPClient ftp = new FTPClient() - FTPClientConfig config = new FTPClientConfig() - - def hostname = "ftp.epnet.com" - def username = "jake" - def password = "gijaq3eV" - def directory = "/kbart" - def filename = "8gh-kbart2.txt" - - try { - ftp.connect(hostname) - ftp.enterLocalPassiveMode() - log.debug("1111 : " + ftp.getReplyString() ) - def loggedIn = ftp.login(username, password) - log.debug("+++ EINGELOGGT... : " + loggedIn ) - - - if (ftp.isConnected()) { - //log.debug(" #### Directories: " + ftp.listDirectories()) - - ftp.changeWorkingDirectory(directory) - log.debug("2222 : " + ftp.getReplyString() ) - log.debug("##### Files: " + ftp.listFiles()) - log.debug("3333 : " + ftp.getReplyString() ) - - InputStream is = ftp.retrieveFileStream(filename) - - ByteArrayOutputStream tmp_result = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - for (int length; (length = is.read(buffer)) != -1; ) { - tmp_result.write(buffer, 0, length); - } - - log.debug("###########################################################################################") - log.debug(tmp_result.toString(StandardCharsets.UTF_8.name())) - log.debug("###########################################################################################") - - - //def file_result = TSVIngestionService.analyseFile(is) - - - ftp.logout() - ftp.disconnect() - } - - } catch (Exception e) { - e.printStackTrace() - } - - return - - } else { result.result = 'ERROR' result.messageCode = 'kbart.errors.url.protocol' @@ -648,4 +612,72 @@ class PackageSourceUpdateService { } } } + + def fetchKbartFileFromFTPServer (File tmp_file ) { + + def result = [content_mime_type: null, file_name: null] + + //if (src_url.getProtocol() in ['ftp', 'sftp']) { + + + FTPClient ftp = new FTPClient() + FTPClientConfig config = new FTPClientConfig() + + def hostname = "ftp.epnet.com" + def username = "jake" + def password = "gijaq3eV" + def directory = "/kbart" + //def filename = "8gh-kbart2.txt" + def filename = "31h-kbart2.txt" + + result.file_name = filename + + try { + ftp.connect(hostname) + ftp.enterLocalPassiveMode() + log.debug("1111 : " + ftp.getReplyString() ) + def loggedIn = ftp.login(username, password) + log.debug("+++ EINGELOGGT... : " + loggedIn ) + + + if (ftp.isConnected()) { + //log.debug(" #### Directories: " + ftp.listDirectories()) + + ftp.changeWorkingDirectory(directory) + log.debug("2222 : " + ftp.getReplyString() ) + log.debug("##### Files: " + ftp.listFiles()) + log.debug("3333 : " + ftp.getReplyString() ) + + InputStream is = ftp.retrieveFileStream(filename) + + OutputStream outStream = new FileOutputStream(tmp_file) + //ByteArrayOutputStream tmp_result = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + for (int length; (length = is.read(buffer)) != -1; ) { + outStream.write(buffer, 0, length); + } + + /* log.debug("###########################################################################################") + log.debug(tmp_result.toString(StandardCharsets.UTF_8.name())) + log.debug("###########################################################################################") + */ + + //def file_result = TSVIngestionService.analyseFile(is) + + outStream.close() + + log.debug("++++++++ Wrote ${tmp_file?.length()} ++++++++++++++++++++++++++++") + + ftp.logout() + ftp.disconnect() + } + + } catch (Exception e) { + e.printStackTrace() + } + + return result + + } + } \ No newline at end of file From b0a3a80f37c6e40ffe398dd2caf469de12c1135f Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Fri, 21 Nov 2025 17:59:41 +0100 Subject: [PATCH 08/28] clean FTP Urls --- .../grails-app/conf/spring/resources.groovy | 4 ++ .../gokb/PackageSourceUpdateService.groovy | 46 +++++++++++++++---- .../views/apptemplates/_web_hook_endpoint.gsp | 4 +- .../WebEndpointPasswordEncoderListener.groovy | 45 ++++++++++++++++++ 4 files changed, 88 insertions(+), 11 deletions(-) create mode 100644 server/src/main/groovy/com/k_int/WebEndpointPasswordEncoderListener.groovy diff --git a/server/grails-app/conf/spring/resources.groovy b/server/grails-app/conf/spring/resources.groovy index c0cc9861b..8bf72cf74 100644 --- a/server/grails-app/conf/spring/resources.groovy +++ b/server/grails-app/conf/spring/resources.groovy @@ -1,4 +1,5 @@ import com.k_int.UserPasswordEncoderListener +import com.k_int.WebEndpointPasswordEncoderListener import com.k_int.utils.DatabaseMessageSource; import org.grails.spring.context.support.PluginAwareResourceBundleMessageSource; import org.springframework.web.servlet.i18n.SessionLocaleResolver @@ -6,6 +7,9 @@ import org.springframework.web.servlet.i18n.SessionLocaleResolver beans = { userPasswordEncoderListener(UserPasswordEncoderListener) + // Passwords must be provided in plain Text to FTP Server + //webEndpointPasswordEncoderListener(WebEndpointPasswordEncoderListener) + messageBundleMessageSource(PluginAwareResourceBundleMessageSource) { basenames = ["WEB-INF/grails-app/i18n/messages"] defaultEncoding = "UTF-8" diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 81314f125..935c407e8 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -153,8 +153,8 @@ class PackageSourceUpdateService { if ( isFtpTransfer ) { log.debug("Fetch KBART File from FTP Server...") log.debug("##### SOURCE ####: " + pkg_source + ", " + pkg_source.getFtpUrl() + ", " + pkg_source.getWebEndpoint().getBa_username()) - return - file_info = fetchKbartFileFromFTPServer(tmp_file) + + file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source) } else { // start not-FTP @@ -626,22 +626,48 @@ class PackageSourceUpdateService { return info_map } - def fetchKbartFileFromFTPServer (File tmp_file ) { + def fetchKbartFileFromFTPServer (File tmp_file, Source source) { def result = [content_mime_type: null, file_name: null] - //if (src_url.getProtocol() in ['ftp', 'sftp']) { - - FTPClient ftp = new FTPClient() FTPClientConfig config = new FTPClientConfig() - def hostname = "ftp.epnet.com" + /* def hostname = "ftp.epnet.com" def username = "jake" def password = "gijaq3eV" def directory = "/kbart" - //def filename = "8gh-kbart2.txt" - def filename = "31h-kbart2.txt" + def filename = "31h-kbart2.txt" */ + + String hostname = source.getWebEndpoint().getUrl() + String filename = null + String directory = null + + //we dont need the protocol + hostname = hostname?.replace("ftp://", "") + if (hostname?.contains("/")) { + directory = hostname.substring(hostname.indexOf("/"), hostname.length()) + hostname = hostname.split("/")[0] + } + log.debug("111 directory: " + directory) + log.debug("111 hostname: " + hostname) + + String username = source.getWebEndpoint().getBa_username() + String password = source.getWebEndpoint().getBa_password() + + String ftpUrl = source.getFtpUrl() + if (ftpUrl?.contains("/")) { + String[] parts = ftpUrl.split("/") + filename = parts[parts.length - 1] + directory = directory + ftpUrl.substring(0, ftpUrl.lastIndexOf("/") + 1) + } + else { + filename = ftpUrl + } + + log.debug("222 directory: " + directory) + log.debug("222 hostname: " + hostname) + log.debug("222 filename: " + filename) result.file_name = filename @@ -686,7 +712,7 @@ class PackageSourceUpdateService { } } catch (Exception e) { - e.printStackTrace() + log.error("Fehler beim FTP") } return result diff --git a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp index fa06d243e..359ef37f9 100644 --- a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp +++ b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp @@ -12,5 +12,7 @@
Passwort
-
+ +
+ \ No newline at end of file diff --git a/server/src/main/groovy/com/k_int/WebEndpointPasswordEncoderListener.groovy b/server/src/main/groovy/com/k_int/WebEndpointPasswordEncoderListener.groovy new file mode 100644 index 000000000..0dff95729 --- /dev/null +++ b/server/src/main/groovy/com/k_int/WebEndpointPasswordEncoderListener.groovy @@ -0,0 +1,45 @@ +package com.k_int + + +import grails.plugin.springsecurity.SpringSecurityService +import org.gokb.cred.WebHookEndpoint +import org.grails.datastore.mapping.engine.event.AbstractPersistenceEvent +import org.grails.datastore.mapping.engine.event.PreInsertEvent +import org.grails.datastore.mapping.engine.event.PreUpdateEvent +import org.springframework.beans.factory.annotation.Autowired +import grails.events.annotation.gorm.Listener +import groovy.transform.CompileStatic + +@CompileStatic +class WebEndpointPasswordEncoderListener { + + + @Autowired + SpringSecurityService springSecurityService + + @Listener(WebHookEndpoint) + void onPreInsertEvent(PreInsertEvent event) { + encodePasswordForEvent(event) + } + + @Listener(WebHookEndpoint) + void onPreUpdateEvent(PreUpdateEvent event) { + encodePasswordForEvent(event) + } + + private void encodePasswordForEvent(AbstractPersistenceEvent event) { + if (event.entityObject instanceof WebHookEndpoint) { + WebHookEndpoint w = event.entityObject as WebHookEndpoint + if (w.ba_password && ((event instanceof PreInsertEvent) || (event instanceof PreUpdateEvent && w.isDirty('ba_password')))) { + event.getEntityAccess().setProperty('ba_password', encodePassword(w.ba_password)) + } + } + } + + private String encodePassword(String password) { + springSecurityService?.passwordEncoder ? springSecurityService.encodePassword(password) : password + + } +} + + From 99724707a34400b1b449a74970dd43068bb34656 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Mon, 24 Nov 2025 14:53:18 +0100 Subject: [PATCH 09/28] handle FTP Urls --- .../gokb/PackageSourceUpdateService.groovy | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 935c407e8..215dfc523 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -639,24 +639,27 @@ class PackageSourceUpdateService { def directory = "/kbart" def filename = "31h-kbart2.txt" */ - String hostname = source.getWebEndpoint().getUrl() - String filename = null - String directory = null - + String username = source.getWebEndpoint().getBa_username() + String password = source.getWebEndpoint().getBa_password() //we dont need the protocol - hostname = hostname?.replace("ftp://", "") + String hostname = source.getWebEndpoint().getUrl()?.replace("ftp://", "") + String filename = "" + String directory = "/" + if (hostname?.contains("/")) { - directory = hostname.substring(hostname.indexOf("/"), hostname.length()) - hostname = hostname.split("/")[0] + String[] parts = hostname.split("/") + hostname = parts[0] + for(int i = 1; i < parts.length; i++){ + directory = directory.concat(parts[i] + "/") + } } - log.debug("111 directory: " + directory) - log.debug("111 hostname: " + hostname) - - String username = source.getWebEndpoint().getBa_username() - String password = source.getWebEndpoint().getBa_password() String ftpUrl = source.getFtpUrl() - if (ftpUrl?.contains("/")) { + if (ftpUrl?.startsWith("/")) { + ftpUrl = ftpUrl.substring(1) + } + + if(ftpUrl?.contains("/")){ String[] parts = ftpUrl.split("/") filename = parts[parts.length - 1] directory = directory + ftpUrl.substring(0, ftpUrl.lastIndexOf("/") + 1) @@ -712,7 +715,7 @@ class PackageSourceUpdateService { } } catch (Exception e) { - log.error("Fehler beim FTP") + log.error("Fehler bei FTP-Verbindung: " + e.getMessage()) } return result From c2f7597f608546526dd47da1a935a4e1fdc79711 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Wed, 26 Nov 2025 16:58:28 +0100 Subject: [PATCH 10/28] add test ftp connection functionality mask passwords in admin ui --- .../controllers/org/gokb/UrlMappings.groovy | 2 + .../gokb/rest/WebEndpointController.groovy | 85 ++++++++++++++++--- .../gokb/PackageSourceUpdateService.groovy | 63 ++------------ .../views/apptemplates/_web_hook_endpoint.gsp | 9 +- 4 files changed, 92 insertions(+), 67 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/UrlMappings.groovy b/server/grails-app/controllers/org/gokb/UrlMappings.groovy index 943051ccb..768b4686f 100644 --- a/server/grails-app/controllers/org/gokb/UrlMappings.groovy +++ b/server/grails-app/controllers/org/gokb/UrlMappings.groovy @@ -193,6 +193,8 @@ class UrlMappings { delete "/jobs/$id" (controller: 'jobs', namespace: 'rest', action: 'delete') get "/web-endpoint"(controller: 'webEndpoint', namespace: 'rest', action: 'index') + get "/web-endpoint/$id"(controller: 'webEndpoint', namespace: 'rest', action: 'show') + post "/web-endpoint/check"(controller: 'webEndpoint', namespace: 'rest', action: 'check') } "/$controller/$action?/$id?" { constraints { diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index a81eeab20..6b15a0f16 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -2,6 +2,10 @@ package org.gokb.rest import grails.converters.JSON import grails.plugin.springsecurity.annotation.Secured +import org.apache.commons.net.ftp.FTPClient +import org.apache.commons.net.ftp.FTPClientConfig +import org.apache.commons.net.ftp.FTPFile +import org.gokb.WebEndpointService import org.gokb.cred.User import org.gokb.cred.WebHookEndpoint @@ -14,6 +18,7 @@ class WebEndpointController { def componentLookupService def springSecurityService + WebEndpointService webEndpointService @Secured(['IS_AUTHENTICATED_ANONYMOUSLY']) def index() { @@ -31,16 +36,12 @@ class WebEndpointController { result = componentLookupService.restLookup(user, WebHookEndpoint, params) //log.debug("DB duration: ${Duration.between(start_db, LocalDateTime.now()).toMillis();}") - log.debug("#### " + result.data.getClass().getName()) - log.debug("#### " + result.data) if (result.data) { def resultList = result.data resultList*.remove('ba_password') resultList*.remove('ba_username') - log.debug("+++++ " + resultList) - if (params['method']) { resultList = resultList.findAll( x -> x.transferMethod?.name == params['method']) } @@ -48,13 +49,10 @@ class WebEndpointController { result.data = resultList } - - - - render result as JSON } + @Secured(['IS_AUTHENTICATED_ANONYMOUSLY']) def show() { def result = [:] def base = grailsApplication.config.getProperty('grails.serverURL') + "/rest" @@ -65,15 +63,80 @@ class WebEndpointController { } def start_db = LocalDateTime.now() - params['_embed'] = params['_embed'] ?: 'identifiedComponents' result = componentLookupService.restLookup(user, WebHookEndpoint, params) - //log.debug("DB duration: ${Duration.between(start_db, LocalDateTime.now()).toMillis();}") - log.debug("#### " + result) + + def resultList = result.data + + if(resultList.size() > 0){ + resultList*.remove('ba_password') + resultList*.remove('ba_username') + + result.data = resultList.get(0) + } render result as JSON } + @Secured(['IS_AUTHENTICATED_ANONYMOUSLY']) + def check() { + def result = [:] + def reqBody = request.JSON + + WebHookEndpoint whe = WebHookEndpoint.findById(reqBody.webhookendpoint) + String path = reqBody.url + String hostname = "" + String directory = "" + String filename = "" + + if(whe){ + def parts= webEndpointService.extractFtpUrlParts(whe.getUrl(), path) + log.debug("*** " + parts.hostname + ", " + parts.directory + ", " + parts.filename) + hostname = parts.hostname + directory = parts.directory + filename = parts.filename + } + + FTPClient ftp = new FTPClient() + FTPClientConfig config = new FTPClientConfig() + + try { + ftp.connect(hostname) + ftp.enterLocalPassiveMode() + def loggedIn = ftp.login(whe.getBa_username(), whe.getBa_password()) + + if (ftp.isConnected()) { + + FTPFile[] files = ftp.listFiles(directory + filename) + if(files.length > 0 && files[0].size > 0){ + result.result = "success" + result.message = "success" + } + else { + result.result = "error" + result.message = "partlySuccessful" + } + + ftp.logout() + ftp.disconnect() + } + else { + result.result = "error" + result.message = "connectError" + } + + } catch (Exception e) { + log.error("Fehler bei FTP-Verbindung: ", e) + result.result = "error" + result.message = "configurationError" + } + + render result as JSON + + } + + + } diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 719f06878..73fd62250 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -28,6 +28,7 @@ class PackageSourceUpdateService { def validationService WekbIngestionService wekbIngestionService boolean isExternalSourceImportOrUpdate + WebEndpointService webEndpointService static Pattern DATE_PLACEHOLDER_PATTERN = ~/[0-9]{4}-[0-9]{2}-[0-9]{2}/ static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ @@ -151,8 +152,7 @@ class PackageSourceUpdateService { pkg_source.save(flush: true) if ( isFtpTransfer ) { - log.debug("Fetch KBART File from FTP Server...") - log.debug("##### SOURCE ####: " + pkg_source + ", " + pkg_source.getFtpUrl() + ", " + pkg_source.getWebEndpoint().getBa_username()) + log.debug("Start FTP Update from Source " + pkg_source ) file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source) } @@ -633,89 +633,42 @@ class PackageSourceUpdateService { FTPClient ftp = new FTPClient() FTPClientConfig config = new FTPClientConfig() - /* def hostname = "ftp.epnet.com" - def username = "jake" - def password = "gijaq3eV" - def directory = "/kbart" - def filename = "31h-kbart2.txt" */ - String username = source.getWebEndpoint().getBa_username() String password = source.getWebEndpoint().getBa_password() - //we dont need the protocol - String hostname = source.getWebEndpoint().getUrl()?.replace("ftp://", "") - String filename = "" - String directory = "/" - - if (hostname?.contains("/")) { - String[] parts = hostname.split("/") - hostname = parts[0] - for(int i = 1; i < parts.length; i++){ - directory = directory.concat(parts[i] + "/") - } - } - String ftpUrl = source.getFtpUrl() - if (ftpUrl?.startsWith("/")) { - ftpUrl = ftpUrl.substring(1) - } - - if(ftpUrl?.contains("/")){ - String[] parts = ftpUrl.split("/") - filename = parts[parts.length - 1] - directory = directory + ftpUrl.substring(0, ftpUrl.lastIndexOf("/") + 1) - } - else { - filename = ftpUrl - } + def urlParts = webEndpointService.extractFtpUrlParts(source.getWebEndpoint().getUrl(), source.getFtpUrl()) - log.debug("222 directory: " + directory) - log.debug("222 hostname: " + hostname) - log.debug("222 filename: " + filename) + String hostname = urlParts.hostname + String filename = urlParts.filename + String directory = urlParts.directory result.file_name = filename try { ftp.connect(hostname) ftp.enterLocalPassiveMode() - log.debug("1111 : " + ftp.getReplyString() ) def loggedIn = ftp.login(username, password) - log.debug("+++ EINGELOGGT... : " + loggedIn ) - if (ftp.isConnected()) { - //log.debug(" #### Directories: " + ftp.listDirectories()) ftp.changeWorkingDirectory(directory) - log.debug("2222 : " + ftp.getReplyString() ) - log.debug("##### Files: " + ftp.listFiles()) - log.debug("3333 : " + ftp.getReplyString() ) InputStream is = ftp.retrieveFileStream(filename) - OutputStream outStream = new FileOutputStream(tmp_file) - //ByteArrayOutputStream tmp_result = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; for (int length; (length = is.read(buffer)) != -1; ) { outStream.write(buffer, 0, length); } - /* log.debug("###########################################################################################") - log.debug(tmp_result.toString(StandardCharsets.UTF_8.name())) - log.debug("###########################################################################################") - */ - - //def file_result = TSVIngestionService.analyseFile(is) - outStream.close() - log.debug("++++++++ Wrote ${tmp_file?.length()} ++++++++++++++++++++++++++++") - ftp.logout() ftp.disconnect() } } catch (Exception e) { - log.error("Fehler bei FTP-Verbindung: " + e.getMessage()) + log.error("Fehler bei FTP-Verbindung ", e) } return result diff --git a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp index 359ef37f9..defc1176d 100644 --- a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp +++ b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp @@ -15,4 +15,11 @@
- \ No newline at end of file + + + \ No newline at end of file From 6dac257c695f1ed9546d169a5d4bed118c6bb80d Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Fri, 28 Nov 2025 10:30:45 +0100 Subject: [PATCH 11/28] added missing Service class --- .../org/gokb/WebEndpointService.groovy | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 server/grails-app/services/org/gokb/WebEndpointService.groovy diff --git a/server/grails-app/services/org/gokb/WebEndpointService.groovy b/server/grails-app/services/org/gokb/WebEndpointService.groovy new file mode 100644 index 000000000..3c1c62673 --- /dev/null +++ b/server/grails-app/services/org/gokb/WebEndpointService.groovy @@ -0,0 +1,51 @@ +package org.gokb + +import grails.gorm.transactions.Transactional + +@Transactional +class WebEndpointService { + + def extractFtpUrlParts (String webEndpointUrl, String sourceUrl) { + def result = [:] + + //we dont need the protocol + String hostname = webEndpointUrl.replace("ftp://", "") + String filename = "" + String directory = "/" + + if (hostname?.contains("/")) { + String[] parts = hostname.split("/") + hostname = parts[0] + for(int i = 1; i < parts.length; i++){ + directory = directory.concat(parts[i] + "/") + } + } + + if (sourceUrl?.startsWith("/")) { + sourceUrl = sourceUrl.substring(1) + } + + if(sourceUrl?.contains("/")){ + String[] parts = sourceUrl.split("/") + filename = parts[parts.length - 1] + directory = directory + sourceUrl.substring(0, sourceUrl.lastIndexOf("/") + 1) + } + else { + filename = sourceUrl + } + + result.hostname = hostname + result.filename = filename + result.directory = directory + + + result + + } + + + + + + +} From 99bae79737d5885ccff01f2dffd44e1527dd51d3 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Fri, 28 Nov 2025 12:48:02 +0100 Subject: [PATCH 12/28] notes --- server/grails-app/domain/org/gokb/cred/Source.groovy | 2 +- server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/grails-app/domain/org/gokb/cred/Source.groovy b/server/grails-app/domain/org/gokb/cred/Source.groovy index 924395240..6d561dbf5 100644 --- a/server/grails-app/domain/org/gokb/cred/Source.groovy +++ b/server/grails-app/domain/org/gokb/cred/Source.groovy @@ -28,7 +28,7 @@ class Source extends KBComponent { RefdataValue importConfig Boolean ignoreSizeLimit = false WebHookEndpoint webEndpoint - String ftpUrl + String ftpUrl //umbenennen in ftpPath RefdataValue transferMethod static manyByCombo = [ diff --git a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy index 4fbdd9d77..522dfee9d 100644 --- a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy +++ b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy @@ -9,7 +9,7 @@ class WebHookEndpoint { String url Long authmethod //legacy RefdataValue supplyMethod //legacy - RefdataValue transferMethod + RefdataValue transferMethod //rausnehmen - nur in Source String principal //legacy String credentials //legacy User owner From 5c2a260e9751dd7a446057c09d5fb6111c5b0dd9 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Wed, 3 Dec 2025 17:54:45 +0100 Subject: [PATCH 13/28] remove unneeded field --- .../domain/org/gokb/cred/WebHookEndpoint.groovy | 3 +-- .../org/gokb/PackageSourceUpdateService.groovy | 6 +++++- .../services/org/gokb/WebEndpointService.groovy | 10 ++++++++++ .../views/apptemplates/_web_hook_endpoint.gsp | 3 --- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy index 522dfee9d..b7152ade6 100644 --- a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy +++ b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy @@ -9,7 +9,7 @@ class WebHookEndpoint { String url Long authmethod //legacy RefdataValue supplyMethod //legacy - RefdataValue transferMethod //rausnehmen - nur in Source + // RefdataValue transferMethod //rausnehmen - nur in Source String principal //legacy String credentials //legacy User owner @@ -32,7 +32,6 @@ class WebHookEndpoint { supplyMethod(nullable:true, blank:true) ba_username(nullable:true, blank:true) ba_password(nullable:true, blank:true) - transferMethod(nullable:true, blank:true) } static def refdataFind(params) { diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 73fd62250..b8ef4a062 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -131,7 +131,11 @@ class PackageSourceUpdateService { } else if (isFtpTransfer) { - + // no op here + String urlFilePart = pkg_source.getWebEndpoint()?.getUrl() + if(urlFilePart =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN){ + dynamic_date = true + } } else { log.debug("No source URL!") diff --git a/server/grails-app/services/org/gokb/WebEndpointService.groovy b/server/grails-app/services/org/gokb/WebEndpointService.groovy index 3c1c62673..f2ba8bc8e 100644 --- a/server/grails-app/services/org/gokb/WebEndpointService.groovy +++ b/server/grails-app/services/org/gokb/WebEndpointService.groovy @@ -2,12 +2,22 @@ package org.gokb import grails.gorm.transactions.Transactional +import java.util.regex.Pattern + @Transactional class WebEndpointService { + static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ + def extractFtpUrlParts (String webEndpointUrl, String sourceUrl) { def result = [:] + boolean isDateMasked = false + if (sourceUrl =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN) { + log.debug("### IS MIT DATUMSMASKIERUNG ###") + isDateMasked = true + } + //we dont need the protocol String hostname = webEndpointUrl.replace("ftp://", "") String filename = "" diff --git a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp index defc1176d..10e40466e 100644 --- a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp +++ b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp @@ -5,9 +5,6 @@
Base URL
-
Methode
-
-
Benutzername
From 643d11c9693adb33f79d2359a4f6b057196cb028 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Wed, 3 Dec 2025 17:56:48 +0100 Subject: [PATCH 14/28] remove pw encryption --- .../grails-app/conf/spring/resources.groovy | 5 +-- .../WebEndpointPasswordEncoderListener.groovy | 45 ------------------- 2 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 server/src/main/groovy/com/k_int/WebEndpointPasswordEncoderListener.groovy diff --git a/server/grails-app/conf/spring/resources.groovy b/server/grails-app/conf/spring/resources.groovy index 8bf72cf74..1b5089e4e 100644 --- a/server/grails-app/conf/spring/resources.groovy +++ b/server/grails-app/conf/spring/resources.groovy @@ -1,5 +1,5 @@ import com.k_int.UserPasswordEncoderListener -import com.k_int.WebEndpointPasswordEncoderListener + import com.k_int.utils.DatabaseMessageSource; import org.grails.spring.context.support.PluginAwareResourceBundleMessageSource; import org.springframework.web.servlet.i18n.SessionLocaleResolver @@ -7,9 +7,6 @@ import org.springframework.web.servlet.i18n.SessionLocaleResolver beans = { userPasswordEncoderListener(UserPasswordEncoderListener) - // Passwords must be provided in plain Text to FTP Server - //webEndpointPasswordEncoderListener(WebEndpointPasswordEncoderListener) - messageBundleMessageSource(PluginAwareResourceBundleMessageSource) { basenames = ["WEB-INF/grails-app/i18n/messages"] defaultEncoding = "UTF-8" diff --git a/server/src/main/groovy/com/k_int/WebEndpointPasswordEncoderListener.groovy b/server/src/main/groovy/com/k_int/WebEndpointPasswordEncoderListener.groovy deleted file mode 100644 index 0dff95729..000000000 --- a/server/src/main/groovy/com/k_int/WebEndpointPasswordEncoderListener.groovy +++ /dev/null @@ -1,45 +0,0 @@ -package com.k_int - - -import grails.plugin.springsecurity.SpringSecurityService -import org.gokb.cred.WebHookEndpoint -import org.grails.datastore.mapping.engine.event.AbstractPersistenceEvent -import org.grails.datastore.mapping.engine.event.PreInsertEvent -import org.grails.datastore.mapping.engine.event.PreUpdateEvent -import org.springframework.beans.factory.annotation.Autowired -import grails.events.annotation.gorm.Listener -import groovy.transform.CompileStatic - -@CompileStatic -class WebEndpointPasswordEncoderListener { - - - @Autowired - SpringSecurityService springSecurityService - - @Listener(WebHookEndpoint) - void onPreInsertEvent(PreInsertEvent event) { - encodePasswordForEvent(event) - } - - @Listener(WebHookEndpoint) - void onPreUpdateEvent(PreUpdateEvent event) { - encodePasswordForEvent(event) - } - - private void encodePasswordForEvent(AbstractPersistenceEvent event) { - if (event.entityObject instanceof WebHookEndpoint) { - WebHookEndpoint w = event.entityObject as WebHookEndpoint - if (w.ba_password && ((event instanceof PreInsertEvent) || (event instanceof PreUpdateEvent && w.isDirty('ba_password')))) { - event.getEntityAccess().setProperty('ba_password', encodePassword(w.ba_password)) - } - } - } - - private String encodePassword(String password) { - springSecurityService?.passwordEncoder ? springSecurityService.encodePassword(password) : password - - } -} - - From e1ebeda8656e80599fcce6bd12fa983af98101bc Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Mon, 8 Dec 2025 19:10:40 +0100 Subject: [PATCH 15/28] merge --- .../gokb/rest/WebEndpointController.groovy | 4 +- .../gokb/PackageSourceUpdateService.groovy | 174 +++++++++--------- 2 files changed, 86 insertions(+), 92 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index 6b15a0f16..10fd8e1e1 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -42,9 +42,9 @@ class WebEndpointController { resultList*.remove('ba_password') resultList*.remove('ba_username') - if (params['method']) { + /* if (params['method']) { resultList = resultList.findAll( x -> x.transferMethod?.name == params['method']) - } + } */ result.data = resultList } diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index cf5cc3b54..411222fad 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -6,7 +6,6 @@ import grails.converters.JSON import groovy.util.logging.Slf4j import org.apache.commons.net.ftp.FTPClient import org.apache.commons.net.ftp.FTPClientConfig - import org.apache.http.HttpEntity import org.apache.http.HttpHeaders import org.apache.http.util.EntityUtils @@ -443,120 +442,115 @@ class PackageSourceUpdateService { def fetchKbartFile(File tmp_file, URL src_url, boolean restrictSize = true) { def result = [content_mime_type: null, file_name: null] - HttpClient client = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(30)) - .followRedirects(HttpClient.Redirect.NORMAL) - .build() - Long max_length = 20971520L // 1024 * 1024 * 20 Long content_length - try { - HttpRequest head_request = HttpRequest.newBuilder() - .uri(src_url.toURI()) - .header("User-Agent", "GOKb KBART Updater") - .method('HEAD', BodyPublishers.noBody()) - .build() + RequestConfig requestConfig = RequestConfig.custom() + .setConnectionRequestTimeout(10000) + .setSocketTimeout(30000) + .build() - def head_response = client.send(head_request, BodyHandlers.discarding()) + HttpClientBuilder builder = HttpClients.custom() + .setDefaultRequestConfig(requestConfig) - if (head_response?.statusCode() == 405) { - log.debug("Unable to send HEAD request ..") - } - else if (head_response?.statusCode()) { - HttpHeaders test_headers = head_response.headers() - content_length = test_headers.firstValue('Content-Length').isPresent() ? Long.valueOf(test_headers.firstValue('Content-Length').get()) : null + try (CloseableHttpClient httpClient = builder.build()) { + HttpHead httpHead = new HttpHead(src_url.toURI()) + HttpGet httpGet = new HttpGet(src_url.toURI()) + + httpHead.setHeader(HttpHeaders.USER_AGENT, "GOKb KBART Updater") + httpGet.setHeader(HttpHeaders.USER_AGENT, "GOKb KBART Updater") + + httpClient.execute(httpHead, classicHttpResponse -> { + int code = classicHttpResponse.getStatusLine().getStatusCode() - log.debug("Got HEAD result headers: ${test_headers}") + if (code == 405) { + log.debug("Unable to send HEAD request ..") + } + else if (code) { + content_length = classicHttpResponse.containsHeader('Content-Length') ? Long.valueOf(classicHttpResponse.getFirstHeader('Content-Length').getValue()) : null + } // reject files bigger than 20 MB if (restrictSize && content_length && content_length > max_length) { result.fileSizeError = true return result } - } - - HttpRequest request = HttpRequest.newBuilder() - .uri(src_url.toURI()) - .header("User-Agent", "GOKb KBART Updater") - .build() - - HttpResponse response = client.send(request, BodyHandlers.ofInputStream()) - HttpHeaders headers = response.headers() + }) - log.debug("Got HEAD result headers: ${headers}") + httpClient.execute(httpGet, classicHttpResponse -> { + int code = classicHttpResponse.getStatusLine().getStatusCode() - def file_name = headers.firstValue('Content-Disposition').isPresent() ? headers.firstValue('Content-Disposition').get() : null + String file_name = classicHttpResponse.containsHeader('Content-Disposition') ? classicHttpResponse.getFirstHeader('Content-Disposition').getValue() : null - if (file_name?.contains('filename=')) { - file_name = file_name.split('filename=')[1] - } - else if (file_name?.contains('filename*=')) { - file_name = file_name.split('filename*=')[1].split("'")[2] - } - - result.content_mime_type = headers.firstValue('Content-Type').isPresent() ? headers.firstValue('Content-Type').get() : null - - if (response.statusCode() >= 400) { - log.debug("KBART fetch status: ${response.statusCode()}") - } - else if (!file_name && result.content_mime_type?.startsWith('text/plain')) { - file_name = src_url.toString().split('/')[src_url.toString().split('/').size() - 1] - } - else if (!file_name && result.content_mime_type?.startsWith('text/html')) { - log.warn("Got HTML result at KBART URL ${src_url}!") - result.accessError = true - return result - } - - content_length = headers.firstValue('Content-Length').isPresent() ? Long.valueOf(headers.firstValue('Content-Length').get()) : null - - if (restrictSize && content_length && content_length > max_length) { - response.body().close() - result.fileSizeError = true - } - else if (file_name?.trim()) { - file_name = file_name.replaceAll(/\"/, '') + if (file_name?.contains('filename=')) { + file_name = file_name.split('filename=')[1] + } + else if (file_name?.contains('filename*=')) { + file_name = file_name.split('filename*=')[1].split("'")[2] + } - if ((file_name?.trim()?.endsWith('.tsv') || file_name?.trim()?.endsWith('.txt') || file_name?.trim()?.endsWith('.kbart')) && - (result.content_mime_type?.startsWith("text/plain") || - result.content_mime_type?.startsWith("text/csv") || - result.content_mime_type?.startsWith("text/tab-separated-values") || - result.content_mime_type == 'application/octet-stream')) { - log.debug("${result.content_mime_type} ${headers.map()}") - result.file_name = file_name + result.content_mime_type = classicHttpResponse.getFirstHeader('Content-Type').getValue() - OutputStream outStream = new FileOutputStream(tmp_file) + if (code > 400) { + log.debug("KBART fetch status: ${code}") + } + else if (!file_name && result.content_mime_type?.startsWith('text/plain')) { + file_name = src_url.toString().split('/')[src_url.toString().split('/').size() - 1] + } + else if (!file_name && result.content_mime_type?.startsWith('text/html')) { + log.warn("Got HTML result at KBART URL ${src_url}!") + result.accessError = true + return result + } - byte[] buffer = new byte[8 * 1024] - int bytesRead - Long current_total = 0 + content_length = classicHttpResponse.containsHeader('Content-Length') ? Long.valueOf(classicHttpResponse.getFirstHeader('Content-Length').getValue()) : null - while ((bytesRead = response.body().read(buffer)) != -1) { - outStream.write(buffer, 0, bytesRead) + if (restrictSize && content_length && content_length > max_length) { + result.fileSizeError = true + } + else if (file_name?.trim()) { + file_name = file_name.replaceAll(/\"/, '') + + if ((file_name?.trim()?.endsWith('.tsv') || file_name?.trim()?.endsWith('.txt') || file_name?.trim()?.endsWith('.kbart')) && + (result.content_mime_type?.startsWith("text/plain") || + result.content_mime_type?.startsWith("text/csv") || + result.content_mime_type?.startsWith("text/tab-separated-values") || + result.content_mime_type == 'application/octet-stream')) { + HttpEntity entity = classicHttpResponse.getEntity(); + result.file_name = file_name + + if (entity != null) { + try (InputStream inputStream = entity.getContent(); FileOutputStream fileOutputStream = new FileOutputStream(tmp_file)) { + byte[] dataBuffer = new byte[1024]; + int bytesRead; + Long current_total = 0; + + while((bytesRead = inputStream.read(dataBuffer)) != -1) { + fileOutputStream.write(dataBuffer, 0, bytesRead); + + current_total += bytesRead + + if (restrictSize && !content_length && current_total > max_length) { + result.fileSizeError = true + break + } + } - current_total += bytesRead + if (!result.fileSizeError) { + EntityUtils.consume(entity) + } + } - if (!content_length && current_total > max_length) { - result.fileSizeError = true - response.body().close() - break + if (result.fileSizeError) { + tmp_file.delete() + } } } - - outStream.close() - - log.debug("Wrote ${tmp_file?.length()}") - - if (result.fileSizeError) { - tmp_file.delete() + else { + result.mimeTypeError = true } } - else { - result.mimeTypeError = true - response.body().close() - } - } + }) } catch (Exception e) { result.connectError = true From 658fc9aa4101a4cbd535fa475b60c13d6604bb50 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Wed, 17 Dec 2025 18:13:59 +0100 Subject: [PATCH 16/28] merge --- .../gokb/rest/WebEndpointController.groovy | 53 +++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index 10fd8e1e1..36e0f4df9 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -11,6 +11,7 @@ import org.gokb.cred.WebHookEndpoint import java.time.Duration import java.time.LocalDateTime +import java.util.regex.Pattern class WebEndpointController { @@ -20,6 +21,9 @@ class WebEndpointController { def springSecurityService WebEndpointService webEndpointService + static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ + static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)$/ + @Secured(['IS_AUTHENTICATED_ANONYMOUSLY']) def index() { def result = [:] @@ -98,6 +102,10 @@ class WebEndpointController { filename = parts.filename } + boolean isDateMasked = (filename =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN) + + log.debug("11111: " + filename + ", " + isDateMasked) + FTPClient ftp = new FTPClient() FTPClientConfig config = new FTPClientConfig() @@ -108,18 +116,45 @@ class WebEndpointController { if (ftp.isConnected()) { - FTPFile[] files = ftp.listFiles(directory + filename) - if(files.length > 0 && files[0].size > 0){ - result.result = "success" - result.message = "success" + FTPFile[] files + if(isDateMasked) { + String fixedPart = filename.split("\\{")[0] + files = ftp.listFiles(directory) + + boolean found = false + + for (FTPFile file : files) { + + log.debug("+++ " + file.name) + + if(file.name.startsWith(fixedPart)){ + result.result = "success" + //TODO: message + result.message = "success, file found with name: ${file.name}" + found = true + break + } + } + if(!found){ + result.result = "error" + //TODO: message + result.message = "dateMaskNotFound" + } } else { - result.result = "error" - result.message = "partlySuccessful" + files = ftp.listFiles(directory + filename) + if (files.length > 0 && files[0].size > 0) { + + result.result = "success" + result.message = "success" + } else { + result.result = "error" + result.message = "partlySuccessful" + } + + ftp.logout() + ftp.disconnect() } - - ftp.logout() - ftp.disconnect() } else { result.result = "error" From be1bc880140e5423ae849b88b948a46bbdd7f4a8 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Thu, 18 Dec 2025 18:36:10 +0100 Subject: [PATCH 17/28] validator url --- .../org/gokb/rest/WebEndpointController.groovy | 8 +++++--- .../domain/org/gokb/cred/WebHookEndpoint.groovy | 11 ++++++++++- server/grails-app/i18n/messages.properties | 3 +++ server/grails-app/i18n/messages_de.properties | 5 ++++- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index 36e0f4df9..20c909902 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -46,9 +46,11 @@ class WebEndpointController { resultList*.remove('ba_password') resultList*.remove('ba_username') - /* if (params['method']) { - resultList = resultList.findAll( x -> x.transferMethod?.name == params['method']) - } */ + if (params['method']) { + //resultList = resultList.findAll( x -> x.transferMethod?.name == params['method']) + log.debug("1111: " + params['method']) + resultList = resultList.findAll( x -> x.url.startsWith(params['method'].toLowerCase()) ) + } result.data = resultList } diff --git a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy index b7152ade6..206f7f307 100644 --- a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy +++ b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy @@ -25,7 +25,16 @@ class WebHookEndpoint { static constraints = { name(nullable:false, blank:false) - url(nullable:false, blank:false) + url(validator: {val, obj -> + if(val) { + if(!val.startsWith("ftp://") && !val.startsWith("http://") && !val.startsWith("https://")){ + return ['webEndpointUrl.missingProtocol'] + } + } + else { + return ['webEndpointUrl.notNull'] + } + }) authmethod(nullable:true, blank:true) principal(nullable:true, blank:true) credentials(nullable:true, blank:true) diff --git a/server/grails-app/i18n/messages.properties b/server/grails-app/i18n/messages.properties index 8a269c40f..ef4d7044a 100644 --- a/server/grails-app/i18n/messages.properties +++ b/server/grails-app/i18n/messages.properties @@ -540,3 +540,6 @@ componentPrice.priceType.nullable.error = The kind of price must be set. admin.support.sizeLimit.email.subject = 'GOKB - Import failed due to active KBART size limit' admin.support.sizeLimit.email.line1 = 'An import for the following package failed due to the default KBART size limit:' + +webEndpointUrl.missingProtocol = 'Please provide the URL including the protocol (e.g., ftp:// or http://).' +webEndpointUrl.notNull = 'Please enter a URL.' diff --git a/server/grails-app/i18n/messages_de.properties b/server/grails-app/i18n/messages_de.properties index 25121ab01..3116d5bd5 100644 --- a/server/grails-app/i18n/messages_de.properties +++ b/server/grails-app/i18n/messages_de.properties @@ -420,4 +420,7 @@ typeMismatch.java.math.BigDecimal=Die Eigenschaft {0} muss eine gültige Zahl se typeMismatch.java.math.BigInteger=Die Eigenschaft {0} muss eine gültige Zahl sein admin.support.sizeLimit.email.subject = 'GOKB - KBART-Import aufgrund von Dateigrößenbeschränkung fehlgeschlagen' -admin.support.sizeLimit.email.line1 = 'Ein KBART-Import für das folgende Paket ist aufgrund der aktiven Größenbeschränkung fehlgeschlagen:' \ No newline at end of file +admin.support.sizeLimit.email.line1 = 'Ein KBART-Import für das folgende Paket ist aufgrund der aktiven Größenbeschränkung fehlgeschlagen:' + +webEndpointUrl.missingProtocol = 'Bitte geben Sie die URL mitsamt Protokoll (z.B. ftp:// oder http://) an.' +webEndpointUrl.notNull = 'Bitte geben Sie eine URL an.' \ No newline at end of file From 8918e3c299717488f377d813bb689c182cc57309 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Fri, 19 Dec 2025 17:01:17 +0100 Subject: [PATCH 18/28] datemask --- .../gokb/rest/WebEndpointController.groovy | 7 +- .../gokb/PackageSourceUpdateService.groovy | 195 ++++++++++-------- .../org/gokb/WebEndpointService.groovy | 9 +- 3 files changed, 112 insertions(+), 99 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index 20c909902..efdb3b665 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -104,9 +104,10 @@ class WebEndpointController { filename = parts.filename } - boolean isDateMasked = (filename =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN) + def dateMaskMatch = (filename =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN) - log.debug("11111: " + filename + ", " + isDateMasked) + log.debug("11111: " + filename + ", " + dateMask) + log.debug("22222: " + dateMask.size() + ", " + dateMask[0] ) FTPClient ftp = new FTPClient() FTPClientConfig config = new FTPClientConfig() @@ -119,7 +120,7 @@ class WebEndpointController { if (ftp.isConnected()) { FTPFile[] files - if(isDateMasked) { + if(dateMaskMatch.size() > 0) { String fixedPart = filename.split("\\{")[0] files = ftp.listFiles(directory) diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 411222fad..72dc60c22 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -6,6 +6,7 @@ import grails.converters.JSON import groovy.util.logging.Slf4j import org.apache.commons.net.ftp.FTPClient import org.apache.commons.net.ftp.FTPClientConfig +import org.apache.commons.net.ftp.FTPFile import org.apache.http.HttpEntity import org.apache.http.HttpHeaders import org.apache.http.util.EntityUtils @@ -27,6 +28,8 @@ import org.gokb.cred.* import org.mozilla.universalchardet.UniversalDetector import org.apache.commons.net.* +import java.util.stream.Collectors + @Slf4j class PackageSourceUpdateService { def concurrencyManagerService @@ -164,105 +167,105 @@ class PackageSourceUpdateService { if ( isFtpTransfer ) { log.debug("Start FTP Update from Source " + pkg_source ) - file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source) + file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source, dynamic_date) } else { // start not-FTP - if (!extracted_date || !lastRunLocal || extracted_date > lastRunLocal) { - log.debug("Request initial URL..") - file_info = fetchKbartFile(tmp_file, src_url, restrictSize) - } + if (!extracted_date || !lastRunLocal || extracted_date > lastRunLocal) { + log.debug("Request initial URL..") + file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + } - if (file_info.connectError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.connection' - result.message = "There was an error trying to fetch KBART via URL!" - result.exceptionMsg = file_info.exceptionMsg + if (file_info.connectError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.connection' + result.message = "There was an error trying to fetch KBART via URL!" + result.exceptionMsg = file_info.exceptionMsg - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - return result - } + return result + } - if (file_info.fileSizeError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.fileSize' - result.message = "The attached KBART file is too big! Files bigger than 20 MB have to be authorized manually by an administrator." + if (file_info.fileSizeError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.fileSize' + result.message = "The attached KBART file is too big! Files bigger than 20 MB have to be authorized manually by an administrator." - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - return result - } - - if (file_info.accessError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.html' - result.message = "URL returned HTML, indicating provider configuration issues!" + return result + } - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + if (file_info.accessError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.html' + result.message = "URL returned HTML, indicating provider configuration issues!" - return result - } else if (file_info.mimeTypeError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.mimeType' - result.message = "KBART URL returned a wrong content type!" - log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + return result + } else if (file_info.mimeTypeError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.mimeType' + result.message = "KBART URL returned a wrong content type!" + log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") - return result - } else if (file_info.status == 403) { - log.debug("URL request failed!") - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.denied' - result.message = "URL request returned 403 ACCESS DENIED, skipping further tries!" + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + return result + } else if (file_info.status == 403) { + log.debug("URL request failed!") + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.denied' + result.message = "URL request returned 403 ACCESS DENIED, skipping further tries!" - return result - } + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - if (!file_info.file_name && (dynamic_date || extracted_date)) { - LocalDate active_date = LocalDate.now() - boolean skipLookupByDate = false - src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) - log.debug("Fetching dated URL for today..") - file_info = fetchKbartFile(tmp_file, src_url, restrictSize) - - // Look at first of this month - if (!file_info.file_name) { - sleep(500) - log.debug("Fetching first of the month..") - def som_date_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.withDayOfMonth(1).toString())) - file_info = fetchKbartFile(tmp_file, som_date_url, restrictSize) + return result } - // Check all days of this month - while (!skipLookupByDate && active_date.isAfter(LocalDate.now().minusDays(30)) && !file_info.file_name) { - active_date = active_date.minusDays(1) + if (!file_info.file_name && (dynamic_date || extracted_date)) { + LocalDate active_date = LocalDate.now() + boolean skipLookupByDate = false src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) - log.debug("Fetching dated URL for date ${active_date}") - sleep(500) + log.debug("Fetching dated URL for today..") file_info = fetchKbartFile(tmp_file, src_url, restrictSize) - if (file_info.mimeTypeError) { - skipLookupByDate = true + // Look at first of this month + if (!file_info.file_name) { + sleep(500) + log.debug("Fetching first of the month..") + def som_date_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.withDayOfMonth(1).toString())) + file_info = fetchKbartFile(tmp_file, som_date_url, restrictSize) + } + + // Check all days of this month + while (!skipLookupByDate && active_date.isAfter(LocalDate.now().minusDays(30)) && !file_info.file_name) { + active_date = active_date.minusDays(1) + src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) + log.debug("Fetching dated URL for date ${active_date}") + sleep(500) + file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + + if (file_info.mimeTypeError) { + skipLookupByDate = true + } } } - } - if (file_info.mimeTypeError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.mimeType' - result.message = "KBART URL returned a wrong content type!" - log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") + if (file_info.mimeTypeError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.mimeType' + result.message = "KBART URL returned a wrong content type!" + log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - return result - } + return result + } - log.debug("Got mime type ${file_info.content_mime_type} for file ${file_info.file_name}") + log.debug("Got mime type ${file_info.content_mime_type} for file ${file_info.file_name}") } // end not-FTP if (file_info.file_name) { @@ -631,7 +634,7 @@ class PackageSourceUpdateService { return info_map } - def fetchKbartFileFromFTPServer (File tmp_file, Source source) { + def fetchKbartFileFromFTPServer (File tmp_file, Source source, boolean dynamic_date) { def result = [content_mime_type: null, file_name: null] @@ -641,37 +644,51 @@ class PackageSourceUpdateService { String username = source.getWebEndpoint().getBa_username() String password = source.getWebEndpoint().getBa_password() - def urlParts = webEndpointService.extractFtpUrlParts(source.getWebEndpoint().getUrl(), source.getFtpUrl()) + def urlParts = webEndpointService.extractFtpUrlParts(source.getWebEndpoint().getUrl(), source.getFtpUrl(), dynamic_date) String hostname = urlParts.hostname String filename = urlParts.filename String directory = urlParts.directory - result.file_name = filename try { - ftp.connect(hostname) - ftp.enterLocalPassiveMode() - def loggedIn = ftp.login(username, password) + ftp.connect(hostname) + ftp.enterLocalPassiveMode() + def loggedIn = ftp.login(username, password) - if (ftp.isConnected()) { + if (ftp.isConnected()) { - ftp.changeWorkingDirectory(directory) + ftp.changeWorkingDirectory(directory) - InputStream is = ftp.retrieveFileStream(filename) - OutputStream outStream = new FileOutputStream(tmp_file) + //TODO: if dynamic_date, find file with latest date - byte[] buffer = new byte[1024]; - for (int length; (length = is.read(buffer)) != -1; ) { - outStream.write(buffer, 0, length); - } + if(dynamic_date) { + String fixFilenamePart = filename.split("\\{")[0] + List files = ftp.listFiles().toList() + List res = files.stream().filter(f -> f.name =~ VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN && f.name.startsWith(fixFilenamePart)).collect(Collectors.toList()).sort() + // file with latest date is the last in list res + if(res.size() > 0) { + filename = res.get(res.size() - 1) + } + } + //TODO: check if filename date is later than last update-date + //TODO: set needed result attributes + result.file_name = filename - outStream.close() + InputStream is = ftp.retrieveFileStream(filename) + OutputStream outStream = new FileOutputStream(tmp_file) - ftp.logout() - ftp.disconnect() + byte[] buffer = new byte[1024]; + for (int length; (length = is.read(buffer)) != -1; ) { + outStream.write(buffer, 0, length); } + outStream.close() + + ftp.logout() + ftp.disconnect() + } + } catch (Exception e) { log.error("Fehler bei FTP-Verbindung ", e) } diff --git a/server/grails-app/services/org/gokb/WebEndpointService.groovy b/server/grails-app/services/org/gokb/WebEndpointService.groovy index f2ba8bc8e..9524dcbad 100644 --- a/server/grails-app/services/org/gokb/WebEndpointService.groovy +++ b/server/grails-app/services/org/gokb/WebEndpointService.groovy @@ -8,16 +8,11 @@ import java.util.regex.Pattern class WebEndpointService { static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ + static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)$/ - def extractFtpUrlParts (String webEndpointUrl, String sourceUrl) { + def extractFtpUrlParts (String webEndpointUrl, String sourceUrl, boolean dynamic_date) { def result = [:] - boolean isDateMasked = false - if (sourceUrl =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN) { - log.debug("### IS MIT DATUMSMASKIERUNG ###") - isDateMasked = true - } - //we dont need the protocol String hostname = webEndpointUrl.replace("ftp://", "") String filename = "" From 2290b1490c0adb39f728da7974d206c837594277 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Thu, 22 Jan 2026 18:48:40 +0100 Subject: [PATCH 19/28] chnages --- .../gokb/rest/WebEndpointController.groovy | 6 ++- .../gokb/PackageSourceUpdateService.groovy | 40 ++++++++++++++----- .../org/gokb/WebEndpointService.groovy | 19 +++++++-- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index efdb3b665..7839ab7aa 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -106,8 +106,8 @@ class WebEndpointController { def dateMaskMatch = (filename =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN) - log.debug("11111: " + filename + ", " + dateMask) - log.debug("22222: " + dateMask.size() + ", " + dateMask[0] ) + log.debug("11111: " + filename + ", " + dateMaskMatch) + FTPClient ftp = new FTPClient() FTPClientConfig config = new FTPClientConfig() @@ -121,7 +121,9 @@ class WebEndpointController { FTPFile[] files if(dateMaskMatch.size() > 0) { + log.debug("22222: " + dateMaskMatch.size() + ", " + dateMaskMatch[0] ) String fixedPart = filename.split("\\{")[0] + log.debug("33333: " + fixedPart ) files = ftp.listFiles(directory) boolean found = false diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 72dc60c22..8dcd4893f 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -40,8 +40,8 @@ class PackageSourceUpdateService { WebEndpointService webEndpointService static Pattern DATE_PLACEHOLDER_PATTERN = ~/[0-9]{4}-[0-9]{2}-[0-9]{2}/ - static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ - static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)$/ + static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)(\?.*)$/ + static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)(\?.*)$/ @javax.annotation.PostConstruct def init() { @@ -81,6 +81,7 @@ class PackageSourceUpdateService { Boolean deleteMissing = false def pkgInfo = [:] def startTime = new Date() + def ftpUrlParts Package.withNewSession { Package p = Package.get(pid) @@ -111,10 +112,19 @@ class PackageSourceUpdateService { def rdv_FTP = RefdataCategory.lookup('Source.TransferMethod', 'FTP') boolean isFtpTransfer = (transferMethod == rdv_FTP) - if (pkg_source?.url || isFtpTransfer) { + if (pkg_source?.url || (isFtpTransfer && pkg_source?.ftpUrl)) { URL src_url = null Boolean dynamic_date = false - def valid_url_string = validationService.checkUrl(pkg_source?.url, true) + String completeFtpUrl = null + if(isFtpTransfer){ + ftpUrlParts = webEndpointService.extractFtpUrlParts(pkg_source.getWebEndpoint()?.getUrl(), pkg_source.getFtpUrl()) + + completeFtpUrl = ftpUrlParts.complete + log.debug("++++ " + completeFtpUrl) + } + def valid_url_string = validationService.checkUrl(isFtpTransfer ? completeFtpUrl : pkg_source?.url, true) + log.debug("##############: " + valid_url_string) + // return LocalDate extracted_date skipInvalid = pkg_source.skipInvalid ?: false def file_info = [:] @@ -139,13 +149,13 @@ class PackageSourceUpdateService { } } - else if (isFtpTransfer) { + /* else if (isFtpTransfer) { // no op here String urlFilePart = pkg_source.getWebEndpoint()?.getUrl() if(urlFilePart =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN){ dynamic_date = true } - } + } */ else { log.debug("No source URL!") result.result = 'ERROR' @@ -166,8 +176,9 @@ class PackageSourceUpdateService { if ( isFtpTransfer ) { log.debug("Start FTP Update from Source " + pkg_source ) - - file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source, dynamic_date) + ftpUrlParts["complete"] = src_url.toString() + log.debug("xxxxxxxx: " + src_url + ", " + ftpUrlParts) + file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source, ftpUrlParts, dynamic_date) } else { // start not-FTP @@ -634,7 +645,7 @@ class PackageSourceUpdateService { return info_map } - def fetchKbartFileFromFTPServer (File tmp_file, Source source, boolean dynamic_date) { + def fetchKbartFileFromFTPServer (File tmp_file, Source source, def urlParts, boolean dynamic_date) { def result = [content_mime_type: null, file_name: null] @@ -644,13 +655,20 @@ class PackageSourceUpdateService { String username = source.getWebEndpoint().getBa_username() String password = source.getWebEndpoint().getBa_password() - def urlParts = webEndpointService.extractFtpUrlParts(source.getWebEndpoint().getUrl(), source.getFtpUrl(), dynamic_date) + // def urlParts = webEndpointService.extractFtpUrlParts(source.getWebEndpoint().getUrl(), source.getFtpUrl(), dynamic_date) String hostname = urlParts.hostname - String filename = urlParts.filename String directory = urlParts.directory + String filename = urlParts.filename + + if(dynamic_date){ + String[] parts = urlParts.complete.split("/") + filename = parts[parts.length - 1] + } + log.debug("+++++ FILENAME: " + filename) + return try { ftp.connect(hostname) ftp.enterLocalPassiveMode() diff --git a/server/grails-app/services/org/gokb/WebEndpointService.groovy b/server/grails-app/services/org/gokb/WebEndpointService.groovy index 9524dcbad..ad7b4b9d6 100644 --- a/server/grails-app/services/org/gokb/WebEndpointService.groovy +++ b/server/grails-app/services/org/gokb/WebEndpointService.groovy @@ -7,16 +7,26 @@ import java.util.regex.Pattern @Transactional class WebEndpointService { - static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ + /*static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)$/ + */ - def extractFtpUrlParts (String webEndpointUrl, String sourceUrl, boolean dynamic_date) { + def extractFtpUrlParts (String webEndpointUrl, String sourceUrl) { def result = [:] + String protocol = "" + if(webEndpointUrl.startsWith("ftp://")){ + protocol = "ftp://" + } + else if(webEndpointUrl.startsWith("ftps://")){ + protocol = "ftps://" + } + //we dont need the protocol - String hostname = webEndpointUrl.replace("ftp://", "") + String hostname = webEndpointUrl.replace(protocol, "") String filename = "" String directory = "/" + String completeUrl = "" if (hostname?.contains("/")) { String[] parts = hostname.split("/") @@ -39,9 +49,12 @@ class WebEndpointService { filename = sourceUrl } + completeUrl = protocol + hostname + directory + filename + result.hostname = hostname result.filename = filename result.directory = directory + result.complete = completeUrl result From 0deed72e6b70a6a92b9a3091d01eea34d988d89e Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Mon, 26 Jan 2026 11:01:01 +0100 Subject: [PATCH 20/28] refactor --- .../gokb/rest/WebEndpointController.groovy | 13 +-- .../gokb/PackageSourceUpdateService.groovy | 89 ++++++++++++------- .../org/gokb/WebEndpointService.groovy | 4 - 3 files changed, 57 insertions(+), 49 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index 7839ab7aa..ca197ab37 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -48,7 +48,6 @@ class WebEndpointController { if (params['method']) { //resultList = resultList.findAll( x -> x.transferMethod?.name == params['method']) - log.debug("1111: " + params['method']) resultList = resultList.findAll( x -> x.url.startsWith(params['method'].toLowerCase()) ) } @@ -98,7 +97,6 @@ class WebEndpointController { if(whe){ def parts= webEndpointService.extractFtpUrlParts(whe.getUrl(), path) - log.debug("*** " + parts.hostname + ", " + parts.directory + ", " + parts.filename) hostname = parts.hostname directory = parts.directory filename = parts.filename @@ -106,9 +104,6 @@ class WebEndpointController { def dateMaskMatch = (filename =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN) - log.debug("11111: " + filename + ", " + dateMaskMatch) - - FTPClient ftp = new FTPClient() FTPClientConfig config = new FTPClientConfig() @@ -121,28 +116,22 @@ class WebEndpointController { FTPFile[] files if(dateMaskMatch.size() > 0) { - log.debug("22222: " + dateMaskMatch.size() + ", " + dateMaskMatch[0] ) String fixedPart = filename.split("\\{")[0] - log.debug("33333: " + fixedPart ) files = ftp.listFiles(directory) boolean found = false for (FTPFile file : files) { - log.debug("+++ " + file.name) - if(file.name.startsWith(fixedPart)){ result.result = "success" - //TODO: message - result.message = "success, file found with name: ${file.name}" + result.message = "dateMaskFound" found = true break } } if(!found){ result.result = "error" - //TODO: message result.message = "dateMaskNotFound" } } diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 8dcd4893f..28ecd975b 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -116,15 +116,14 @@ class PackageSourceUpdateService { URL src_url = null Boolean dynamic_date = false String completeFtpUrl = null + if(isFtpTransfer){ ftpUrlParts = webEndpointService.extractFtpUrlParts(pkg_source.getWebEndpoint()?.getUrl(), pkg_source.getFtpUrl()) - completeFtpUrl = ftpUrlParts.complete - log.debug("++++ " + completeFtpUrl) } + def valid_url_string = validationService.checkUrl(isFtpTransfer ? completeFtpUrl : pkg_source?.url, true) - log.debug("##############: " + valid_url_string) - // return + LocalDate extracted_date skipInvalid = pkg_source.skipInvalid ?: false def file_info = [:] @@ -149,13 +148,6 @@ class PackageSourceUpdateService { } } - /* else if (isFtpTransfer) { - // no op here - String urlFilePart = pkg_source.getWebEndpoint()?.getUrl() - if(urlFilePart =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN){ - dynamic_date = true - } - } */ else { log.debug("No source URL!") result.result = 'ERROR' @@ -177,8 +169,8 @@ class PackageSourceUpdateService { if ( isFtpTransfer ) { log.debug("Start FTP Update from Source " + pkg_source ) ftpUrlParts["complete"] = src_url.toString() - log.debug("xxxxxxxx: " + src_url + ", " + ftpUrlParts) - file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source, ftpUrlParts, dynamic_date) + log.debug("URL: " + src_url + ", " + ftpUrlParts) + file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source, ftpUrlParts, dynamic_date, extracted_date, restrictSize) } else { // start not-FTP @@ -645,9 +637,11 @@ class PackageSourceUpdateService { return info_map } - def fetchKbartFileFromFTPServer (File tmp_file, Source source, def urlParts, boolean dynamic_date) { + def fetchKbartFileFromFTPServer (File tmp_file, Source source, def urlParts, boolean dynamic_date, LocalDate extracted_date, boolean restrictSize = true) { def result = [content_mime_type: null, file_name: null] + Long max_length = 20971520L // 1024 * 1024 * 20 + LocalDate lastRunLocal = source.lastRun ? source.lastRun.toInstant().atZone(ZoneId.systemDefault()).toLocalDate() : null FTPClient ftp = new FTPClient() FTPClientConfig config = new FTPClientConfig() @@ -655,20 +649,18 @@ class PackageSourceUpdateService { String username = source.getWebEndpoint().getBa_username() String password = source.getWebEndpoint().getBa_password() - // def urlParts = webEndpointService.extractFtpUrlParts(source.getWebEndpoint().getUrl(), source.getFtpUrl(), dynamic_date) - String hostname = urlParts.hostname String directory = urlParts.directory String filename = urlParts.filename + String foundFileName = null + FTPFile foundFile = null if(dynamic_date){ + // in case of dynamic_date, in the filename the pattern is already replaced by the actual date String[] parts = urlParts.complete.split("/") filename = parts[parts.length - 1] } - log.debug("+++++ FILENAME: " + filename) - - return try { ftp.connect(hostname) ftp.enterLocalPassiveMode() @@ -677,31 +669,61 @@ class PackageSourceUpdateService { if (ftp.isConnected()) { ftp.changeWorkingDirectory(directory) + List files - //TODO: if dynamic_date, find file with latest date - - if(dynamic_date) { + if(dynamic_date || extracted_date) { String fixFilenamePart = filename.split("\\{")[0] - List files = ftp.listFiles().toList() + files = ftp.listFiles().toList() List res = files.stream().filter(f -> f.name =~ VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN && f.name.startsWith(fixFilenamePart)).collect(Collectors.toList()).sort() // file with latest date is the last in list res if(res.size() > 0) { - filename = res.get(res.size() - 1) + foundFile = res.get(res.size() - 1) + foundFileName = foundFile.name } } - //TODO: check if filename date is later than last update-date - //TODO: set needed result attributes - result.file_name = filename + else { + files = ftp.listFiles(filename).toList() + if(files.size() > 0) { + foundFile = files.get(0) + foundFileName = filename + } + } + + + if(foundFile){ + LocalDate foundFileDate = LocalDate.ofInstant(foundFile.getTimestampInstant(), ZoneId.systemDefault()) + if(lastRunLocal && (lastRunLocal > foundFileDate)){ + // no update needed + return result + } + + Long foundFileSize = foundFile.getSize() + if(foundFileSize > max_length && restrictSize){ + result.fileSizeError = true + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.fileSize' + result.message = "The attached KBART file is too big! Files bigger than 20 MB have to be authorized manually by an administrator." + //result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + tmp_file.delete() + return result + } + + result.file_name = foundFileName - InputStream is = ftp.retrieveFileStream(filename) - OutputStream outStream = new FileOutputStream(tmp_file) + InputStream is = ftp.retrieveFileStream(foundFileName) + OutputStream outStream = new FileOutputStream(tmp_file) - byte[] buffer = new byte[1024]; - for (int length; (length = is.read(buffer)) != -1; ) { + byte[] buffer = new byte[1024]; + for (int length; (length = is.read(buffer)) != -1; ) { outStream.write(buffer, 0, length); - } + } - outStream.close() + outStream.close() + + } + else { + // no filename --> handled in calling method + } ftp.logout() ftp.disconnect() @@ -709,6 +731,7 @@ class PackageSourceUpdateService { } catch (Exception e) { log.error("Fehler bei FTP-Verbindung ", e) + } return result diff --git a/server/grails-app/services/org/gokb/WebEndpointService.groovy b/server/grails-app/services/org/gokb/WebEndpointService.groovy index ad7b4b9d6..35e9bced0 100644 --- a/server/grails-app/services/org/gokb/WebEndpointService.groovy +++ b/server/grails-app/services/org/gokb/WebEndpointService.groovy @@ -7,10 +7,6 @@ import java.util.regex.Pattern @Transactional class WebEndpointService { - /*static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ - static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)$/ - */ - def extractFtpUrlParts (String webEndpointUrl, String sourceUrl) { def result = [:] From 44571cc15e2abf35e7b58bbf67fde9268c52a5ac Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Mon, 26 Jan 2026 11:10:39 +0100 Subject: [PATCH 21/28] refactor --- .../services/org/gokb/PackageSourceUpdateService.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 28ecd975b..4ae9db9ed 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -694,6 +694,7 @@ class PackageSourceUpdateService { LocalDate foundFileDate = LocalDate.ofInstant(foundFile.getTimestampInstant(), ZoneId.systemDefault()) if(lastRunLocal && (lastRunLocal > foundFileDate)){ // no update needed + tmp_file.delete() return result } From 4f6ee754cd456b0a0f3ebf5e8a9ca735e71b5a2c Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Wed, 28 Jan 2026 18:15:46 +0100 Subject: [PATCH 22/28] mock ftp --- server/build.gradle | 2 + .../org/gokb/WebEndpointService.groovy | 2 + .../org/gokb/UpdatePackageRunFTPSpec.groovy | 119 ++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy diff --git a/server/build.gradle b/server/build.gradle index d96ecbcbf..388a99dc6 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -139,6 +139,8 @@ dependencies { testImplementation "org.seleniumhq.selenium:selenium-remote-driver:$seleniumVersion" testImplementation "org.seleniumhq.selenium:selenium-api:$seleniumVersion" testImplementation "org.seleniumhq.selenium:selenium-support:$seleniumVersion" + + testImplementation 'org.mockftpserver:MockFtpServer:3.2.0' } bootRun { diff --git a/server/grails-app/services/org/gokb/WebEndpointService.groovy b/server/grails-app/services/org/gokb/WebEndpointService.groovy index 35e9bced0..39c9ce69e 100644 --- a/server/grails-app/services/org/gokb/WebEndpointService.groovy +++ b/server/grails-app/services/org/gokb/WebEndpointService.groovy @@ -24,6 +24,8 @@ class WebEndpointService { String directory = "/" String completeUrl = "" + // leading and trailing slashes that could be part of host or filename + // are set in the directory part if (hostname?.contains("/")) { String[] parts = hostname.split("/") hostname = parts[0] diff --git a/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy new file mode 100644 index 000000000..5f9154c4b --- /dev/null +++ b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy @@ -0,0 +1,119 @@ +package org.gokb + +import grails.core.GrailsApplication +import grails.gorm.transactions.Rollback +import grails.testing.mixin.integration.Integration +import io.micronaut.http.client.BlockingHttpClient +import io.micronaut.http.client.HttpClient +import org.apache.commons.net.ftp.FTPClient +import org.gokb.cred.CuratoryGroup +import org.gokb.cred.KBComponent +import org.gokb.cred.Org +import org.gokb.cred.Package +import org.gokb.cred.RefdataCategory +import org.gokb.cred.Source +import org.gokb.cred.WebHookEndpoint +import org.mockftpserver.fake.* +import org.mockftpserver.fake.filesystem.* +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.context.WebApplicationContext +import spock.lang.Specification + +@Integration +@Rollback +class UpdatePackageRunFTPSpec extends Specification{ + + GrailsApplication grailsApplication + + @Autowired + WebApplicationContext ctx + + BlockingHttpClient http + + private static final String HOME_DIR = "/"; + private static final String FILE = "/dir/sample.txt"; + private static final String CONTENTS = "abcdef 1234567890"; + private static final String USER = "user"; + private static final String PASSWORD = "password"; + private FakeFtpServer ftpServer + + private String getUrlPath() { + return "http://localhost:${serverPort}${grailsApplication.config.getProperty('server.servlet.context-path') ?: ''}".toString() + } + + private void setupMockFtpBase(){ + ftpServer = new FakeFtpServer(); + ftpServer.setServerControlPort(0); // use any free port + FileSystem fileSystem = new UnixFakeFileSystem(); + fileSystem.add(new FileEntry(FILE, CONTENTS)); + ftpServer.setFileSystem(fileSystem); + ftpServer.addUserAccount(new UserAccount(USER, PASSWORD, HOME_DIR)) + ftpServer.start(); + } + + void setUpBasic(){ + + } + + def setup() { + this.setupMockFtpBase() + + if (!http) { + http = HttpClient.create(new URL(getUrlPath())).toBlocking() + } + + def new_cg = CuratoryGroup.findByName('TestGroup1') ?: new CuratoryGroup(name: "TestGroup1").save(flush: true) + def acs_org = Org.findByName("American Chemical Society") ?: new Org(name: "American Chemical Society").save(flush: true) + + WebHookEndpoint whe = new WebHookEndpoint(name: "MOCK FTP on Localhost" ,url: "ftp://localhost", ba_username: USER, ba_password: PASSWORD).save(flush: true) + Source mock_src = new Source(webEndpoint: whe, ftpUrl: FILE, transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) + // TODO: Pflichtfelder + Package pckg = new Package(name: "Update Package", source: mock_src).save(flush: true) + + } + + def cleanup() { + ftpServer.stop() + } + + void "test WebEndpointService extractUrlParts :: split parts and format them correct"() { + + when: "The Slashes are not set correctly in URL Field" + String endpointUrl = "ftp://servername.com/home/" + String sourceUrl = "/filename.txt" + + def res = new WebEndpointService().extractFtpUrlParts(endpointUrl, sourceUrl) + + + then: "The URL is normalized and splitted correctly into parts" + + res.hostname == "servername.com" + res.directory == "/home/" + res.filename == "filename.txt" + res.complete == "ftp://servername.com/home/filename.txt" + + } + + + + void "test Mock FTP Server runs"() { + + FTPClient ftpClient = new FTPClient() + ftpClient.connect("localhost", ftpServer.getServerControlPort()) + ftpClient.login(USER, PASSWORD); + + expect: "Directory exists" + ftpClient.listDirectories().length > 0 + } + + + /* void "Test updateFromSource :: "() { + + given: + + } */ + + + + +} From 7ebb8ceb2466a3eb37b9c0c13ca5f4e62ce03b4f Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Thu, 29 Jan 2026 18:50:11 +0100 Subject: [PATCH 23/28] test --- .../org/gokb/UpdatePackageRunFTPSpec.groovy | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy index 5f9154c4b..cce207449 100644 --- a/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy +++ b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy @@ -1,5 +1,6 @@ package org.gokb +import com.k_int.ConcurrencyManagerService import grails.core.GrailsApplication import grails.gorm.transactions.Rollback import grails.testing.mixin.integration.Integration @@ -16,10 +17,12 @@ import org.gokb.cred.WebHookEndpoint import org.mockftpserver.fake.* import org.mockftpserver.fake.filesystem.* import org.springframework.beans.factory.annotation.Autowired +import org.springframework.transaction.annotation.Transactional import org.springframework.web.context.WebApplicationContext import spock.lang.Specification @Integration +// @Transactional @Rollback class UpdatePackageRunFTPSpec extends Specification{ @@ -28,6 +31,9 @@ class UpdatePackageRunFTPSpec extends Specification{ @Autowired WebApplicationContext ctx + + def concurrencyManagerService + BlockingHttpClient http private static final String HOME_DIR = "/"; @@ -68,7 +74,23 @@ class UpdatePackageRunFTPSpec extends Specification{ WebHookEndpoint whe = new WebHookEndpoint(name: "MOCK FTP on Localhost" ,url: "ftp://localhost", ba_username: USER, ba_password: PASSWORD).save(flush: true) Source mock_src = new Source(webEndpoint: whe, ftpUrl: FILE, transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) // TODO: Pflichtfelder - Package pckg = new Package(name: "Update Package", source: mock_src).save(flush: true) + Package pckg = Package.findByName("Update Package") ?: new Package(name: "Update Package").save(flush: true) + pckg.setSource(mock_src) + pckg.save(flush: true) + + PackageSourceUpdateService service = new PackageSourceUpdateService() + + System.out.println("111111: " + pckg.lastUpdated) + + ConcurrencyManagerService.Job pkg_job = concurrencyManagerService.createJob { pjob -> + service.updateFromSource(pckg.id, null, pkg_job, new_cg.id, false, true) + } + + System.out.println("222222: " + pckg.lastUpdated) + + sleep(15000) + + System.out.println("333333: " + pckg.lastUpdated) } From 3e0d6648e0690a2af92398f85e3c65dbd0e8cb19 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Fri, 30 Jan 2026 17:46:20 +0100 Subject: [PATCH 24/28] test ftp --- .../gokb/PackageSourceUpdateService.groovy | 7 +- .../org/gokb/ValidationService.groovy | 2 +- .../org/gokb/UpdatePackageRunFTPSpec.groovy | 132 +++++++++++++----- .../resources/test_ftp_kbart_update.txt | 71 ++++++++++ 4 files changed, 173 insertions(+), 39 deletions(-) create mode 100644 server/src/integration-test/resources/test_ftp_kbart_update.txt diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 4ae9db9ed..92100f24b 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -50,8 +50,9 @@ class PackageSourceUpdateService { def updateFromSource(Long pkgId, def user = null, Job job = null, Long activeGroupId = null, boolean dryRun = false, boolean restrictSize = true) { log.debug("updateFromSource ${pkgId}") + log.info("111111111111111111111111111111111111111111111111111111111111111111111111111111111") def result = [result: 'OK'] - def activeJobs = concurrencyManagerService.getComponentJobs(pkgId) + def activeJobs = concurrencyManagerService?.getComponentJobs(pkgId) if (job || activeJobs?.data?.size() == 0) { log.debug("UpdateFromSource started") @@ -70,6 +71,7 @@ class PackageSourceUpdateService { private def startSourceUpdate(pid, user, job, activeGroupId, dryRun, restrictSize) { log.debug("Source update start..") + log.info("22222222222222222222222222222222222222222222222222222222222222222222") def result = [result: 'OK', dryRun: dryRun] Boolean async = (user ? true : false) def preferred_group @@ -120,10 +122,11 @@ class PackageSourceUpdateService { if(isFtpTransfer){ ftpUrlParts = webEndpointService.extractFtpUrlParts(pkg_source.getWebEndpoint()?.getUrl(), pkg_source.getFtpUrl()) completeFtpUrl = ftpUrlParts.complete + log.debug("xxxxxxxx: " + ftpUrlParts) } def valid_url_string = validationService.checkUrl(isFtpTransfer ? completeFtpUrl : pkg_source?.url, true) - + log.debug("yyyyyyy: " + valid_url_string) LocalDate extracted_date skipInvalid = pkg_source.skipInvalid ?: false def file_info = [:] diff --git a/server/grails-app/services/org/gokb/ValidationService.groovy b/server/grails-app/services/org/gokb/ValidationService.groovy index 310da94c2..ccd528eb8 100644 --- a/server/grails-app/services/org/gokb/ValidationService.groovy +++ b/server/grails-app/services/org/gokb/ValidationService.groovy @@ -925,7 +925,7 @@ class ValidationService { // log.debug("Final URL to check: ${final_val}") - return new UrlValidator().isValid(final_val) ? value : null + return new UrlValidator(UrlValidator.ALLOW_LOCAL_URLS).isValid(final_val) ? value : null } private String encodeUrlPart(String value) { diff --git a/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy index cce207449..bf551687b 100644 --- a/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy +++ b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy @@ -6,96 +6,146 @@ import grails.gorm.transactions.Rollback import grails.testing.mixin.integration.Integration import io.micronaut.http.client.BlockingHttpClient import io.micronaut.http.client.HttpClient +import org.apache.commons.io.IOUtils import org.apache.commons.net.ftp.FTPClient +import org.apache.commons.net.ftp.FTPFile import org.gokb.cred.CuratoryGroup import org.gokb.cred.KBComponent import org.gokb.cred.Org import org.gokb.cred.Package +import org.gokb.cred.Platform import org.gokb.cred.RefdataCategory import org.gokb.cred.Source +import org.gokb.cred.User import org.gokb.cred.WebHookEndpoint import org.mockftpserver.fake.* import org.mockftpserver.fake.filesystem.* import org.springframework.beans.factory.annotation.Autowired +import org.springframework.core.io.ClassPathResource import org.springframework.transaction.annotation.Transactional import org.springframework.web.context.WebApplicationContext import spock.lang.Specification +import java.nio.charset.Charset + @Integration // @Transactional @Rollback class UpdatePackageRunFTPSpec extends Specification{ GrailsApplication grailsApplication + ConcurrencyManagerService concurrencyManagerService + PackageSourceUpdateService packageSourceUpdateService @Autowired WebApplicationContext ctx - - def concurrencyManagerService - - BlockingHttpClient http - private static final String HOME_DIR = "/"; private static final String FILE = "/dir/sample.txt"; private static final String CONTENTS = "abcdef 1234567890"; private static final String USER = "user"; private static final String PASSWORD = "password"; private FakeFtpServer ftpServer + private Package packageUnderInspection + private CuratoryGroup new_cg + private Org provider + private WebHookEndpoint whe + private Source source + private Platform platform + private String getUrlPath() { return "http://localhost:${serverPort}${grailsApplication.config.getProperty('server.servlet.context-path') ?: ''}".toString() } private void setupMockFtpBase(){ - ftpServer = new FakeFtpServer(); - ftpServer.setServerControlPort(0); // use any free port - FileSystem fileSystem = new UnixFakeFileSystem(); - fileSystem.add(new FileEntry(FILE, CONTENTS)); - ftpServer.setFileSystem(fileSystem); + ftpServer = new FakeFtpServer() + ftpServer.setServerControlPort(21) + FileSystem fileSystem = new UnixFakeFileSystem() + fileSystem.add(new FileEntry(FILE, CONTENTS)) + ftpServer.setFileSystem(fileSystem) ftpServer.addUserAccount(new UserAccount(USER, PASSWORD, HOME_DIR)) - ftpServer.start(); - } - void setUpBasic(){ + ftpServer.start() + } + void setUpKbartExistsNormalName(){ + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart.txt", fileContent)) + } } def setup() { this.setupMockFtpBase() - if (!http) { - http = HttpClient.create(new URL(getUrlPath())).toBlocking() - } + new_cg = CuratoryGroup.findByName('TestGroup1') ?: new CuratoryGroup(name: "TestGroup1").save(flush: true) - def new_cg = CuratoryGroup.findByName('TestGroup1') ?: new CuratoryGroup(name: "TestGroup1").save(flush: true) - def acs_org = Org.findByName("American Chemical Society") ?: new Org(name: "American Chemical Society").save(flush: true) + provider = Org.findByName("American Chemical Society") ?: new Org(name: "American Chemical Society").save(flush: true) + platform = Platform.findByName("Test Platform") ?: new Platform(name: "Test Platform", primaryUrl: "https://search.ebscohost.com", provider: provider).save(flush: true) + whe = WebHookEndpoint.findByName("MOCK FTP on Localhost") ?: new WebHookEndpoint(name: "MOCK FTP on Localhost" ,url: "ftp://localhost", ba_username: USER, ba_password: PASSWORD).save(flush: true) + source = Source.findByName("Test Source") ?: new Source(name: "Test Source", webEndpoint: whe, ftpUrl: "/dir/kbart.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) - WebHookEndpoint whe = new WebHookEndpoint(name: "MOCK FTP on Localhost" ,url: "ftp://localhost", ba_username: USER, ba_password: PASSWORD).save(flush: true) - Source mock_src = new Source(webEndpoint: whe, ftpUrl: FILE, transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) // TODO: Pflichtfelder - Package pckg = Package.findByName("Update Package") ?: new Package(name: "Update Package").save(flush: true) - pckg.setSource(mock_src) - pckg.save(flush: true) + packageUnderInspection = Package.findByName("Update Package") ?: new Package(name: "Update Package").save(flush: true) + packageUnderInspection.setSource(source) + + packageUnderInspection.save(flush: true) + + + } + + def cleanup() { + ftpServer.stop() + CuratoryGroup.findByName('TestGroup1')?.expunge() + Org.findByName("American Chemical Society")?.expunge() + Platform.findByName("Test Platform")?.expunge() + + //Source.findByName("Test Source")?.expunge() + //whe.delete() + Package.findByName("Update Package")?.expunge() + //TODO: von KBComponent ableiten??? + //WebHookEndpoint.findByName("MOCK FTP on Localhost")?.expunge() - PackageSourceUpdateService service = new PackageSourceUpdateService() + } + + + void "Test updateFromSource :: usual initial Import by filename"() { + + given: "asked Kbart exists without date mask" + setUpKbartExistsNormalName() + + def user = User.findByUsername('admin') - System.out.println("111111: " + pckg.lastUpdated) + /* PackageSourceUpdateService service = new PackageSourceUpdateService() ConcurrencyManagerService.Job pkg_job = concurrencyManagerService.createJob { pjob -> - service.updateFromSource(pckg.id, null, pkg_job, new_cg.id, false, true) + service.updateFromSource(packageUnderInspection.id, null, pkg_job, new_cg.id, false, true) } + Package.withNewSession { + pkg_job.startOrQueue() + } + def job_result = pkg_job.get() + */ - System.out.println("222222: " + pckg.lastUpdated) + /* ConcurrencyManagerService.Job pkg_job = concurrencyManagerService.createJob { pjob -> + packageSourceUpdateService.updateFromSource(packageUnderInspection.id, null, pjob, new_cg.id, false, true) + } - sleep(15000) + pkg_job.startOrQueue() + def job_result = pkg_job.get() + */ - System.out.println("333333: " + pckg.lastUpdated) + def result = packageSourceUpdateService.updateFromSource(packageUnderInspection.id, null, null, new_cg.id, false, true) + + System.out.println("222222222222222222: " + result) + expect: "file is found, package titles imported" + def titles = packageUnderInspection.getTitles(true, 100, 0) + titles != null + System.out.println("######### " + titles) - } - def cleanup() { - ftpServer.stop() } void "test WebEndpointService extractUrlParts :: split parts and format them correct"() { @@ -126,14 +176,24 @@ class UpdatePackageRunFTPSpec extends Specification{ expect: "Directory exists" ftpClient.listDirectories().length > 0 - } + + /* and: "find file and read its contents" + ftpClient.changeWorkingDirectory("dir") + def files = ftpClient.listFiles() + System.out.println("+++ " + files) + + String fileContent = "" + try(InputStream is = ftpClient.retrieveFileStream("kbart.txt")){ + fileContent = IOUtils.toString(is, Charset.defaultCharset()) + } + */ + //System.out.println(fileContent) - /* void "Test updateFromSource :: "() { - given: + } + - } */ diff --git a/server/src/integration-test/resources/test_ftp_kbart_update.txt b/server/src/integration-test/resources/test_ftp_kbart_update.txt new file mode 100644 index 000000000..922c44f8d --- /dev/null +++ b/server/src/integration-test/resources/test_ftp_kbart_update.txt @@ -0,0 +1,71 @@ +publication_title print_identifier online_identifier date_first_issue_online num_first_vol_online num_first_issue_online date_last_issue_online num_last_vol_online num_last_issue_online title_url first_author title_id embargo_info coverage_depth notes publisher_name publication_type date_monograph_published_print date_monograph_published_online monograph_volume monograph_edition first_editor parent_publication_title_id preceding_publication_title_id access_type package_name package_id ebsco_resource_type +(parenthetical) 2368-0202 2016-05-01 2017-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JURX&scope=site JURX fulltext words(on)pages serial P Canadian Literary Centre cjh Periodical +Acta Victoriana 0700-8406 2017-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUQY&scope=site JUQY fulltext Acta Victoriana serial P Canadian Literary Centre cjh Periodical +Antigonish Review 0003-5661 2004-04-01 2021-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=FVI&scope=site FVI abstracts Antigonish Review serial P Canadian Literary Centre cjh Periodical +Antigonish Review 0003-5661 2005-04-01 2010-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=FVI&scope=site FVI fulltext Antigonish Review serial P Canadian Literary Centre cjh Periodical +Arc Poetry Magazine 1910-3239 2008-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BGAO&scope=site BGAO fulltext ARC: Canada's National Poetry Magazine serial P Canadian Literary Centre cjh Periodical +Books in Canada 0045-2564 1971-05-01 2008-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=4TI&scope=site 4TI fulltext Canadian Review of Books Ltd. serial P Canadian Literary Centre cjh Periodical +Brick: A Literary Journal 0382-8565 2818-5390 2011-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BCPO&scope=site BCPO fulltext Brick: A Literary Journal serial P Canadian Literary Centre cjh Periodical +Broken Pencil 1201-8996 2012-01-01 2024-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=90TX&scope=site 90TX fulltext Broken Pencil serial P Canadian Literary Centre cjh Periodical +Canadian Journal of Film & Media Studies 2819-4748 2819-4756 2025-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=NV4N&scope=site NV4N fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +Canadian Journal of Film Studies 0847-5911 2003-09-01 2024-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I2G&scope=site I2G abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +Canadian Journal of Film Studies 0847-5911 2006-05-01 2024-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I2G&scope=site I2G fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +Canadian Literature 0008-4360 1992-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=CLI&scope=site CLI abstracts Canadian Literature serial P Canadian Literary Centre cjh Academic Journal +Canadian Literature 0008-4360 2003-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=CLI&scope=site CLI fulltext Canadian Literature serial P Canadian Literary Centre cjh Academic Journal +Canadian Modern Language Review 0008-4506 1710-1131 1997-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=7RY&scope=site 7RY abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +Canadian Publishers Directory 0008-4859 2009-01-01 2010-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=47CE&scope=site 47CE fulltext St. Joseph Media serial P Canadian Literary Centre cjh Periodical +Canadian Theatre Review 0315-0836 1920-941X 1974-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=6FB&scope=site 6FB abstracts University of Toronto Press serial P Canadian Literary Centre cjh Periodical +Canadian Theatre Review 0315-0836 1920-941X 1997-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=6FB&scope=site 6FB fulltext University of Toronto Press serial P Canadian Literary Centre cjh Periodical +Capilano Review 0315-3754 2003-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UJO&scope=site UJO fulltext Capilano Review serial P Canadian Literary Centre cjh Periodical +Cinephile 1712-9265 2012-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=EV4A&scope=site EV4A fulltext University of British Columbia, Department of Theatre & Film serial P Canadian Literary Centre cjh Academic Journal +CNQ: Canadian Notes & Queries 0576-5803 2016-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=11WQ&scope=site 11WQ fulltext CNQ: Canadian Notes & Queries serial P Canadian Literary Centre cjh Periodical +Contemporary Verse 2 0831-9502 2016-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BCPU&scope=site BCPU fulltext Contemporary Verse 2 serial P Canadian Literary Centre cjh Periodical +Dalhousie Review 0011-5827 1987-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=DHR&scope=site DHR abstracts Dalhousie Review serial P Canadian Literary Centre cjh Academic Journal +Dalhousie Review 0011-5827 2010-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=DHR&scope=site DHR fulltext Dalhousie Review serial P Canadian Literary Centre cjh Academic Journal +Écrits 1200-7935 2015-11-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=FXMB&scope=site FXMB fulltext Academie des Lettres du Quebec serial P Canadian Literary Centre cjh Periodical +Essays on Canadian Writing 0316-0300 1974-12-01 2009-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=ECW&scope=site ECW abstracts ECW Press Ltd. serial P Canadian Literary Centre cjh Academic Journal +Essays on Canadian Writing 0316-0300 1977-03-01 2009-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=ECW&scope=site ECW fulltext ECW Press Ltd. serial P Canadian Literary Centre cjh Academic Journal +Event (0315-3770) 0315-3770 2003-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UL2&scope=site UL2 fulltext Douglas College - Creative Writing Dept. serial P Canadian Literary Centre cjh Periodical +Existere 1710-386X 2021-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUR4&scope=site JUR4 fulltext Professional Writing Program, York University Writing Department serial P Canadian Literary Centre cjh Periodical +Fiddlehead 0015-0630 2003-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=ULM&scope=site ULM abstracts University of New Brunswick serial P Canadian Literary Centre cjh Periodical +Filling Station 1198-0060 2019-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BCPQ&scope=site BCPQ fulltext Filling Station Publications serial P Canadian Literary Centre cjh Periodical +Glass Buffalo 1929-8587 2015-07-01 2019-10-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JESI&scope=site JESI fulltext Matthew Stepanic serial P Canadian Literary Centre cjh Periodical +Grain Magazine 1491-0497 2004-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UX0&scope=site UX0 abstracts Saskatchewan Writers Guild serial P Canadian Literary Centre cjh Periodical +Grain Magazine 1491-0497 2007-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UX0&scope=site UX0 fulltext Saskatchewan Writers Guild serial P Canadian Literary Centre cjh Periodical +Letters in Canada 0315-4955 1993-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=J0I&scope=site J0I abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +Letters in Canada 0315-4955 1994-12-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=J0I&scope=site J0I fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +Literary Review of Canada 1188-7494 2020-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=12DW&scope=site 12DW fulltext Literary Review of Canada serial P Canadian Literary Centre cjh Periodical +Lurelu 0705-6567 1923-2330 2023-10-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BFFG&scope=site BFFG fulltext Lurelu serial P Canadian Literary Centre cjh Periodical +MacroMicroCosm Literary & Art Journal 2368-9781 2016-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JURE&scope=site JURE fulltext Vraeyda Media serial P Canadian Literary Centre cjh Periodical +Malahat Review 1923-7502 2004-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UM1&scope=site UM1 abstracts Malahat Review serial P Canadian Literary Centre cjh Periodical +Maple Tree Literary Supplement, MTLS 1916-341X 2017-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=L03R&scope=site L03R fulltext Maple Tree Literary Supplement, MTLS, Inc. serial P Canadian Literary Centre cjh Academic Journal +Margaret Atwood's Alias Grace: A Reader's Guide 978-0-8264-5706-6 https://search.ebscohost.com/direct.asp?db=cjh&jid=1GBQ&scope=site 1GBQ fulltext Bloomsbury Publishing Plc monograph 2002 2002 P Canadian Literary Centre cjh Book +Modern Drama 0026-7694 1712-5286 1965-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=MOD&scope=site MOD abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +Modern Drama 0026-7694 1712-5286 1997-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=MOD&scope=site MOD fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +Montreal Review 1920-2911 2016-09-01 2019-02-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=KP08&scope=site KP08 fulltext Tsonchev Publishing & Design serial P Canadian Literary Centre cjh Academic Journal +Mouvances Francophones 2371-7211 2023-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=N9P1&scope=site N9P1 fulltext Western University, Department of French Studies serial P Canadian Literary Centre cjh Academic Journal +Nashwaak Review 1205-7681 2009-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=90AD&scope=site 90AD fulltext Nashwaak Review serial P Canadian Literary Centre cjh Academic Journal +New Quarterly 0227-0455 2017-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BCPS&scope=site BCPS fulltext New Quarterly Literary Society Inc. serial P Canadian Literary Centre cjh Periodical +Nouveau Projet 1927-8039 2016-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JURI&scope=site JURI fulltext Atelier 10 serial P Canadian Literary Centre cjh Periodical +On Spec: The Canadian Magazine of the Fantastic 0843-476X 2018-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=LB2T&scope=site LB2T fulltext Copper Pig Writers' Society serial P Canadian Literary Centre cjh Periodical +Passages North 0278-0828 2002-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UOM&scope=site UOM abstracts Passages North serial P Canadian Literary Centre cjh Periodical +Prairie Fire 0821-1124 2016-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=3QXT&scope=site 3QXT fulltext Prairie Fire Press, Inc. serial P Canadian Literary Centre cjh Periodical +Prairie Journal of Canadian Literature 0827-2921 2002-06-01 2020-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=E01&scope=site E01 abstracts Prairie Journal Trust serial P Canadian Literary Centre cjh Academic Journal +Queen's Quarterly 0033-6041 2008-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=7JU&scope=site 7JU fulltext Queens University serial P Canadian Literary Centre cjh Academic Journal +Quill & Quire 0033-6491 1996-01-01 2020-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=01N&scope=site 01N fulltext St. Joseph Media serial P Canadian Literary Centre cjh Periodical +Quill & Quire 0033-6491 1996-01-01 2022-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=01N&scope=site 01N abstracts St. Joseph Media serial P Canadian Literary Centre cjh Periodical +QWERTY 1203-8768 2018-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUS1&scope=site JUS1 fulltext Qwerty serial P Canadian Literary Centre cjh Periodical +Reappraisals: Canadian Writers 1189-6787 2017-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=LEX3&scope=site LEX3 fulltext University of Ottawa Press serial P Canadian Literary Centre cjh Academic Journal +Rhetor. Journal of the Canadian Society for the Study of Rhetoric 1712-2333 2004-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=1G8D&scope=site 1G8D fulltext Canadian Society for the Study of Rhetoric serial P Canadian Literary Centre cjh Academic Journal +Room Magazine 1914-4083 2013-05-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=46AO&scope=site 46AO fulltext Growing Room Collective serial P Canadian Literary Centre cjh Periodical +Site Magazine 2369-9566 2018-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=LS18&scope=site LS18 fulltext Site Magazine serial P Canadian Literary Centre cjh Periodical +Solaris 0709-8863 2021-10-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=LNR6&scope=site LNR6 fulltext Solaris serial P Canadian Literary Centre cjh Periodical +Studies in Canadian Literature / Études en Littérature Canadienne 0380-6995 1718-7850 2003-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=4TM&scope=site 4TM abstracts Studies in Canadian Literature / Etudes en Litterature Canadienne serial P Canadian Literary Centre cjh Academic Journal +Sub-Terrain 0840-7533 2003-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I3B&scope=site I3B abstracts SubTerrain Magazine serial P Canadian Literary Centre cjh Periodical +Sub-Terrain 0840-7533 2014-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I3B&scope=site I3B fulltext SubTerrain Magazine serial P Canadian Literary Centre cjh Periodical +Taddle Creek 1480-2481 1710-8632 2016-12-01 2022-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUS5&scope=site JUS5 fulltext Vitalis Publishing serial P Canadian Literary Centre cjh Periodical +Textual Studies in Canada 1183-854X 1991-09-01 2004-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=4TN&scope=site 4TN fulltext Textual Studies in Canada serial P Canadian Literary Centre cjh Academic Journal +Theatre Research in Canada 1196-1198 1913-9101 2002-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I2T&scope=site I2T abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +University of Toronto Quarterly 0042-0247 1712-5278 1974-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UTQ&scope=site UTQ abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +University of Toronto Quarterly 0042-0247 1712-5278 1975-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UTQ&scope=site UTQ fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal +Vallum: Contemporary Poetry 1496-5178 2016-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUS6&scope=site JUS6 fulltext Joshua Auerbach serial P Canadian Literary Centre cjh Periodical From 93759a8704a6c5bb64d76c7c844a117820efc383 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Wed, 4 Feb 2026 15:14:20 +0100 Subject: [PATCH 25/28] integration tests --- .../gokb/rest/WebEndpointController.groovy | 3 - .../gokb/PackageSourceUpdateService.groovy | 37 ++- .../org/gokb/ValidationService.groovy | 9 +- .../org/gokb/UpdatePackageRunFTPSpec.groovy | 274 +++++++++++------- .../resources/test_ftp_kbart_update.txt | 68 ----- .../test_ftp_kbart_update_new_file.txt | 4 + 6 files changed, 207 insertions(+), 188 deletions(-) create mode 100644 server/src/integration-test/resources/test_ftp_kbart_update_new_file.txt diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index ca197ab37..93de68e03 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -33,13 +33,10 @@ class WebEndpointController { if (springSecurityService.isLoggedIn()) { user = User.get(springSecurityService.principal?.id) } - def start_db = LocalDateTime.now() - params['_embed'] = params['_embed'] ?: 'identifiedComponents' result = componentLookupService.restLookup(user, WebHookEndpoint, params) - //log.debug("DB duration: ${Duration.between(start_db, LocalDateTime.now()).toMillis();}") if (result.data) { def resultList = result.data diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 92100f24b..49a40c086 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -3,6 +3,7 @@ package org.gokb import com.k_int.ConcurrencyManagerService.Job import grails.converters.JSON +import grails.util.Environment import groovy.util.logging.Slf4j import org.apache.commons.net.ftp.FTPClient import org.apache.commons.net.ftp.FTPClientConfig @@ -40,8 +41,9 @@ class PackageSourceUpdateService { WebEndpointService webEndpointService static Pattern DATE_PLACEHOLDER_PATTERN = ~/[0-9]{4}-[0-9]{2}-[0-9]{2}/ - static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)(\?.*)$/ - static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)(\?.*)$/ + static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)(\?.*)?$/ + static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)(\?.*)?$/ + @javax.annotation.PostConstruct def init() { @@ -50,7 +52,6 @@ class PackageSourceUpdateService { def updateFromSource(Long pkgId, def user = null, Job job = null, Long activeGroupId = null, boolean dryRun = false, boolean restrictSize = true) { log.debug("updateFromSource ${pkgId}") - log.info("111111111111111111111111111111111111111111111111111111111111111111111111111111111") def result = [result: 'OK'] def activeJobs = concurrencyManagerService?.getComponentJobs(pkgId) @@ -71,7 +72,6 @@ class PackageSourceUpdateService { private def startSourceUpdate(pid, user, job, activeGroupId, dryRun, restrictSize) { log.debug("Source update start..") - log.info("22222222222222222222222222222222222222222222222222222222222222222222") def result = [result: 'OK', dryRun: dryRun] Boolean async = (user ? true : false) def preferred_group @@ -122,11 +122,9 @@ class PackageSourceUpdateService { if(isFtpTransfer){ ftpUrlParts = webEndpointService.extractFtpUrlParts(pkg_source.getWebEndpoint()?.getUrl(), pkg_source.getFtpUrl()) completeFtpUrl = ftpUrlParts.complete - log.debug("xxxxxxxx: " + ftpUrlParts) } def valid_url_string = validationService.checkUrl(isFtpTransfer ? completeFtpUrl : pkg_source?.url, true) - log.debug("yyyyyyy: " + valid_url_string) LocalDate extracted_date skipInvalid = pkg_source.skipInvalid ?: false def file_info = [:] @@ -172,8 +170,7 @@ class PackageSourceUpdateService { if ( isFtpTransfer ) { log.debug("Start FTP Update from Source " + pkg_source ) ftpUrlParts["complete"] = src_url.toString() - log.debug("URL: " + src_url + ", " + ftpUrlParts) - file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source, ftpUrlParts, dynamic_date, extracted_date, restrictSize) + file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source, ftpUrlParts, dynamic_date, extracted_date, lastRunLocal, restrictSize) } else { // start not-FTP @@ -640,11 +637,10 @@ class PackageSourceUpdateService { return info_map } - def fetchKbartFileFromFTPServer (File tmp_file, Source source, def urlParts, boolean dynamic_date, LocalDate extracted_date, boolean restrictSize = true) { + def fetchKbartFileFromFTPServer (File tmp_file, Source source, def urlParts, boolean dynamic_date, LocalDate extracted_date, LocalDate lastRunLocal, boolean restrictSize = true) { def result = [content_mime_type: null, file_name: null] Long max_length = 20971520L // 1024 * 1024 * 20 - LocalDate lastRunLocal = source.lastRun ? source.lastRun.toInstant().atZone(ZoneId.systemDefault()).toLocalDate() : null FTPClient ftp = new FTPClient() FTPClientConfig config = new FTPClientConfig() @@ -665,7 +661,13 @@ class PackageSourceUpdateService { } try { - ftp.connect(hostname) + // for integration test purpose + if (Environment.current == Environment.TEST) { + ftp.connect(hostname, 12345) + } + else { + ftp.connect(hostname) + } ftp.enterLocalPassiveMode() def loggedIn = ftp.login(username, password) @@ -675,9 +677,17 @@ class PackageSourceUpdateService { List files if(dynamic_date || extracted_date) { - String fixFilenamePart = filename.split("\\{")[0] + + def dateMaskMatch = (filename =~ VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN) + String matchedDate = dateMaskMatch[0][1] + String fixFilenamePart = filename.substring(0, filename.indexOf(matchedDate)) + files = ftp.listFiles().toList() - List res = files.stream().filter(f -> f.name =~ VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN && f.name.startsWith(fixFilenamePart)).collect(Collectors.toList()).sort() + List unorderedRes = files.stream().filter(f -> + f.name =~ VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN && f.name.startsWith(fixFilenamePart)) + .collect(Collectors.toList()) + List res = unorderedRes.sort((f1, f2) -> f1.getName().compareTo(f2.getName())) + // file with latest date is the last in list res if(res.size() > 0) { foundFile = res.get(res.size() - 1) @@ -727,6 +737,7 @@ class PackageSourceUpdateService { } else { // no filename --> handled in calling method + } ftp.logout() diff --git a/server/grails-app/services/org/gokb/ValidationService.groovy b/server/grails-app/services/org/gokb/ValidationService.groovy index ccd528eb8..54f88c714 100644 --- a/server/grails-app/services/org/gokb/ValidationService.groovy +++ b/server/grails-app/services/org/gokb/ValidationService.groovy @@ -6,6 +6,7 @@ import com.opencsv.CSVReader import com.opencsv.CSVReaderBuilder import com.opencsv.CSVParser import com.opencsv.CSVParserBuilder +import grails.util.Environment import grails.validation.ValidationException import java.time.LocalDate import org.apache.commons.io.ByteOrderMark @@ -925,7 +926,13 @@ class ValidationService { // log.debug("Final URL to check: ${final_val}") - return new UrlValidator(UrlValidator.ALLOW_LOCAL_URLS).isValid(final_val) ? value : null + // needed for FTP integration Tests to allow FTP URLS from FTP Mock Server + if (Environment.current == Environment.TEST) { + return new UrlValidator(UrlValidator.ALLOW_LOCAL_URLS).isValid(final_val) ? value : null + } + else { + return new UrlValidator().isValid(final_val) ? value : null + } } private String encodeUrlPart(String value) { diff --git a/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy index bf551687b..e9e8aef5f 100644 --- a/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy +++ b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy @@ -4,32 +4,23 @@ import com.k_int.ConcurrencyManagerService import grails.core.GrailsApplication import grails.gorm.transactions.Rollback import grails.testing.mixin.integration.Integration -import io.micronaut.http.client.BlockingHttpClient -import io.micronaut.http.client.HttpClient import org.apache.commons.io.IOUtils import org.apache.commons.net.ftp.FTPClient import org.apache.commons.net.ftp.FTPFile import org.gokb.cred.CuratoryGroup -import org.gokb.cred.KBComponent import org.gokb.cred.Org import org.gokb.cred.Package import org.gokb.cred.Platform import org.gokb.cred.RefdataCategory import org.gokb.cred.Source -import org.gokb.cred.User import org.gokb.cred.WebHookEndpoint import org.mockftpserver.fake.* import org.mockftpserver.fake.filesystem.* -import org.springframework.beans.factory.annotation.Autowired import org.springframework.core.io.ClassPathResource -import org.springframework.transaction.annotation.Transactional -import org.springframework.web.context.WebApplicationContext import spock.lang.Specification - import java.nio.charset.Charset @Integration -// @Transactional @Rollback class UpdatePackageRunFTPSpec extends Specification{ @@ -37,62 +28,56 @@ class UpdatePackageRunFTPSpec extends Specification{ ConcurrencyManagerService concurrencyManagerService PackageSourceUpdateService packageSourceUpdateService - @Autowired - WebApplicationContext ctx private static final String HOME_DIR = "/"; - private static final String FILE = "/dir/sample.txt"; + private static final String NON_KBART_FILE = "/dir/sample.txt"; private static final String CONTENTS = "abcdef 1234567890"; private static final String USER = "user"; private static final String PASSWORD = "password"; + private static final int MOCKSERVER_PORT = 12345 private FakeFtpServer ftpServer - private Package packageUnderInspection private CuratoryGroup new_cg - private Org provider - private WebHookEndpoint whe - private Source source - private Platform platform - private String getUrlPath() { - return "http://localhost:${serverPort}${grailsApplication.config.getProperty('server.servlet.context-path') ?: ''}".toString() - } - private void setupMockFtpBase(){ + + def setup() { ftpServer = new FakeFtpServer() - ftpServer.setServerControlPort(21) + ftpServer.setServerControlPort(MOCKSERVER_PORT) FileSystem fileSystem = new UnixFakeFileSystem() - fileSystem.add(new FileEntry(FILE, CONTENTS)) + fileSystem.add(new FileEntry(NON_KBART_FILE, CONTENTS)) ftpServer.setFileSystem(fileSystem) ftpServer.addUserAccount(new UserAccount(USER, PASSWORD, HOME_DIR)) ftpServer.start() - } - - void setUpKbartExistsNormalName(){ - def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") - try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ - String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) - ftpServer.getFileSystem().add(new FileEntry("/dir/kbart.txt", fileContent)) - } - } - - def setup() { - this.setupMockFtpBase() new_cg = CuratoryGroup.findByName('TestGroup1') ?: new CuratoryGroup(name: "TestGroup1").save(flush: true) - provider = Org.findByName("American Chemical Society") ?: new Org(name: "American Chemical Society").save(flush: true) - platform = Platform.findByName("Test Platform") ?: new Platform(name: "Test Platform", primaryUrl: "https://search.ebscohost.com", provider: provider).save(flush: true) - whe = WebHookEndpoint.findByName("MOCK FTP on Localhost") ?: new WebHookEndpoint(name: "MOCK FTP on Localhost" ,url: "ftp://localhost", ba_username: USER, ba_password: PASSWORD).save(flush: true) - source = Source.findByName("Test Source") ?: new Source(name: "Test Source", webEndpoint: whe, ftpUrl: "/dir/kbart.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) - - // TODO: Pflichtfelder - packageUnderInspection = Package.findByName("Update Package") ?: new Package(name: "Update Package").save(flush: true) - packageUnderInspection.setSource(source) - - packageUnderInspection.save(flush: true) - + Org provider = Org.findByName("American Chemical Society") ?: new Org(name: "American Chemical Society").save(flush: true) + Platform.findByName("Test Platform") ?: new Platform(name: "Test Platform", primaryUrl: "https://search.ebscohost.com", provider: provider).save(flush: true) + WebHookEndpoint.findByName("whe1") ?: new WebHookEndpoint(name: "whe1" ,url: "ftp://localhost/dir", ba_username: USER, ba_password: PASSWORD).save(flush: true) + Source.findByName("source1") ?: new Source(name: "source1", webEndpoint: WebHookEndpoint.findByName("whe1"), ftpUrl: "/kbart.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) + Package.findByName("package1") ?: new Package(name: "package1").save(flush: true) + Package.findByName("package1").setSource(Source.findByName("source1")) + Package.findByName("package1").save(flush: true) + + WebHookEndpoint.findByName("whe2") ?: new WebHookEndpoint(name: "whe2" ,url: "ftp://localhost/dir", ba_username: USER, ba_password: PASSWORD).save(flush: true) + Source.findByName("source2") ?: new Source(name: "source2", webEndpoint: WebHookEndpoint.findByName("whe2"), ftpUrl: "/kbart_not_exists.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) + Package.findByName("package2") ?: new Package(name: "package2").save(flush: true) + Package.findByName("package2").setSource(Source.findByName("source2")) + Package.findByName("package2").save(flush: true) + + WebHookEndpoint.findByName("whe3") ?: new WebHookEndpoint(name: "whe3" ,url: "ftp://localhost/dir", ba_username: USER, ba_password: PASSWORD).save(flush: true) + Source.findByName("source3") ?: new Source(name: "source3", webEndpoint: WebHookEndpoint.findByName("whe3"), ftpUrl: "/kbart_{YYYY-MM-DD}.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) + Package.findByName("package3") ?: new Package(name: "package3").save(flush: true) + Package.findByName("package3").setSource(Source.findByName("source3")) + Package.findByName("package3").save(flush: true) + + WebHookEndpoint.findByName("whe4") ?: new WebHookEndpoint(name: "whe4" ,url: "ftp://localhost/dir", ba_username: USER, ba_password: PASSWORD).save(flush: true) + Source.findByName("source4") ?: new Source(name: "source4", webEndpoint: WebHookEndpoint.findByName("whe4"), ftpUrl: "/kbart_2026-01-11.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) + Package.findByName("package4") ?: new Package(name: "package4").save(flush: true) + Package.findByName("package4").setSource(Source.findByName("source4")) + Package.findByName("package4").save(flush: true) } @@ -102,99 +87,182 @@ class UpdatePackageRunFTPSpec extends Specification{ Org.findByName("American Chemical Society")?.expunge() Platform.findByName("Test Platform")?.expunge() - //Source.findByName("Test Source")?.expunge() - //whe.delete() - Package.findByName("Update Package")?.expunge() - //TODO: von KBComponent ableiten??? - //WebHookEndpoint.findByName("MOCK FTP on Localhost")?.expunge() + Package.findByName("package1")?.expunge() + Source.findByName("source1")?.expunge() + WebHookEndpoint.findByName("whe1")?.delete() - } + Package.findByName("package2")?.expunge() + Source.findByName("source2")?.expunge() + WebHookEndpoint.findByName("whe2")?.delete() + Package.findByName("package3")?.expunge() + Source.findByName("source3")?.expunge() + WebHookEndpoint.findByName("whe3")?.delete() - void "Test updateFromSource :: usual initial Import by filename"() { + Package.findByName("package4")?.expunge() + Source.findByName("source4")?.expunge() + WebHookEndpoint.findByName("whe4")?.delete() - given: "asked Kbart exists without date mask" - setUpKbartExistsNormalName() + } - def user = User.findByUsername('admin') + void "test Mock FTP Server runs"() { - /* PackageSourceUpdateService service = new PackageSourceUpdateService() + FTPClient ftpClient = new FTPClient() + ftpClient.connect("localhost", ftpServer.getServerControlPort()) + ftpClient.login(USER, PASSWORD); - ConcurrencyManagerService.Job pkg_job = concurrencyManagerService.createJob { pjob -> - service.updateFromSource(packageUnderInspection.id, null, pkg_job, new_cg.id, false, true) - } - Package.withNewSession { - pkg_job.startOrQueue() - } - def job_result = pkg_job.get() - */ + expect: "Directory exists" + ftpClient.listDirectories().length > 0 + ftpServer.getServerControlPort() == MOCKSERVER_PORT - /* ConcurrencyManagerService.Job pkg_job = concurrencyManagerService.createJob { pjob -> - packageSourceUpdateService.updateFromSource(packageUnderInspection.id, null, pjob, new_cg.id, false, true) - } + } - pkg_job.startOrQueue() - def job_result = pkg_job.get() - */ + void "Test updateFromSource :: usual initial direct Import by filename"() { - def result = packageSourceUpdateService.updateFromSource(packageUnderInspection.id, null, null, new_cg.id, false, true) + given: "requested Kbart exists without date mask" + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart.txt", fileContent)) + } + + Package p = Package.findByName("package1") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) - System.out.println("222222222222222222: " + result) expect: "file is found, package titles imported" - def titles = packageUnderInspection.getTitles(true, 100, 0) - titles != null - System.out.println("######### " + titles) + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 + + } + void "Test updateFromSource :: KBART does not exist on Server"() { + given: "configured KBART file does not exist on server" + Package p = Package.findByName("package2") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + expect: + result.result == 'SKIPPED' + result.messageCode == 'kbart.transmission.skipped.noFile' + p.getTitles(true, 10, 0).size() == 0 } - void "test WebEndpointService extractUrlParts :: split parts and format them correct"() { - when: "The Slashes are not set correctly in URL Field" - String endpointUrl = "ftp://servername.com/home/" - String sourceUrl = "/filename.txt" - def res = new WebEndpointService().extractFtpUrlParts(endpointUrl, sourceUrl) + void "Test updateFromSource :: KBART with date mask is found and imported"() { + given: "FTP URL with date mask configured, file with date exists" + + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-01-01.txt", fileContent)) + } + Package p = Package.findByName("package3") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + expect: + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 - then: "The URL is normalized and splitted correctly into parts" + } - res.hostname == "servername.com" - res.directory == "/home/" - res.filename == "filename.txt" - res.complete == "ftp://servername.com/home/filename.txt" + void "Test updateFromSource :: KBART with latest date mask is found"() { + given: "FTP URL with date mask configured, several files with dates exist" + // source.ftpUrl = "kbart_{YYYY-MM-DD}.txt" + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + String fakeContent = "AAAAAAAAAAAA" + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-02-01.txt", fileContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-01-01.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2023-10-30.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2025-12-24.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/other_package_2026-02-02.txt", fakeContent)) + } + //same WHE as in previous test + Package p = Package.findByName("package3") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + expect: "find the file with the latest date, i.e. the only file with KBART content" + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 } - void "test Mock FTP Server runs"() { + void "Test updateFromSource :: KBART with specified date is found"() { + given: "FTP URL with specific date configured, several files with dates exist" + // source.ftpUrl = "kbart_2026-01-11.txt" + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + String fakeContent = "AAAAAAAAAAAA" + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-02-01.txt", fileContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-01-01.txt", "fakeContent")) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2023-10-30.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2025-12-24.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/other_package_2026-02-02.txt", fakeContent)) + } + Package p = Package.findByName("package4") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + expect: "find the file with the latest date, i.e. the only file with KBART content" + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 + } - FTPClient ftpClient = new FTPClient() - ftpClient.connect("localhost", ftpServer.getServerControlPort()) - ftpClient.login(USER, PASSWORD); - expect: "Directory exists" - ftpClient.listDirectories().length > 0 + void "Test updateFromSource :: Package is updated"() { + given: "Package is imported initially" + //source.ftpUrl = "/kbart_{YYYY-MM-DD}.txt" + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-02-01.txt", fileContent)) + } + // source with date mask + Package p = Package.findByName("package3") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) - /* and: "find file and read its contents" - ftpClient.changeWorkingDirectory("dir") - def files = ftpClient.listFiles() - System.out.println("+++ " + files) + expect: "Package should be imported correctly" - String fileContent = "" - try(InputStream is = ftpClient.retrieveFileStream("kbart.txt")){ - fileContent = IOUtils.toString(is, Charset.defaultCharset()) - } - */ - //System.out.println(fileContent) + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 + def kbart_update_file = new ClassPathResource("/test_ftp_kbart_update_new_file.txt") + try(FileInputStream fis = new FileInputStream(kbart_update_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-02-03.txt", fileContent)) + } + def update_result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + and: "Package should be updated by a new file" + update_result.result == 'OK' + p.getTitles(true, 100, 0).size() == 3 + + } + + + void "test WebEndpointService extractUrlParts :: split parts and format them correct"() { + + when: "The Slashes are not set correctly in URL Field" + String endpointUrl = "ftp://servername.com/home/" + String sourceUrl = "/filename.txt" + + def res = new WebEndpointService().extractFtpUrlParts(endpointUrl, sourceUrl) - } + then: "The URL is normalized and splitted correctly into parts" + res.hostname == "servername.com" + res.directory == "/home/" + res.filename == "filename.txt" + res.complete == "ftp://servername.com/home/filename.txt" + } diff --git a/server/src/integration-test/resources/test_ftp_kbart_update.txt b/server/src/integration-test/resources/test_ftp_kbart_update.txt index 922c44f8d..78760b9ab 100644 --- a/server/src/integration-test/resources/test_ftp_kbart_update.txt +++ b/server/src/integration-test/resources/test_ftp_kbart_update.txt @@ -1,71 +1,3 @@ publication_title print_identifier online_identifier date_first_issue_online num_first_vol_online num_first_issue_online date_last_issue_online num_last_vol_online num_last_issue_online title_url first_author title_id embargo_info coverage_depth notes publisher_name publication_type date_monograph_published_print date_monograph_published_online monograph_volume monograph_edition first_editor parent_publication_title_id preceding_publication_title_id access_type package_name package_id ebsco_resource_type -(parenthetical) 2368-0202 2016-05-01 2017-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JURX&scope=site JURX fulltext words(on)pages serial P Canadian Literary Centre cjh Periodical -Acta Victoriana 0700-8406 2017-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUQY&scope=site JUQY fulltext Acta Victoriana serial P Canadian Literary Centre cjh Periodical -Antigonish Review 0003-5661 2004-04-01 2021-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=FVI&scope=site FVI abstracts Antigonish Review serial P Canadian Literary Centre cjh Periodical -Antigonish Review 0003-5661 2005-04-01 2010-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=FVI&scope=site FVI fulltext Antigonish Review serial P Canadian Literary Centre cjh Periodical Arc Poetry Magazine 1910-3239 2008-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BGAO&scope=site BGAO fulltext ARC: Canada's National Poetry Magazine serial P Canadian Literary Centre cjh Periodical Books in Canada 0045-2564 1971-05-01 2008-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=4TI&scope=site 4TI fulltext Canadian Review of Books Ltd. serial P Canadian Literary Centre cjh Periodical -Brick: A Literary Journal 0382-8565 2818-5390 2011-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BCPO&scope=site BCPO fulltext Brick: A Literary Journal serial P Canadian Literary Centre cjh Periodical -Broken Pencil 1201-8996 2012-01-01 2024-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=90TX&scope=site 90TX fulltext Broken Pencil serial P Canadian Literary Centre cjh Periodical -Canadian Journal of Film & Media Studies 2819-4748 2819-4756 2025-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=NV4N&scope=site NV4N fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -Canadian Journal of Film Studies 0847-5911 2003-09-01 2024-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I2G&scope=site I2G abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -Canadian Journal of Film Studies 0847-5911 2006-05-01 2024-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I2G&scope=site I2G fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -Canadian Literature 0008-4360 1992-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=CLI&scope=site CLI abstracts Canadian Literature serial P Canadian Literary Centre cjh Academic Journal -Canadian Literature 0008-4360 2003-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=CLI&scope=site CLI fulltext Canadian Literature serial P Canadian Literary Centre cjh Academic Journal -Canadian Modern Language Review 0008-4506 1710-1131 1997-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=7RY&scope=site 7RY abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -Canadian Publishers Directory 0008-4859 2009-01-01 2010-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=47CE&scope=site 47CE fulltext St. Joseph Media serial P Canadian Literary Centre cjh Periodical -Canadian Theatre Review 0315-0836 1920-941X 1974-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=6FB&scope=site 6FB abstracts University of Toronto Press serial P Canadian Literary Centre cjh Periodical -Canadian Theatre Review 0315-0836 1920-941X 1997-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=6FB&scope=site 6FB fulltext University of Toronto Press serial P Canadian Literary Centre cjh Periodical -Capilano Review 0315-3754 2003-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UJO&scope=site UJO fulltext Capilano Review serial P Canadian Literary Centre cjh Periodical -Cinephile 1712-9265 2012-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=EV4A&scope=site EV4A fulltext University of British Columbia, Department of Theatre & Film serial P Canadian Literary Centre cjh Academic Journal -CNQ: Canadian Notes & Queries 0576-5803 2016-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=11WQ&scope=site 11WQ fulltext CNQ: Canadian Notes & Queries serial P Canadian Literary Centre cjh Periodical -Contemporary Verse 2 0831-9502 2016-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BCPU&scope=site BCPU fulltext Contemporary Verse 2 serial P Canadian Literary Centre cjh Periodical -Dalhousie Review 0011-5827 1987-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=DHR&scope=site DHR abstracts Dalhousie Review serial P Canadian Literary Centre cjh Academic Journal -Dalhousie Review 0011-5827 2010-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=DHR&scope=site DHR fulltext Dalhousie Review serial P Canadian Literary Centre cjh Academic Journal -Écrits 1200-7935 2015-11-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=FXMB&scope=site FXMB fulltext Academie des Lettres du Quebec serial P Canadian Literary Centre cjh Periodical -Essays on Canadian Writing 0316-0300 1974-12-01 2009-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=ECW&scope=site ECW abstracts ECW Press Ltd. serial P Canadian Literary Centre cjh Academic Journal -Essays on Canadian Writing 0316-0300 1977-03-01 2009-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=ECW&scope=site ECW fulltext ECW Press Ltd. serial P Canadian Literary Centre cjh Academic Journal -Event (0315-3770) 0315-3770 2003-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UL2&scope=site UL2 fulltext Douglas College - Creative Writing Dept. serial P Canadian Literary Centre cjh Periodical -Existere 1710-386X 2021-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUR4&scope=site JUR4 fulltext Professional Writing Program, York University Writing Department serial P Canadian Literary Centre cjh Periodical -Fiddlehead 0015-0630 2003-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=ULM&scope=site ULM abstracts University of New Brunswick serial P Canadian Literary Centre cjh Periodical -Filling Station 1198-0060 2019-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BCPQ&scope=site BCPQ fulltext Filling Station Publications serial P Canadian Literary Centre cjh Periodical -Glass Buffalo 1929-8587 2015-07-01 2019-10-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JESI&scope=site JESI fulltext Matthew Stepanic serial P Canadian Literary Centre cjh Periodical -Grain Magazine 1491-0497 2004-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UX0&scope=site UX0 abstracts Saskatchewan Writers Guild serial P Canadian Literary Centre cjh Periodical -Grain Magazine 1491-0497 2007-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UX0&scope=site UX0 fulltext Saskatchewan Writers Guild serial P Canadian Literary Centre cjh Periodical -Letters in Canada 0315-4955 1993-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=J0I&scope=site J0I abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -Letters in Canada 0315-4955 1994-12-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=J0I&scope=site J0I fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -Literary Review of Canada 1188-7494 2020-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=12DW&scope=site 12DW fulltext Literary Review of Canada serial P Canadian Literary Centre cjh Periodical -Lurelu 0705-6567 1923-2330 2023-10-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BFFG&scope=site BFFG fulltext Lurelu serial P Canadian Literary Centre cjh Periodical -MacroMicroCosm Literary & Art Journal 2368-9781 2016-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JURE&scope=site JURE fulltext Vraeyda Media serial P Canadian Literary Centre cjh Periodical -Malahat Review 1923-7502 2004-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UM1&scope=site UM1 abstracts Malahat Review serial P Canadian Literary Centre cjh Periodical -Maple Tree Literary Supplement, MTLS 1916-341X 2017-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=L03R&scope=site L03R fulltext Maple Tree Literary Supplement, MTLS, Inc. serial P Canadian Literary Centre cjh Academic Journal -Margaret Atwood's Alias Grace: A Reader's Guide 978-0-8264-5706-6 https://search.ebscohost.com/direct.asp?db=cjh&jid=1GBQ&scope=site 1GBQ fulltext Bloomsbury Publishing Plc monograph 2002 2002 P Canadian Literary Centre cjh Book -Modern Drama 0026-7694 1712-5286 1965-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=MOD&scope=site MOD abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -Modern Drama 0026-7694 1712-5286 1997-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=MOD&scope=site MOD fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -Montreal Review 1920-2911 2016-09-01 2019-02-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=KP08&scope=site KP08 fulltext Tsonchev Publishing & Design serial P Canadian Literary Centre cjh Academic Journal -Mouvances Francophones 2371-7211 2023-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=N9P1&scope=site N9P1 fulltext Western University, Department of French Studies serial P Canadian Literary Centre cjh Academic Journal -Nashwaak Review 1205-7681 2009-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=90AD&scope=site 90AD fulltext Nashwaak Review serial P Canadian Literary Centre cjh Academic Journal -New Quarterly 0227-0455 2017-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BCPS&scope=site BCPS fulltext New Quarterly Literary Society Inc. serial P Canadian Literary Centre cjh Periodical -Nouveau Projet 1927-8039 2016-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JURI&scope=site JURI fulltext Atelier 10 serial P Canadian Literary Centre cjh Periodical -On Spec: The Canadian Magazine of the Fantastic 0843-476X 2018-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=LB2T&scope=site LB2T fulltext Copper Pig Writers' Society serial P Canadian Literary Centre cjh Periodical -Passages North 0278-0828 2002-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UOM&scope=site UOM abstracts Passages North serial P Canadian Literary Centre cjh Periodical -Prairie Fire 0821-1124 2016-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=3QXT&scope=site 3QXT fulltext Prairie Fire Press, Inc. serial P Canadian Literary Centre cjh Periodical -Prairie Journal of Canadian Literature 0827-2921 2002-06-01 2020-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=E01&scope=site E01 abstracts Prairie Journal Trust serial P Canadian Literary Centre cjh Academic Journal -Queen's Quarterly 0033-6041 2008-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=7JU&scope=site 7JU fulltext Queens University serial P Canadian Literary Centre cjh Academic Journal -Quill & Quire 0033-6491 1996-01-01 2020-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=01N&scope=site 01N fulltext St. Joseph Media serial P Canadian Literary Centre cjh Periodical -Quill & Quire 0033-6491 1996-01-01 2022-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=01N&scope=site 01N abstracts St. Joseph Media serial P Canadian Literary Centre cjh Periodical -QWERTY 1203-8768 2018-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUS1&scope=site JUS1 fulltext Qwerty serial P Canadian Literary Centre cjh Periodical -Reappraisals: Canadian Writers 1189-6787 2017-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=LEX3&scope=site LEX3 fulltext University of Ottawa Press serial P Canadian Literary Centre cjh Academic Journal -Rhetor. Journal of the Canadian Society for the Study of Rhetoric 1712-2333 2004-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=1G8D&scope=site 1G8D fulltext Canadian Society for the Study of Rhetoric serial P Canadian Literary Centre cjh Academic Journal -Room Magazine 1914-4083 2013-05-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=46AO&scope=site 46AO fulltext Growing Room Collective serial P Canadian Literary Centre cjh Periodical -Site Magazine 2369-9566 2018-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=LS18&scope=site LS18 fulltext Site Magazine serial P Canadian Literary Centre cjh Periodical -Solaris 0709-8863 2021-10-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=LNR6&scope=site LNR6 fulltext Solaris serial P Canadian Literary Centre cjh Periodical -Studies in Canadian Literature / Études en Littérature Canadienne 0380-6995 1718-7850 2003-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=4TM&scope=site 4TM abstracts Studies in Canadian Literature / Etudes en Litterature Canadienne serial P Canadian Literary Centre cjh Academic Journal -Sub-Terrain 0840-7533 2003-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I3B&scope=site I3B abstracts SubTerrain Magazine serial P Canadian Literary Centre cjh Periodical -Sub-Terrain 0840-7533 2014-07-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I3B&scope=site I3B fulltext SubTerrain Magazine serial P Canadian Literary Centre cjh Periodical -Taddle Creek 1480-2481 1710-8632 2016-12-01 2022-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUS5&scope=site JUS5 fulltext Vitalis Publishing serial P Canadian Literary Centre cjh Periodical -Textual Studies in Canada 1183-854X 1991-09-01 2004-06-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=4TN&scope=site 4TN fulltext Textual Studies in Canada serial P Canadian Literary Centre cjh Academic Journal -Theatre Research in Canada 1196-1198 1913-9101 2002-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I2T&scope=site I2T abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -University of Toronto Quarterly 0042-0247 1712-5278 1974-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UTQ&scope=site UTQ abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -University of Toronto Quarterly 0042-0247 1712-5278 1975-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=UTQ&scope=site UTQ fulltext University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal -Vallum: Contemporary Poetry 1496-5178 2016-04-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=JUS6&scope=site JUS6 fulltext Joshua Auerbach serial P Canadian Literary Centre cjh Periodical diff --git a/server/src/integration-test/resources/test_ftp_kbart_update_new_file.txt b/server/src/integration-test/resources/test_ftp_kbart_update_new_file.txt new file mode 100644 index 000000000..657f0ff43 --- /dev/null +++ b/server/src/integration-test/resources/test_ftp_kbart_update_new_file.txt @@ -0,0 +1,4 @@ +publication_title print_identifier online_identifier date_first_issue_online num_first_vol_online num_first_issue_online date_last_issue_online num_last_vol_online num_last_issue_online title_url first_author title_id embargo_info coverage_depth notes publisher_name publication_type date_monograph_published_print date_monograph_published_online monograph_volume monograph_edition first_editor parent_publication_title_id preceding_publication_title_id access_type package_name package_id ebsco_resource_type +Arc Poetry Magazine 1910-3239 2008-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BGAO&scope=site BGAO fulltext ARC: Canada's National Poetry Magazine serial P Canadian Literary Centre cjh Periodical +Books in Canada 0045-2564 1971-05-01 2008-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=4TI&scope=site 4TI fulltext Canadian Review of Books Ltd. serial P Canadian Literary Centre cjh Periodical +Canadian Journal of Film Studies 0847-5911 2003-09-01 2024-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I2G&scope=site I2G abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal \ No newline at end of file From 98837f2077aa83bab3dc135e158fcee118844d6e Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Thu, 5 Feb 2026 13:40:43 +0100 Subject: [PATCH 26/28] - change authentication for webendpoint controller mehtods - regard expected and retired titles for tipp-count in wekb-import --- .../controllers/org/gokb/rest/WebEndpointController.groovy | 6 +++--- .../services/org/gokb/WekbIngestionService.groovy | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index 93de68e03..cb613beff 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -24,7 +24,7 @@ class WebEndpointController { static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)$/ - @Secured(['IS_AUTHENTICATED_ANONYMOUSLY']) + @Secured(value = ["hasRole('ROLE_CONTRIBUTOR')", 'IS_AUTHENTICATED_FULLY']) def index() { def result = [:] def base = grailsApplication.config.getProperty('grails.serverURL') + "/rest" @@ -54,7 +54,7 @@ class WebEndpointController { render result as JSON } - @Secured(['IS_AUTHENTICATED_ANONYMOUSLY']) + @Secured(value = ["hasRole('ROLE_CONTRIBUTOR')", 'IS_AUTHENTICATED_FULLY']) def show() { def result = [:] def base = grailsApplication.config.getProperty('grails.serverURL') + "/rest" @@ -81,7 +81,7 @@ class WebEndpointController { render result as JSON } - @Secured(['IS_AUTHENTICATED_ANONYMOUSLY']) + @Secured(value = ["hasRole('ROLE_CONTRIBUTOR')", 'IS_AUTHENTICATED_FULLY']) def check() { def result = [:] def reqBody = request.JSON diff --git a/server/grails-app/services/org/gokb/WekbIngestionService.groovy b/server/grails-app/services/org/gokb/WekbIngestionService.groovy index 21f67fb5c..5ac874ee8 100644 --- a/server/grails-app/services/org/gokb/WekbIngestionService.groovy +++ b/server/grails-app/services/org/gokb/WekbIngestionService.groovy @@ -93,8 +93,8 @@ class WekbIngestionService { where c.fromComponent.id = :pkg and c.toComponent = tipp and c.type = :ct - and tipp.status = :sc''', - [pkg: pkg.id, ct: combo_pkg, sc: rdv_current])[0] + and tipp.status != :sd''', + [pkg: pkg.id, ct: combo_pkg, sd: rdv_deleted])[0] result.report = [ numRows : titleCount, From db9dca69c213350136b4baa766ec4dc7984ff7a4 Mon Sep 17 00:00:00 2001 From: christophkohl1 Date: Thu, 5 Feb 2026 18:19:24 +0100 Subject: [PATCH 27/28] refactor --- .../org/gokb/rest/WebEndpointController.groovy | 14 +++++++------- .../domain/org/gokb/cred/WebHookEndpoint.groovy | 17 +++++++++++++---- .../org/gokb/PackageSourceUpdateService.groovy | 4 ++-- .../views/apptemplates/_web_hook_endpoint.gsp | 12 ++++++------ .../org/gokb/UpdatePackageRunFTPSpec.groovy | 11 +++++++---- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy index cb613beff..e3a85a41c 100644 --- a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -34,14 +34,14 @@ class WebEndpointController { user = User.get(springSecurityService.principal?.id) } - params['_embed'] = params['_embed'] ?: 'identifiedComponents' + // params['_embed'] = params['_embed'] ?: 'identifiedComponents' result = componentLookupService.restLookup(user, WebHookEndpoint, params) if (result.data) { def resultList = result.data - resultList*.remove('ba_password') - resultList*.remove('ba_username') + resultList*.remove('epPassword') + resultList*.remove('epUsername') if (params['method']) { //resultList = resultList.findAll( x -> x.transferMethod?.name == params['method']) @@ -65,15 +65,15 @@ class WebEndpointController { } def start_db = LocalDateTime.now() - params['_embed'] = params['_embed'] ?: 'identifiedComponents' + // params['_embed'] = params['_embed'] ?: 'identifiedComponents' result = componentLookupService.restLookup(user, WebHookEndpoint, params) def resultList = result.data if(resultList.size() > 0){ - resultList*.remove('ba_password') - resultList*.remove('ba_username') + resultList*.remove('epPassword') + resultList*.remove('epUsername') result.data = resultList.get(0) } @@ -107,7 +107,7 @@ class WebEndpointController { try { ftp.connect(hostname) ftp.enterLocalPassiveMode() - def loggedIn = ftp.login(whe.getBa_username(), whe.getBa_password()) + def loggedIn = ftp.login(whe.getEpUsername(), whe.getEpPassword()) if (ftp.isConnected()) { diff --git a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy index 206f7f307..f5559bd37 100644 --- a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy +++ b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy @@ -13,8 +13,8 @@ class WebHookEndpoint { String principal //legacy String credentials //legacy User owner - String ba_username - String ba_password + String epUsername + String epPassword static mapping = { url column:'ep_url' @@ -39,10 +39,19 @@ class WebHookEndpoint { principal(nullable:true, blank:true) credentials(nullable:true, blank:true) supplyMethod(nullable:true, blank:true) - ba_username(nullable:true, blank:true) - ba_password(nullable:true, blank:true) + epUsername(nullable:true, blank:true) + epPassword(nullable:true, blank:true) } + static jsonMapping = [ + 'ignore' : [ + 'epPassword', + 'epUsername', + 'credentials', + 'authmethod' + ] + ] + static def refdataFind(params) { log.debug("refdataFind(${params})"); diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 49a40c086..9b4573a36 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -645,8 +645,8 @@ class PackageSourceUpdateService { FTPClient ftp = new FTPClient() FTPClientConfig config = new FTPClientConfig() - String username = source.getWebEndpoint().getBa_username() - String password = source.getWebEndpoint().getBa_password() + String username = source.getWebEndpoint().getEpUsername() + String password = source.getWebEndpoint().getEpPassword() String hostname = urlParts.hostname String directory = urlParts.directory diff --git a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp index 10e40466e..36a28e90b 100644 --- a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp +++ b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp @@ -5,17 +5,17 @@
Base URL
-
Benutzername
-
+
Benutzername
+
-
Passwort
- -
+
Passwort
+ +