diff --git a/server/build.gradle b/server/build.gradle index bbb66cb6e..388a99dc6 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -117,6 +117,7 @@ dependencies { implementation 'org.apache.tika:tika-core:3.2.3' implementation 'dk.glasius:external-config:4.0.0' + implementation 'commons-net:commons-net:3.12.0' implementation 'org.apache.httpcomponents:httpclient:4.5.14' implementation "io.micronaut:micronaut-http-client" @@ -138,6 +139,8 @@ dependencies { testImplementation "org.seleniumhq.selenium:selenium-remote-driver:$seleniumVersion" testImplementation "org.seleniumhq.selenium:selenium-api:$seleniumVersion" testImplementation "org.seleniumhq.selenium:selenium-support:$seleniumVersion" + + testImplementation 'org.mockftpserver:MockFtpServer:3.2.0' } bootRun { diff --git a/server/grails-app/conf/application.groovy b/server/grails-app/conf/application.groovy index d5ca4424d..84dcb3a21 100644 --- a/server/grails-app/conf/application.groovy +++ b/server/grails-app/conf/application.groovy @@ -1273,6 +1273,30 @@ globalSearchTemplates = [ ] ] ], + 'WebHookEndpoints':[ + baseclass:'org.gokb.cred.WebHookEndpoint', + title:'Web Endpoints', + group:'Secondary', + defaultSort:'id', + defaultOrder:'desc', + qbeConfig:[ + qbeForm:[ + [ + prompt:'Name', + qparam:'qp_name', + placeholder:'Name of Web Endpoint', + contextTree:['ctxtp':'qry', 'comparator' : 'ilike', 'prop':'name','wildcard':'R'] + ], + ], + qbeGlobals:[ + ], + qbeResults:[ + [heading:'Name', property:'name', link:[controller:'resource', action:'show', id:'x.r.uuid'] ], + [heading:'URL', property:'url'], + [heading:'Methode', property:'supplyMethod'], + ] + ] + ], ] diff --git a/server/grails-app/conf/spring/resources.groovy b/server/grails-app/conf/spring/resources.groovy index c0cc9861b..1b5089e4e 100644 --- a/server/grails-app/conf/spring/resources.groovy +++ b/server/grails-app/conf/spring/resources.groovy @@ -1,4 +1,5 @@ import com.k_int.UserPasswordEncoderListener + import com.k_int.utils.DatabaseMessageSource; import org.grails.spring.context.support.PluginAwareResourceBundleMessageSource; import org.springframework.web.servlet.i18n.SessionLocaleResolver diff --git a/server/grails-app/controllers/org/gokb/UrlMappings.groovy b/server/grails-app/controllers/org/gokb/UrlMappings.groovy index 9a439854b..768b4686f 100644 --- a/server/grails-app/controllers/org/gokb/UrlMappings.groovy +++ b/server/grails-app/controllers/org/gokb/UrlMappings.groovy @@ -191,6 +191,10 @@ class UrlMappings { get "/jobs/$id" (controller: 'jobs', namespace: 'rest', action: 'show') patch "/jobs/$id/cancel" (controller: 'jobs', namespace: 'rest', action: 'cancel') delete "/jobs/$id" (controller: 'jobs', namespace: 'rest', action: 'delete') + + get "/web-endpoint"(controller: 'webEndpoint', namespace: 'rest', action: 'index') + get "/web-endpoint/$id"(controller: 'webEndpoint', namespace: 'rest', action: 'show') + post "/web-endpoint/check"(controller: 'webEndpoint', namespace: 'rest', action: 'check') } "/$controller/$action?/$id?" { constraints { diff --git a/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy new file mode 100644 index 000000000..e3a85a41c --- /dev/null +++ b/server/grails-app/controllers/org/gokb/rest/WebEndpointController.groovy @@ -0,0 +1,168 @@ +package org.gokb.rest + +import grails.converters.JSON +import grails.plugin.springsecurity.annotation.Secured +import org.apache.commons.net.ftp.FTPClient +import org.apache.commons.net.ftp.FTPClientConfig +import org.apache.commons.net.ftp.FTPFile +import org.gokb.WebEndpointService +import org.gokb.cred.User +import org.gokb.cred.WebHookEndpoint + +import java.time.Duration +import java.time.LocalDateTime +import java.util.regex.Pattern + +class WebEndpointController { + + static namespace = 'rest' + + def componentLookupService + def springSecurityService + WebEndpointService webEndpointService + + static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ + static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)$/ + + @Secured(value = ["hasRole('ROLE_CONTRIBUTOR')", 'IS_AUTHENTICATED_FULLY']) + def index() { + def result = [:] + def base = grailsApplication.config.getProperty('grails.serverURL') + "/rest" + User user = null + + if (springSecurityService.isLoggedIn()) { + user = User.get(springSecurityService.principal?.id) + } + + // params['_embed'] = params['_embed'] ?: 'identifiedComponents' + + result = componentLookupService.restLookup(user, WebHookEndpoint, params) + + if (result.data) { + def resultList = result.data + resultList*.remove('epPassword') + resultList*.remove('epUsername') + + if (params['method']) { + //resultList = resultList.findAll( x -> x.transferMethod?.name == params['method']) + resultList = resultList.findAll( x -> x.url.startsWith(params['method'].toLowerCase()) ) + } + + result.data = resultList + } + + render result as JSON + } + + @Secured(value = ["hasRole('ROLE_CONTRIBUTOR')", 'IS_AUTHENTICATED_FULLY']) + def show() { + def result = [:] + def base = grailsApplication.config.getProperty('grails.serverURL') + "/rest" + User user = null + + if (springSecurityService.isLoggedIn()) { + user = User.get(springSecurityService.principal?.id) + } + def start_db = LocalDateTime.now() + + // params['_embed'] = params['_embed'] ?: 'identifiedComponents' + + result = componentLookupService.restLookup(user, WebHookEndpoint, params) + + def resultList = result.data + + if(resultList.size() > 0){ + resultList*.remove('epPassword') + resultList*.remove('epUsername') + + result.data = resultList.get(0) + } + + render result as JSON + } + + @Secured(value = ["hasRole('ROLE_CONTRIBUTOR')", 'IS_AUTHENTICATED_FULLY']) + def check() { + def result = [:] + def reqBody = request.JSON + + WebHookEndpoint whe = WebHookEndpoint.findById(reqBody.webhookendpoint) + String path = reqBody.url + String hostname = "" + String directory = "" + String filename = "" + + if(whe){ + def parts= webEndpointService.extractFtpUrlParts(whe.getUrl(), path) + hostname = parts.hostname + directory = parts.directory + filename = parts.filename + } + + def dateMaskMatch = (filename =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN) + + FTPClient ftp = new FTPClient() + FTPClientConfig config = new FTPClientConfig() + + try { + ftp.connect(hostname) + ftp.enterLocalPassiveMode() + def loggedIn = ftp.login(whe.getEpUsername(), whe.getEpPassword()) + + if (ftp.isConnected()) { + + FTPFile[] files + if(dateMaskMatch.size() > 0) { + String fixedPart = filename.split("\\{")[0] + files = ftp.listFiles(directory) + + boolean found = false + + for (FTPFile file : files) { + + if(file.name.startsWith(fixedPart)){ + result.result = "success" + result.message = "dateMaskFound" + found = true + break + } + } + if(!found){ + result.result = "error" + result.message = "dateMaskNotFound" + } + } + else { + files = ftp.listFiles(directory + filename) + if (files.length > 0 && files[0].size > 0) { + + result.result = "success" + result.message = "success" + } else { + result.result = "error" + result.message = "partlySuccessful" + } + + ftp.logout() + ftp.disconnect() + } + } + else { + result.result = "error" + result.message = "connectError" + } + + } catch (Exception e) { + log.error("Fehler bei FTP-Verbindung: ", e) + result.result = "error" + result.message = "configurationError" + } + + render result as JSON + + } + + + + +} diff --git a/server/grails-app/domain/org/gokb/cred/Source.groovy b/server/grails-app/domain/org/gokb/cred/Source.groovy index 3201be492..6d561dbf5 100644 --- a/server/grails-app/domain/org/gokb/cred/Source.groovy +++ b/server/grails-app/domain/org/gokb/cred/Source.groovy @@ -27,6 +27,9 @@ class Source extends KBComponent { BulkImportListConfig bulkConfig RefdataValue importConfig Boolean ignoreSizeLimit = false + WebHookEndpoint webEndpoint + String ftpUrl //umbenennen in ftpPath + RefdataValue transferMethod static manyByCombo = [ curatoryGroups: CuratoryGroup @@ -58,6 +61,9 @@ class Source extends KBComponent { bulkConfig(nullable: true, blank: false) importConfig(nullable: true, blank: true) ignoreSizeLimit(nullable: true, blank: true) + webEndpoint(nullable: true, blank: true) + ftpUrl(nullable: true, blank: true) + transferMethod(nullable: true, blank: true) } public static final String restPath = "/sources" diff --git a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy index e9aa3b548..f5559bd37 100644 --- a/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy +++ b/server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy @@ -7,10 +7,14 @@ import groovy.util.logging.* class WebHookEndpoint { String name String url - Long authmethod - String principal - String credentials + Long authmethod //legacy + RefdataValue supplyMethod //legacy + // RefdataValue transferMethod //rausnehmen - nur in Source + String principal //legacy + String credentials //legacy User owner + String epUsername + String epPassword static mapping = { url column:'ep_url' @@ -21,12 +25,33 @@ class WebHookEndpoint { static constraints = { name(nullable:false, blank:false) - url(nullable:false, blank:false) + url(validator: {val, obj -> + if(val) { + if(!val.startsWith("ftp://") && !val.startsWith("http://") && !val.startsWith("https://")){ + return ['webEndpointUrl.missingProtocol'] + } + } + else { + return ['webEndpointUrl.notNull'] + } + }) authmethod(nullable:true, blank:true) principal(nullable:true, blank:true) credentials(nullable:true, blank:true) + supplyMethod(nullable:true, blank:true) + epUsername(nullable:true, blank:true) + epPassword(nullable:true, blank:true) } + static jsonMapping = [ + 'ignore' : [ + 'epPassword', + 'epUsername', + 'credentials', + 'authmethod' + ] + ] + static def refdataFind(params) { log.debug("refdataFind(${params})"); diff --git a/server/grails-app/i18n/messages.properties b/server/grails-app/i18n/messages.properties index 8a269c40f..ef4d7044a 100644 --- a/server/grails-app/i18n/messages.properties +++ b/server/grails-app/i18n/messages.properties @@ -540,3 +540,6 @@ componentPrice.priceType.nullable.error = The kind of price must be set. admin.support.sizeLimit.email.subject = 'GOKB - Import failed due to active KBART size limit' admin.support.sizeLimit.email.line1 = 'An import for the following package failed due to the default KBART size limit:' + +webEndpointUrl.missingProtocol = 'Please provide the URL including the protocol (e.g., ftp:// or http://).' +webEndpointUrl.notNull = 'Please enter a URL.' diff --git a/server/grails-app/i18n/messages_de.properties b/server/grails-app/i18n/messages_de.properties index 25121ab01..3116d5bd5 100644 --- a/server/grails-app/i18n/messages_de.properties +++ b/server/grails-app/i18n/messages_de.properties @@ -420,4 +420,7 @@ typeMismatch.java.math.BigDecimal=Die Eigenschaft {0} muss eine gültige Zahl se typeMismatch.java.math.BigInteger=Die Eigenschaft {0} muss eine gültige Zahl sein admin.support.sizeLimit.email.subject = 'GOKB - KBART-Import aufgrund von Dateigrößenbeschränkung fehlgeschlagen' -admin.support.sizeLimit.email.line1 = 'Ein KBART-Import für das folgende Paket ist aufgrund der aktiven Größenbeschränkung fehlgeschlagen:' \ No newline at end of file +admin.support.sizeLimit.email.line1 = 'Ein KBART-Import für das folgende Paket ist aufgrund der aktiven Größenbeschränkung fehlgeschlagen:' + +webEndpointUrl.missingProtocol = 'Bitte geben Sie die URL mitsamt Protokoll (z.B. ftp:// oder http://) an.' +webEndpointUrl.notNull = 'Bitte geben Sie eine URL an.' \ No newline at end of file diff --git a/server/grails-app/init/org/gokb/BootStrap.groovy b/server/grails-app/init/org/gokb/BootStrap.groovy index 8ec365655..94966f22b 100644 --- a/server/grails-app/init/org/gokb/BootStrap.groovy +++ b/server/grails-app/init/org/gokb/BootStrap.groovy @@ -1122,6 +1122,9 @@ class BootStrap { RefdataCategory.lookupOrCreate('Source.Frequency', 'Quarterly', '090').save(flush: true, failOnError: true) RefdataCategory.lookupOrCreate('Source.Frequency', 'Yearly', '365').save(flush: true, failOnError: true) + RefdataCategory.lookupOrCreate('Source.TransferMethod', 'HTTP').save(flush: true, failOnError: true) + RefdataCategory.lookupOrCreate('Source.TransferMethod', 'FTP').save(flush: true, failOnError: true) + RefdataCategory.lookupOrCreate('BulkImportListConfig.Frequency', 'Daily', '001').save(flush: true, failOnError: true) RefdataCategory.lookupOrCreate('BulkImportListConfig.Frequency', 'Weekly', '007').save(flush: true, failOnError: true) RefdataCategory.lookupOrCreate('BulkImportListConfig.Frequency', 'Monthly', '030').save(flush: true, failOnError: true) diff --git a/server/grails-app/services/org/gokb/DisplayTemplateService.groovy b/server/grails-app/services/org/gokb/DisplayTemplateService.groovy index d505598c0..f13fd4b34 100644 --- a/server/grails-app/services/org/gokb/DisplayTemplateService.groovy +++ b/server/grails-app/services/org/gokb/DisplayTemplateService.groovy @@ -41,6 +41,7 @@ public class DisplayTemplateService { globalDisplayTemplates.put('org.gokb.cred.Folder',[ type:'staticgsp', rendername:'folder' ]); globalDisplayTemplates.put('org.gokb.cred.Work',[ type:'staticgsp', rendername:'work' ]); globalDisplayTemplates.put('org.gokb.cred.BulkImportListConfig',[ type:'staticgsp', rendername:'bulkconfig', noCreate:true ]); + globalDisplayTemplates.put('org.gokb.cred.WebHookEndpoint',[ type:'staticgsp', rendername:'web_hook_endpoint' ]); } public Map getTemplateInfo(String type) { diff --git a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy index 3a7e51791..9b4573a36 100644 --- a/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy +++ b/server/grails-app/services/org/gokb/PackageSourceUpdateService.groovy @@ -3,8 +3,11 @@ package org.gokb import com.k_int.ConcurrencyManagerService.Job import grails.converters.JSON +import grails.util.Environment import groovy.util.logging.Slf4j - +import org.apache.commons.net.ftp.FTPClient +import org.apache.commons.net.ftp.FTPClientConfig +import org.apache.commons.net.ftp.FTPFile import org.apache.http.HttpEntity import org.apache.http.HttpHeaders import org.apache.http.util.EntityUtils @@ -14,7 +17,7 @@ import org.apache.http.client.config.RequestConfig import org.apache.http.impl.client.CloseableHttpClient import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.impl.client.HttpClients - +import java.nio.charset.StandardCharsets import java.security.MessageDigest import java.time.Duration import java.time.LocalDate @@ -24,6 +27,9 @@ import java.util.regex.Pattern import org.gokb.cred.* import org.mozilla.universalchardet.UniversalDetector +import org.apache.commons.net.* + +import java.util.stream.Collectors @Slf4j class PackageSourceUpdateService { @@ -32,10 +38,12 @@ class PackageSourceUpdateService { def validationService WekbIngestionService wekbIngestionService boolean isExternalSourceImportOrUpdate + WebEndpointService webEndpointService static Pattern DATE_PLACEHOLDER_PATTERN = ~/[0-9]{4}-[0-9]{2}-[0-9]{2}/ - static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/ - static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)$/ + static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)(\?.*)?$/ + static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)(\?.*)?$/ + @javax.annotation.PostConstruct def init() { @@ -45,7 +53,7 @@ class PackageSourceUpdateService { def updateFromSource(Long pkgId, def user = null, Job job = null, Long activeGroupId = null, boolean dryRun = false, boolean restrictSize = true) { log.debug("updateFromSource ${pkgId}") def result = [result: 'OK'] - def activeJobs = concurrencyManagerService.getComponentJobs(pkgId) + def activeJobs = concurrencyManagerService?.getComponentJobs(pkgId) if (job || activeJobs?.data?.size() == 0) { log.debug("UpdateFromSource started") @@ -75,6 +83,7 @@ class PackageSourceUpdateService { Boolean deleteMissing = false def pkgInfo = [:] def startTime = new Date() + def ftpUrlParts Package.withNewSession { Package p = Package.get(pid) @@ -101,10 +110,21 @@ class PackageSourceUpdateService { result = wekbIngestionService.startTitleImport(pkgInfo, pkg_source, pkg_plt, pkg_prov, p, job, async, restrictSize) } else { - if (pkg_source?.url) { + def transferMethod = pkg_source?.getTransferMethod() + def rdv_FTP = RefdataCategory.lookup('Source.TransferMethod', 'FTP') + boolean isFtpTransfer = (transferMethod == rdv_FTP) + + if (pkg_source?.url || (isFtpTransfer && pkg_source?.ftpUrl)) { URL src_url = null Boolean dynamic_date = false - def valid_url_string = validationService.checkUrl(pkg_source?.url, true) + String completeFtpUrl = null + + if(isFtpTransfer){ + ftpUrlParts = webEndpointService.extractFtpUrlParts(pkg_source.getWebEndpoint()?.getUrl(), pkg_source.getFtpUrl()) + completeFtpUrl = ftpUrlParts.complete + } + + def valid_url_string = validationService.checkUrl(isFtpTransfer ? completeFtpUrl : pkg_source?.url, true) LocalDate extracted_date skipInvalid = pkg_source.skipInvalid ?: false def file_info = [:] @@ -127,7 +147,9 @@ class PackageSourceUpdateService { src_url = new URL(valid_url_string) } - } else { + + } + else { log.debug("No source URL!") result.result = 'ERROR' result.messageCode = 'kbart.errors.url.invalid' @@ -137,8 +159,7 @@ class PackageSourceUpdateService { return result } - - if (src_url?.getProtocol() in ['http', 'https']) { + if (src_url?.getProtocol() in ['http', 'https'] || isFtpTransfer) { def deposit_token = java.util.UUID.randomUUID().toString() File tmp_file = TSVIngestionService.handleTempFile(deposit_token) def lastRunLocal = pkg_source.lastRun ? pkg_source.lastRun.toInstant().atZone(ZoneId.systemDefault()).toLocalDate() : null @@ -146,102 +167,110 @@ class PackageSourceUpdateService { pkg_source.lastRun = new Date() pkg_source.save(flush: true) - if (!extracted_date || !lastRunLocal || extracted_date > lastRunLocal) { - log.debug("Request initial URL..") - file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + if ( isFtpTransfer ) { + log.debug("Start FTP Update from Source " + pkg_source ) + ftpUrlParts["complete"] = src_url.toString() + file_info = fetchKbartFileFromFTPServer(tmp_file, pkg_source, ftpUrlParts, dynamic_date, extracted_date, lastRunLocal, restrictSize) } + else { // start not-FTP - if (file_info.connectError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.connection' - result.message = "There was an error trying to fetch KBART via URL!" - result.exceptionMsg = file_info.exceptionMsg + if (!extracted_date || !lastRunLocal || extracted_date > lastRunLocal) { + log.debug("Request initial URL..") + file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + } - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + if (file_info.connectError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.connection' + result.message = "There was an error trying to fetch KBART via URL!" + result.exceptionMsg = file_info.exceptionMsg - return result - } + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - if (file_info.fileSizeError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.fileSize' - result.message = "The attached KBART file is too big! Files bigger than 20 MB have to be authorized manually by an administrator." + return result + } - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + if (file_info.fileSizeError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.fileSize' + result.message = "The attached KBART file is too big! Files bigger than 20 MB have to be authorized manually by an administrator." - return result - } + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - if (file_info.accessError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.html' - result.message = "URL returned HTML, indicating provider configuration issues!" + return result + } - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + if (file_info.accessError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.html' + result.message = "URL returned HTML, indicating provider configuration issues!" - return result - } else if (file_info.mimeTypeError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.mimeType' - result.message = "KBART URL returned a wrong content type!" - log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + return result + } else if (file_info.mimeTypeError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.mimeType' + result.message = "KBART URL returned a wrong content type!" + log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") - return result - } else if (file_info.status == 403) { - log.debug("URL request failed!") - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.denied' - result.message = "URL request returned 403 ACCESS DENIED, skipping further tries!" + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + return result + } else if (file_info.status == 403) { + log.debug("URL request failed!") + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.denied' + result.message = "URL request returned 403 ACCESS DENIED, skipping further tries!" - return result - } + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - if (!file_info.file_name && (dynamic_date || extracted_date)) { - LocalDate active_date = LocalDate.now() - boolean skipLookupByDate = false - src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) - log.debug("Fetching dated URL for today..") - file_info = fetchKbartFile(tmp_file, src_url, restrictSize) - - // Look at first of this month - if (!file_info.file_name) { - sleep(500) - log.debug("Fetching first of the month..") - def som_date_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.withDayOfMonth(1).toString())) - file_info = fetchKbartFile(tmp_file, som_date_url, restrictSize) + return result } - // Check all days of this month - while (!skipLookupByDate && active_date.isAfter(LocalDate.now().minusDays(30)) && !file_info.file_name) { - active_date = active_date.minusDays(1) + if (!file_info.file_name && (dynamic_date || extracted_date)) { + LocalDate active_date = LocalDate.now() + boolean skipLookupByDate = false src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) - log.debug("Fetching dated URL for date ${active_date}") - sleep(500) + log.debug("Fetching dated URL for today..") file_info = fetchKbartFile(tmp_file, src_url, restrictSize) - if (file_info.mimeTypeError) { - skipLookupByDate = true + // Look at first of this month + if (!file_info.file_name) { + sleep(500) + log.debug("Fetching first of the month..") + def som_date_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.withDayOfMonth(1).toString())) + file_info = fetchKbartFile(tmp_file, som_date_url, restrictSize) + } + + // Check all days of this month + while (!skipLookupByDate && active_date.isAfter(LocalDate.now().minusDays(30)) && !file_info.file_name) { + active_date = active_date.minusDays(1) + src_url = new URL(src_url.toString().replaceFirst(DATE_PLACEHOLDER_PATTERN, active_date.toString())) + log.debug("Fetching dated URL for date ${active_date}") + sleep(500) + file_info = fetchKbartFile(tmp_file, src_url, restrictSize) + + if (file_info.mimeTypeError) { + skipLookupByDate = true + } } } - } - if (file_info.mimeTypeError) { - result.result = 'ERROR' - result.messageCode = 'kbart.errors.url.mimeType' - result.message = "KBART URL returned a wrong content type!" - log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") + if (file_info.mimeTypeError) { + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.mimeType' + result.message = "KBART URL returned a wrong content type!" + log.error("KBART url ${src_url} returned MIME type ${file_info.content_mime_type} for file ${file_info.file_name}") - result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) - return result - } + return result + } - log.debug("Got mime type ${file_info.content_mime_type} for file ${file_info.file_name}") + log.debug("Got mime type ${file_info.content_mime_type} for file ${file_info.file_name}") + } // end not-FTP if (file_info.file_name) { try { MessageDigest md5_digest = MessageDigest.getInstance("MD5") @@ -331,7 +360,6 @@ class PackageSourceUpdateService { return result } } - // else if (src_url.getProtocol() in ['ftp', 'sftp']) { else { result.result = 'ERROR' result.messageCode = 'kbart.errors.url.protocol' @@ -424,12 +452,12 @@ class PackageSourceUpdateService { Long content_length RequestConfig requestConfig = RequestConfig.custom() - .setConnectionRequestTimeout(10000) - .setSocketTimeout(30000) - .build() + .setConnectionRequestTimeout(10000) + .setSocketTimeout(30000) + .build() HttpClientBuilder builder = HttpClients.custom() - .setDefaultRequestConfig(requestConfig) + .setDefaultRequestConfig(requestConfig) try (CloseableHttpClient httpClient = builder.build()) { HttpHead httpHead = new HttpHead(src_url.toURI()) @@ -490,10 +518,10 @@ class PackageSourceUpdateService { file_name = file_name.replaceAll(/\"/, '') if ((file_name?.trim()?.endsWith('.tsv') || file_name?.trim()?.endsWith('.txt') || file_name?.trim()?.endsWith('.kbart')) && - (result.content_mime_type?.startsWith("text/plain") || - result.content_mime_type?.startsWith("text/csv") || - result.content_mime_type?.startsWith("text/tab-separated-values") || - result.content_mime_type == 'application/octet-stream')) { + (result.content_mime_type?.startsWith("text/plain") || + result.content_mime_type?.startsWith("text/csv") || + result.content_mime_type?.startsWith("text/tab-separated-values") || + result.content_mime_type == 'application/octet-stream')) { HttpEntity entity = classicHttpResponse.getEntity(); result.file_name = file_name @@ -608,4 +636,121 @@ class PackageSourceUpdateService { return info_map } + + def fetchKbartFileFromFTPServer (File tmp_file, Source source, def urlParts, boolean dynamic_date, LocalDate extracted_date, LocalDate lastRunLocal, boolean restrictSize = true) { + + def result = [content_mime_type: null, file_name: null] + Long max_length = 20971520L // 1024 * 1024 * 20 + + FTPClient ftp = new FTPClient() + FTPClientConfig config = new FTPClientConfig() + + String username = source.getWebEndpoint().getEpUsername() + String password = source.getWebEndpoint().getEpPassword() + + String hostname = urlParts.hostname + String directory = urlParts.directory + String filename = urlParts.filename + String foundFileName = null + FTPFile foundFile = null + + if(dynamic_date){ + // in case of dynamic_date, in the filename the pattern is already replaced by the actual date + String[] parts = urlParts.complete.split("/") + filename = parts[parts.length - 1] + } + + try { + // for integration test purpose + if (Environment.current == Environment.TEST) { + ftp.connect(hostname, 12345) + } + else { + ftp.connect(hostname) + } + ftp.enterLocalPassiveMode() + def loggedIn = ftp.login(username, password) + + if (ftp.isConnected()) { + + ftp.changeWorkingDirectory(directory) + List files + + if(dynamic_date || extracted_date) { + + def dateMaskMatch = (filename =~ VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN) + String matchedDate = dateMaskMatch[0][1] + String fixFilenamePart = filename.substring(0, filename.indexOf(matchedDate)) + + files = ftp.listFiles().toList() + List unorderedRes = files.stream().filter(f -> + f.name =~ VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN && f.name.startsWith(fixFilenamePart)) + .collect(Collectors.toList()) + List res = unorderedRes.sort((f1, f2) -> f1.getName().compareTo(f2.getName())) + + // file with latest date is the last in list res + if(res.size() > 0) { + foundFile = res.get(res.size() - 1) + foundFileName = foundFile.name + } + } + else { + files = ftp.listFiles(filename).toList() + if(files.size() > 0) { + foundFile = files.get(0) + foundFileName = filename + } + } + + + if(foundFile){ + LocalDate foundFileDate = LocalDate.ofInstant(foundFile.getTimestampInstant(), ZoneId.systemDefault()) + if(lastRunLocal && (lastRunLocal > foundFileDate)){ + // no update needed + tmp_file.delete() + return result + } + + Long foundFileSize = foundFile.getSize() + if(foundFileSize > max_length && restrictSize){ + result.fileSizeError = true + result.result = 'ERROR' + result.messageCode = 'kbart.errors.url.fileSize' + result.message = "The attached KBART file is too big! Files bigger than 20 MB have to be authorized manually by an administrator." + //result.jobInfo = createJobResult(p, job, startTime, dryRun, user, preferred_group, result) + tmp_file.delete() + return result + } + + result.file_name = foundFileName + + InputStream is = ftp.retrieveFileStream(foundFileName) + OutputStream outStream = new FileOutputStream(tmp_file) + + byte[] buffer = new byte[1024]; + for (int length; (length = is.read(buffer)) != -1; ) { + outStream.write(buffer, 0, length); + } + + outStream.close() + + } + else { + // no filename --> handled in calling method + + } + + ftp.logout() + ftp.disconnect() + } + + } catch (Exception e) { + log.error("Fehler bei FTP-Verbindung ", e) + + } + + return result + + } + } \ No newline at end of file diff --git a/server/grails-app/services/org/gokb/ValidationService.groovy b/server/grails-app/services/org/gokb/ValidationService.groovy index 310da94c2..54f88c714 100644 --- a/server/grails-app/services/org/gokb/ValidationService.groovy +++ b/server/grails-app/services/org/gokb/ValidationService.groovy @@ -6,6 +6,7 @@ import com.opencsv.CSVReader import com.opencsv.CSVReaderBuilder import com.opencsv.CSVParser import com.opencsv.CSVParserBuilder +import grails.util.Environment import grails.validation.ValidationException import java.time.LocalDate import org.apache.commons.io.ByteOrderMark @@ -925,7 +926,13 @@ class ValidationService { // log.debug("Final URL to check: ${final_val}") - return new UrlValidator().isValid(final_val) ? value : null + // needed for FTP integration Tests to allow FTP URLS from FTP Mock Server + if (Environment.current == Environment.TEST) { + return new UrlValidator(UrlValidator.ALLOW_LOCAL_URLS).isValid(final_val) ? value : null + } + else { + return new UrlValidator().isValid(final_val) ? value : null + } } private String encodeUrlPart(String value) { diff --git a/server/grails-app/services/org/gokb/WebEndpointService.groovy b/server/grails-app/services/org/gokb/WebEndpointService.groovy new file mode 100644 index 000000000..39c9ce69e --- /dev/null +++ b/server/grails-app/services/org/gokb/WebEndpointService.groovy @@ -0,0 +1,67 @@ +package org.gokb + +import grails.gorm.transactions.Transactional + +import java.util.regex.Pattern + +@Transactional +class WebEndpointService { + + def extractFtpUrlParts (String webEndpointUrl, String sourceUrl) { + def result = [:] + + String protocol = "" + if(webEndpointUrl.startsWith("ftp://")){ + protocol = "ftp://" + } + else if(webEndpointUrl.startsWith("ftps://")){ + protocol = "ftps://" + } + + //we dont need the protocol + String hostname = webEndpointUrl.replace(protocol, "") + String filename = "" + String directory = "/" + String completeUrl = "" + + // leading and trailing slashes that could be part of host or filename + // are set in the directory part + if (hostname?.contains("/")) { + String[] parts = hostname.split("/") + hostname = parts[0] + for(int i = 1; i < parts.length; i++){ + directory = directory.concat(parts[i] + "/") + } + } + + if (sourceUrl?.startsWith("/")) { + sourceUrl = sourceUrl.substring(1) + } + + if(sourceUrl?.contains("/")){ + String[] parts = sourceUrl.split("/") + filename = parts[parts.length - 1] + directory = directory + sourceUrl.substring(0, sourceUrl.lastIndexOf("/") + 1) + } + else { + filename = sourceUrl + } + + completeUrl = protocol + hostname + directory + filename + + result.hostname = hostname + result.filename = filename + result.directory = directory + result.complete = completeUrl + + + result + + } + + + + + + +} diff --git a/server/grails-app/services/org/gokb/WekbIngestionService.groovy b/server/grails-app/services/org/gokb/WekbIngestionService.groovy index 21f67fb5c..5ac874ee8 100644 --- a/server/grails-app/services/org/gokb/WekbIngestionService.groovy +++ b/server/grails-app/services/org/gokb/WekbIngestionService.groovy @@ -93,8 +93,8 @@ class WekbIngestionService { where c.fromComponent.id = :pkg and c.toComponent = tipp and c.type = :ct - and tipp.status = :sc''', - [pkg: pkg.id, ct: combo_pkg, sc: rdv_current])[0] + and tipp.status != :sd''', + [pkg: pkg.id, ct: combo_pkg, sd: rdv_deleted])[0] result.report = [ numRows : titleCount, diff --git a/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp new file mode 100644 index 000000000..36a28e90b --- /dev/null +++ b/server/grails-app/views/apptemplates/_web_hook_endpoint.gsp @@ -0,0 +1,22 @@ +
+
Name
+
+ +
Base URL
+
+ +
Benutzername
+
+ +
Passwort
+ +
+ +
+ + \ No newline at end of file diff --git a/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy new file mode 100644 index 000000000..57b0a2c8c --- /dev/null +++ b/server/src/integration-test/groovy/org/gokb/UpdatePackageRunFTPSpec.groovy @@ -0,0 +1,274 @@ +package org.gokb + +import com.k_int.ConcurrencyManagerService +import grails.core.GrailsApplication +import grails.gorm.transactions.Rollback +import grails.testing.mixin.integration.Integration +import org.apache.commons.io.IOUtils +import org.apache.commons.net.ftp.FTPClient +import org.apache.commons.net.ftp.FTPFile +import org.gokb.cred.CuratoryGroup +import org.gokb.cred.Org +import org.gokb.cred.Package +import org.gokb.cred.Platform +import org.gokb.cred.RefdataCategory +import org.gokb.cred.Source +import org.gokb.cred.WebHookEndpoint +import org.mockftpserver.fake.* +import org.mockftpserver.fake.filesystem.* +import org.springframework.core.io.ClassPathResource +import spock.lang.Specification +import java.nio.charset.Charset + +@Integration +@Rollback +class UpdatePackageRunFTPSpec extends Specification{ + + GrailsApplication grailsApplication + ConcurrencyManagerService concurrencyManagerService + PackageSourceUpdateService packageSourceUpdateService + + + private static final String HOME_DIR = "/"; + private static final String NON_KBART_FILE = "/dir/sample.txt"; + private static final String CONTENTS = "abcdef 1234567890"; + private static final String USER = "user"; + private static final String PASSWORD = "password"; + private static final int MOCKSERVER_PORT = 12345 + private FakeFtpServer ftpServer + private CuratoryGroup new_cg + + + + + def setup() { + + if(ftpServer == null || !ftpServer.isStarted()) { + ftpServer = new FakeFtpServer() + ftpServer.setServerControlPort(MOCKSERVER_PORT) + FileSystem fileSystem = new UnixFakeFileSystem() + fileSystem.add(new FileEntry(NON_KBART_FILE, CONTENTS)) + ftpServer.setFileSystem(fileSystem) + ftpServer.addUserAccount(new UserAccount(USER, PASSWORD, HOME_DIR)) + + ftpServer.start() + } + + new_cg = CuratoryGroup.findByName('TestGroup1') ?: new CuratoryGroup(name: "TestGroup1").save(flush: true) + + Org provider = Org.findByName("American Chemical Society") ?: new Org(name: "American Chemical Society").save(flush: true) + Platform.findByName("Test Platform") ?: new Platform(name: "Test Platform", primaryUrl: "https://search.ebscohost.com", provider: provider).save(flush: true) + WebHookEndpoint.findByName("whe1") ?: new WebHookEndpoint(name: "whe1" ,url: "ftp://localhost/dir", epUsername: USER, epPassword: PASSWORD).save(flush: true) + Source.findByName("source1") ?: new Source(name: "source1", webEndpoint: WebHookEndpoint.findByName("whe1"), ftpUrl: "/kbart.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) + Package.findByName("package1") ?: new Package(name: "package1").save(flush: true) + Package.findByName("package1").setSource(Source.findByName("source1")) + Package.findByName("package1").save(flush: true) + + WebHookEndpoint.findByName("whe2") ?: new WebHookEndpoint(name: "whe2" ,url: "ftp://localhost/dir", epUsername: USER, epPassword: PASSWORD).save(flush: true) + Source.findByName("source2") ?: new Source(name: "source2", webEndpoint: WebHookEndpoint.findByName("whe2"), ftpUrl: "/kbart_not_exists.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) + Package.findByName("package2") ?: new Package(name: "package2").save(flush: true) + Package.findByName("package2").setSource(Source.findByName("source2")) + Package.findByName("package2").save(flush: true) + + WebHookEndpoint.findByName("whe3") ?: new WebHookEndpoint(name: "whe3" ,url: "ftp://localhost/dir", epUsername: USER, epPassword: PASSWORD).save(flush: true) + Source.findByName("source3") ?: new Source(name: "source3", webEndpoint: WebHookEndpoint.findByName("whe3"), ftpUrl: "/kbart_{YYYY-MM-DD}.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) + Package.findByName("package3") ?: new Package(name: "package3").save(flush: true) + Package.findByName("package3").setSource(Source.findByName("source3")) + Package.findByName("package3").save(flush: true) + + WebHookEndpoint.findByName("whe4") ?: new WebHookEndpoint(name: "whe4" ,url: "ftp://localhost/dir", epUsername: USER, epPassword: PASSWORD).save(flush: true) + Source.findByName("source4") ?: new Source(name: "source4", webEndpoint: WebHookEndpoint.findByName("whe4"), ftpUrl: "/kbart_2026-01-11.txt", transferMethod: RefdataCategory.lookup('Source.TransferMethod', 'FTP')).save(flush: true) + Package.findByName("package4") ?: new Package(name: "package4").save(flush: true) + Package.findByName("package4").setSource(Source.findByName("source4")) + Package.findByName("package4").save(flush: true) + + } + + def cleanup() { + + ftpServer?.stop() + + CuratoryGroup.findByName('TestGroup1')?.expunge() + Org.findByName("American Chemical Society")?.expunge() + Platform.findByName("Test Platform")?.expunge() + + Package.findByName("package1")?.expunge() + Source.findByName("source1")?.expunge() + WebHookEndpoint.findByName("whe1")?.delete() + + Package.findByName("package2")?.expunge() + Source.findByName("source2")?.expunge() + WebHookEndpoint.findByName("whe2")?.delete() + + Package.findByName("package3")?.expunge() + Source.findByName("source3")?.expunge() + WebHookEndpoint.findByName("whe3")?.delete() + + Package.findByName("package4")?.expunge() + Source.findByName("source4")?.expunge() + WebHookEndpoint.findByName("whe4")?.delete() + + } + + void "test Mock FTP Server runs"() { + + FTPClient ftpClient = new FTPClient() + ftpClient.connect("localhost", ftpServer.getServerControlPort()) + boolean loggedin = ftpClient.login(USER, PASSWORD); + + expect: "FTP Server runs and can be connected by client" + + ftpServer.getServerControlPort() == MOCKSERVER_PORT + loggedin + ftpClient.logout() + } + + void "Test updateFromSource :: usual initial direct Import by filename"() { + + given: "requested Kbart exists without date mask" + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart.txt", fileContent)) + } + + Package p = Package.findByName("package1") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + + expect: "file is found, package titles imported" + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 + + } + + void "Test updateFromSource :: KBART does not exist on Server"() { + given: "configured KBART file does not exist on server" + Package p = Package.findByName("package2") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + expect: + result.result == 'SKIPPED' + result.messageCode == 'kbart.transmission.skipped.noFile' + p.getTitles(true, 10, 0).size() == 0 + + } + + + + void "Test updateFromSource :: KBART with date mask is found and imported"() { + given: "FTP URL with date mask configured, file with date exists" + + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-01-01.txt", fileContent)) + } + Package p = Package.findByName("package3") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + expect: + + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 + + } + + + void "Test updateFromSource :: KBART with latest date mask is found"() { + given: "FTP URL with date mask configured, several files with dates exist" + // source.ftpUrl = "kbart_{YYYY-MM-DD}.txt" + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + String fakeContent = "AAAAAAAAAAAA" + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-02-01.txt", fileContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-01-01.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2023-10-30.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2025-12-24.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/other_package_2026-02-02.txt", fakeContent)) + } + //same WHE as in previous test + Package p = Package.findByName("package3") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + expect: "find the file with the latest date, i.e. the only file with KBART content" + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 + } + + + + void "Test updateFromSource :: KBART with specified date is found"() { + given: "FTP URL with specific date configured, several files with dates exist" + // source.ftpUrl = "kbart_2026-01-11.txt" + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + String fakeContent = "AAAAAAAAAAAA" + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-02-01.txt", fileContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-01-01.txt", "fakeContent")) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2023-10-30.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2025-12-24.txt", fakeContent)) + ftpServer.getFileSystem().add(new FileEntry("/dir/other_package_2026-02-02.txt", fakeContent)) + } + Package p = Package.findByName("package4") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + expect: "find the file with the latest date, i.e. the only file with KBART content" + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 + } + + + void "Test updateFromSource :: Package is updated"() { + given: "Package is imported initially" + //source.ftpUrl = "/kbart_{YYYY-MM-DD}.txt" + def kbart_file = new ClassPathResource("/test_ftp_kbart_update.txt") + try(FileInputStream fis = new FileInputStream(kbart_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-02-01.txt", fileContent)) + } + // source with date mask + Package p = Package.findByName("package3") + def result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + + expect: "Package should be imported correctly" + + result.result == 'OK' + def titles = p.getTitles(true, 100, 0) + titles.size() == 2 + + def kbart_update_file = new ClassPathResource("/test_ftp_kbart_update_new_file.txt") + try(FileInputStream fis = new FileInputStream(kbart_update_file.getFile())){ + String fileContent = IOUtils.toString(fis, Charset.defaultCharset()) + ftpServer.getFileSystem().add(new FileEntry("/dir/kbart_2026-02-03.txt", fileContent)) + } + + def update_result = packageSourceUpdateService.updateFromSource(p.id, null, null, new_cg.id, false, true) + and: "Package should be updated by a new file" + update_result.result == 'OK' + p.getTitles(true, 100, 0).size() == 3 + + } + + + void "test WebEndpointService extractUrlParts :: split parts and format them correct"() { + + when: "The Slashes are not set correctly in URL Field" + String endpointUrl = "ftp://servername.com/home/" + String sourceUrl = "/filename.txt" + + def res = new WebEndpointService().extractFtpUrlParts(endpointUrl, sourceUrl) + + then: "The URL is normalized and splitted correctly into parts" + + res.hostname == "servername.com" + res.directory == "/home/" + res.filename == "filename.txt" + res.complete == "ftp://servername.com/home/filename.txt" + + } + + + +} diff --git a/server/src/integration-test/resources/test_ftp_kbart_update.txt b/server/src/integration-test/resources/test_ftp_kbart_update.txt new file mode 100644 index 000000000..78760b9ab --- /dev/null +++ b/server/src/integration-test/resources/test_ftp_kbart_update.txt @@ -0,0 +1,3 @@ +publication_title print_identifier online_identifier date_first_issue_online num_first_vol_online num_first_issue_online date_last_issue_online num_last_vol_online num_last_issue_online title_url first_author title_id embargo_info coverage_depth notes publisher_name publication_type date_monograph_published_print date_monograph_published_online monograph_volume monograph_edition first_editor parent_publication_title_id preceding_publication_title_id access_type package_name package_id ebsco_resource_type +Arc Poetry Magazine 1910-3239 2008-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BGAO&scope=site BGAO fulltext ARC: Canada's National Poetry Magazine serial P Canadian Literary Centre cjh Periodical +Books in Canada 0045-2564 1971-05-01 2008-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=4TI&scope=site 4TI fulltext Canadian Review of Books Ltd. serial P Canadian Literary Centre cjh Periodical diff --git a/server/src/integration-test/resources/test_ftp_kbart_update_new_file.txt b/server/src/integration-test/resources/test_ftp_kbart_update_new_file.txt new file mode 100644 index 000000000..657f0ff43 --- /dev/null +++ b/server/src/integration-test/resources/test_ftp_kbart_update_new_file.txt @@ -0,0 +1,4 @@ +publication_title print_identifier online_identifier date_first_issue_online num_first_vol_online num_first_issue_online date_last_issue_online num_last_vol_online num_last_issue_online title_url first_author title_id embargo_info coverage_depth notes publisher_name publication_type date_monograph_published_print date_monograph_published_online monograph_volume monograph_edition first_editor parent_publication_title_id preceding_publication_title_id access_type package_name package_id ebsco_resource_type +Arc Poetry Magazine 1910-3239 2008-01-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=BGAO&scope=site BGAO fulltext ARC: Canada's National Poetry Magazine serial P Canadian Literary Centre cjh Periodical +Books in Canada 0045-2564 1971-05-01 2008-03-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=4TI&scope=site 4TI fulltext Canadian Review of Books Ltd. serial P Canadian Literary Centre cjh Periodical +Canadian Journal of Film Studies 0847-5911 2003-09-01 2024-09-01 https://search.ebscohost.com/direct.asp?db=cjh&jid=I2G&scope=site I2G abstracts University of Toronto Press serial P Canadian Literary Centre cjh Academic Journal \ No newline at end of file