Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
f0ff987
ftp initial
christophkohl1 Aug 15, 2025
39977e6
add GSP for creating endpoints in legacy UI
christophkohl1 Aug 19, 2025
fb8044e
apply create and search functionality for WebHookEndpoint to old UI
christophkohl1 Aug 20, 2025
2bb2e5b
controller
christophkohl1 Aug 22, 2025
3c4d8de
add RDV for source data transfer methods
christophkohl1 Aug 27, 2025
2890c25
changes
christophkohl1 Aug 29, 2025
0d5e739
changes
christophkohl1 Oct 1, 2025
a684a01
Merge branch 'g5_master' into 688-ftp-update
christophkohl1 Nov 12, 2025
b0a3a80
clean FTP Urls
christophkohl1 Nov 21, 2025
9972470
handle FTP Urls
christophkohl1 Nov 24, 2025
a64b6c7
Merge branch 'g5_master' into 688-ftp-update
christophkohl1 Nov 24, 2025
c2f7597
add test ftp connection functionality
christophkohl1 Nov 26, 2025
6dac257
added missing Service class
christophkohl1 Nov 28, 2025
99bae79
notes
christophkohl1 Nov 28, 2025
5c2a260
remove unneeded field
christophkohl1 Dec 3, 2025
643d11c
remove pw encryption
christophkohl1 Dec 3, 2025
8ce35cc
Merge branch 'g5_master' into 688-ftp-update
christophkohl1 Dec 8, 2025
e1ebeda
merge
christophkohl1 Dec 8, 2025
e85792e
Merge branch 'g5_master' into 688-ftp-update
christophkohl1 Dec 17, 2025
658fc9a
merge
christophkohl1 Dec 17, 2025
be1bc88
validator url
christophkohl1 Dec 18, 2025
8918e3c
datemask
christophkohl1 Dec 19, 2025
2290b14
chnages
christophkohl1 Jan 22, 2026
0deed72
refactor
christophkohl1 Jan 26, 2026
b3666e5
Merge branch 'g5_master' into 688-ftp-update
christophkohl1 Jan 26, 2026
44571cc
refactor
christophkohl1 Jan 26, 2026
4f6ee75
mock ftp
christophkohl1 Jan 28, 2026
7ebb8ce
test
christophkohl1 Jan 29, 2026
3e0d664
test ftp
christophkohl1 Jan 30, 2026
93759a8
integration tests
christophkohl1 Feb 4, 2026
65a5cf3
Merge branch 'g5_master' into 688-ftp-update
christophkohl1 Feb 4, 2026
98837f2
- change authentication for webendpoint controller mehtods
christophkohl1 Feb 5, 2026
db9dca6
refactor
christophkohl1 Feb 5, 2026
4ae7d8e
refactor test
christophkohl1 Feb 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions server/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ dependencies {
implementation 'org.apache.tika:tika-core:3.2.3'
implementation 'dk.glasius:external-config:4.0.0'

implementation 'commons-net:commons-net:3.12.0'
implementation 'org.apache.httpcomponents:httpclient:4.5.14'

implementation "io.micronaut:micronaut-http-client"
Expand All @@ -138,6 +139,8 @@ dependencies {
testImplementation "org.seleniumhq.selenium:selenium-remote-driver:$seleniumVersion"
testImplementation "org.seleniumhq.selenium:selenium-api:$seleniumVersion"
testImplementation "org.seleniumhq.selenium:selenium-support:$seleniumVersion"

testImplementation 'org.mockftpserver:MockFtpServer:3.2.0'
}

bootRun {
Expand Down
24 changes: 24 additions & 0 deletions server/grails-app/conf/application.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -1273,6 +1273,30 @@ globalSearchTemplates = [
]
]
],
'WebHookEndpoints':[
baseclass:'org.gokb.cred.WebHookEndpoint',
title:'Web Endpoints',
group:'Secondary',
defaultSort:'id',
defaultOrder:'desc',
qbeConfig:[
qbeForm:[
[
prompt:'Name',
qparam:'qp_name',
placeholder:'Name of Web Endpoint',
contextTree:['ctxtp':'qry', 'comparator' : 'ilike', 'prop':'name','wildcard':'R']
],
],
qbeGlobals:[
],
qbeResults:[
[heading:'Name', property:'name', link:[controller:'resource', action:'show', id:'x.r.uuid'] ],
[heading:'URL', property:'url'],
[heading:'Methode', property:'supplyMethod'],
]
]
],

]

Expand Down
1 change: 1 addition & 0 deletions server/grails-app/conf/spring/resources.groovy
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import com.k_int.UserPasswordEncoderListener

import com.k_int.utils.DatabaseMessageSource;
import org.grails.spring.context.support.PluginAwareResourceBundleMessageSource;
import org.springframework.web.servlet.i18n.SessionLocaleResolver
Expand Down
4 changes: 4 additions & 0 deletions server/grails-app/controllers/org/gokb/UrlMappings.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,10 @@ class UrlMappings {
get "/jobs/$id" (controller: 'jobs', namespace: 'rest', action: 'show')
patch "/jobs/$id/cancel" (controller: 'jobs', namespace: 'rest', action: 'cancel')
delete "/jobs/$id" (controller: 'jobs', namespace: 'rest', action: 'delete')

get "/web-endpoint"(controller: 'webEndpoint', namespace: 'rest', action: 'index')
get "/web-endpoint/$id"(controller: 'webEndpoint', namespace: 'rest', action: 'show')
post "/web-endpoint/check"(controller: 'webEndpoint', namespace: 'rest', action: 'check')
}
"/$controller/$action?/$id?" {
constraints {
Expand Down
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All of the methods in this controller should be restricted to ROLE_CONTRIBUTOR.

Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
package org.gokb.rest

import grails.converters.JSON
import grails.plugin.springsecurity.annotation.Secured
import org.apache.commons.net.ftp.FTPClient
import org.apache.commons.net.ftp.FTPClientConfig
import org.apache.commons.net.ftp.FTPFile
import org.gokb.WebEndpointService
import org.gokb.cred.User
import org.gokb.cred.WebHookEndpoint

import java.time.Duration
import java.time.LocalDateTime
import java.util.regex.Pattern

class WebEndpointController {

static namespace = 'rest'

def componentLookupService
def springSecurityService
WebEndpointService webEndpointService

static Pattern FIXED_DATE_ENDING_PLACEHOLDER_PATTERN = ~/\{YYYY-MM-DD\}\.(tsv|txt)$/
static Pattern VARIABLE_DATE_ENDING_PLACEHOLDER_PATTERN = ~/([12][0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]))\.(tsv|txt)$/

@Secured(value = ["hasRole('ROLE_CONTRIBUTOR')", 'IS_AUTHENTICATED_FULLY'])
def index() {
def result = [:]
def base = grailsApplication.config.getProperty('grails.serverURL') + "/rest"
User user = null

if (springSecurityService.isLoggedIn()) {
user = User.get(springSecurityService.principal?.id)
}

// params['_embed'] = params['_embed'] ?: 'identifiedComponents'

result = componentLookupService.restLookup(user, WebHookEndpoint, params)

if (result.data) {
def resultList = result.data
resultList*.remove('epPassword')
resultList*.remove('epUsername')

if (params['method']) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Filtering after the fact is not permissible, since it may invalidate the pagination info. Custom logic should be added in componentLookupService.restLookup if necessary.

//resultList = resultList.findAll( x -> x.transferMethod?.name == params['method'])
resultList = resultList.findAll( x -> x.url.startsWith(params['method'].toLowerCase()) )
}

result.data = resultList
}

render result as JSON
}

@Secured(value = ["hasRole('ROLE_CONTRIBUTOR')", 'IS_AUTHENTICATED_FULLY'])
def show() {
def result = [:]
def base = grailsApplication.config.getProperty('grails.serverURL') + "/rest"
User user = null

if (springSecurityService.isLoggedIn()) {
user = User.get(springSecurityService.principal?.id)
}
def start_db = LocalDateTime.now()

// params['_embed'] = params['_embed'] ?: 'identifiedComponents'

result = componentLookupService.restLookup(user, WebHookEndpoint, params)

def resultList = result.data

if(resultList.size() > 0){
resultList*.remove('epPassword')
resultList*.remove('epUsername')

result.data = resultList.get(0)
}

render result as JSON
}

@Secured(value = ["hasRole('ROLE_CONTRIBUTOR')", 'IS_AUTHENTICATED_FULLY'])
def check() {
def result = [:]
def reqBody = request.JSON

WebHookEndpoint whe = WebHookEndpoint.findById(reqBody.webhookendpoint)
String path = reqBody.url
String hostname = ""
String directory = ""
String filename = ""

if(whe){
def parts= webEndpointService.extractFtpUrlParts(whe.getUrl(), path)
hostname = parts.hostname
directory = parts.directory
filename = parts.filename
}

def dateMaskMatch = (filename =~ FIXED_DATE_ENDING_PLACEHOLDER_PATTERN)

FTPClient ftp = new FTPClient()
FTPClientConfig config = new FTPClientConfig()

try {
ftp.connect(hostname)
ftp.enterLocalPassiveMode()
def loggedIn = ftp.login(whe.getEpUsername(), whe.getEpPassword())

if (ftp.isConnected()) {

FTPFile[] files
if(dateMaskMatch.size() > 0) {
String fixedPart = filename.split("\\{")[0]
files = ftp.listFiles(directory)

boolean found = false

for (FTPFile file : files) {

if(file.name.startsWith(fixedPart)){
result.result = "success"
result.message = "dateMaskFound"
found = true
break
}
}
if(!found){
result.result = "error"
result.message = "dateMaskNotFound"
}
}
else {
files = ftp.listFiles(directory + filename)
if (files.length > 0 && files[0].size > 0) {

result.result = "success"
result.message = "success"
} else {
result.result = "error"
result.message = "partlySuccessful"
}

ftp.logout()
ftp.disconnect()
}
}
else {
result.result = "error"
result.message = "connectError"
}

} catch (Exception e) {
log.error("Fehler bei FTP-Verbindung: ", e)
result.result = "error"
result.message = "configurationError"
}

render result as JSON

}




}
6 changes: 6 additions & 0 deletions server/grails-app/domain/org/gokb/cred/Source.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ class Source extends KBComponent {
BulkImportListConfig bulkConfig
RefdataValue importConfig
Boolean ignoreSizeLimit = false
WebHookEndpoint webEndpoint
String ftpUrl //umbenennen in ftpPath
RefdataValue transferMethod

static manyByCombo = [
curatoryGroups: CuratoryGroup
Expand Down Expand Up @@ -58,6 +61,9 @@ class Source extends KBComponent {
bulkConfig(nullable: true, blank: false)
importConfig(nullable: true, blank: true)
ignoreSizeLimit(nullable: true, blank: true)
webEndpoint(nullable: true, blank: true)
ftpUrl(nullable: true, blank: true)
transferMethod(nullable: true, blank: true)
}

public static final String restPath = "/sources"
Expand Down
33 changes: 29 additions & 4 deletions server/grails-app/domain/org/gokb/cred/WebHookEndpoint.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,14 @@ import groovy.util.logging.*
class WebHookEndpoint {
String name
String url
Long authmethod
String principal
String credentials
Long authmethod //legacy
RefdataValue supplyMethod //legacy
// RefdataValue transferMethod //rausnehmen - nur in Source
String principal //legacy
String credentials //legacy
User owner
String epUsername
String epPassword

static mapping = {
url column:'ep_url'
Expand All @@ -21,12 +25,33 @@ class WebHookEndpoint {

static constraints = {
name(nullable:false, blank:false)
url(nullable:false, blank:false)
url(validator: {val, obj ->
if(val) {
if(!val.startsWith("ftp://") && !val.startsWith("http://") && !val.startsWith("https://")){
return ['webEndpointUrl.missingProtocol']
}
}
else {
return ['webEndpointUrl.notNull']
}
})
authmethod(nullable:true, blank:true)
principal(nullable:true, blank:true)
credentials(nullable:true, blank:true)
supplyMethod(nullable:true, blank:true)
epUsername(nullable:true, blank:true)
epPassword(nullable:true, blank:true)
}

static jsonMapping = [
'ignore' : [
'epPassword',
'epUsername',
'credentials',
'authmethod'
]
]

static def refdataFind(params) {

log.debug("refdataFind(${params})");
Expand Down
3 changes: 3 additions & 0 deletions server/grails-app/i18n/messages.properties
Original file line number Diff line number Diff line change
Expand Up @@ -540,3 +540,6 @@ componentPrice.priceType.nullable.error = The kind of price must be set.

admin.support.sizeLimit.email.subject = 'GOKB - Import failed due to active KBART size limit'
admin.support.sizeLimit.email.line1 = 'An import for the following package failed due to the default KBART size limit:'

webEndpointUrl.missingProtocol = 'Please provide the URL including the protocol (e.g., ftp:// or http://).'
webEndpointUrl.notNull = 'Please enter a URL.'
5 changes: 4 additions & 1 deletion server/grails-app/i18n/messages_de.properties
Original file line number Diff line number Diff line change
Expand Up @@ -420,4 +420,7 @@ typeMismatch.java.math.BigDecimal=Die Eigenschaft {0} muss eine gültige Zahl se
typeMismatch.java.math.BigInteger=Die Eigenschaft {0} muss eine gültige Zahl sein

admin.support.sizeLimit.email.subject = 'GOKB - KBART-Import aufgrund von Dateigrößenbeschränkung fehlgeschlagen'
admin.support.sizeLimit.email.line1 = 'Ein KBART-Import für das folgende Paket ist aufgrund der aktiven Größenbeschränkung fehlgeschlagen:'
admin.support.sizeLimit.email.line1 = 'Ein KBART-Import für das folgende Paket ist aufgrund der aktiven Größenbeschränkung fehlgeschlagen:'

webEndpointUrl.missingProtocol = 'Bitte geben Sie die URL mitsamt Protokoll (z.B. ftp:// oder http://) an.'
webEndpointUrl.notNull = 'Bitte geben Sie eine URL an.'
3 changes: 3 additions & 0 deletions server/grails-app/init/org/gokb/BootStrap.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -1122,6 +1122,9 @@ class BootStrap {
RefdataCategory.lookupOrCreate('Source.Frequency', 'Quarterly', '090').save(flush: true, failOnError: true)
RefdataCategory.lookupOrCreate('Source.Frequency', 'Yearly', '365').save(flush: true, failOnError: true)

RefdataCategory.lookupOrCreate('Source.TransferMethod', 'HTTP').save(flush: true, failOnError: true)
RefdataCategory.lookupOrCreate('Source.TransferMethod', 'FTP').save(flush: true, failOnError: true)

RefdataCategory.lookupOrCreate('BulkImportListConfig.Frequency', 'Daily', '001').save(flush: true, failOnError: true)
RefdataCategory.lookupOrCreate('BulkImportListConfig.Frequency', 'Weekly', '007').save(flush: true, failOnError: true)
RefdataCategory.lookupOrCreate('BulkImportListConfig.Frequency', 'Monthly', '030').save(flush: true, failOnError: true)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public class DisplayTemplateService {
globalDisplayTemplates.put('org.gokb.cred.Folder',[ type:'staticgsp', rendername:'folder' ]);
globalDisplayTemplates.put('org.gokb.cred.Work',[ type:'staticgsp', rendername:'work' ]);
globalDisplayTemplates.put('org.gokb.cred.BulkImportListConfig',[ type:'staticgsp', rendername:'bulkconfig', noCreate:true ]);
globalDisplayTemplates.put('org.gokb.cred.WebHookEndpoint',[ type:'staticgsp', rendername:'web_hook_endpoint' ]);
}

public Map getTemplateInfo(String type) {
Expand Down
Loading