From 467aa909f30855aaa70fe2168685bbae22b2133e Mon Sep 17 00:00:00 2001 From: mohamedlajmileanix Date: Thu, 31 Oct 2024 11:28:30 +0100 Subject: [PATCH 1/4] CID-3105: React to manifest file creation via Webhook --- .../config/GitHubEnterpriseProperties.kt | 1 - .../githubagent/dto/ManifestFileUpdateDto.kt | 3 +- .../services/GitHubScanningService.kt | 6 +- .../services/WebhookEventService.kt | 80 ++++------ .../leanix/githubagent/shared/Constants.kt | 6 +- .../services/WebhookEventServiceTest.kt | 140 +++++++++++++----- 6 files changed, 134 insertions(+), 102 deletions(-) diff --git a/src/main/kotlin/net/leanix/githubagent/config/GitHubEnterpriseProperties.kt b/src/main/kotlin/net/leanix/githubagent/config/GitHubEnterpriseProperties.kt index bf93f3b..c1f5762 100644 --- a/src/main/kotlin/net/leanix/githubagent/config/GitHubEnterpriseProperties.kt +++ b/src/main/kotlin/net/leanix/githubagent/config/GitHubEnterpriseProperties.kt @@ -7,6 +7,5 @@ data class GitHubEnterpriseProperties( val baseUrl: String, val gitHubAppId: String, val pemFile: String, - val manifestFileDirectory: String, val webhookSecret: String ) diff --git a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFileUpdateDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFileUpdateDto.kt index 35b85d0..db0374f 100644 --- a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFileUpdateDto.kt +++ b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFileUpdateDto.kt @@ -3,7 +3,8 @@ package net.leanix.githubagent.dto data class ManifestFileUpdateDto( val repositoryFullName: String, val action: ManifestFileAction, - val manifestFileContent: String? + val manifestFileContent: String?, + val manifestFilePath: String? ) enum class ManifestFileAction { diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt index 93bd6d0..cb6cc35 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt @@ -12,7 +12,7 @@ import net.leanix.githubagent.dto.RepositoryDto import net.leanix.githubagent.dto.Trigger import net.leanix.githubagent.exceptions.JwtTokenNotFound import net.leanix.githubagent.exceptions.ManifestFileNotFoundException -import net.leanix.githubagent.shared.ManifestFileName +import net.leanix.githubagent.shared.MANIFEST_FILE_NAME import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.util.UUID @@ -132,7 +132,7 @@ class GitHubScanningService( gitHubClient.searchManifestFiles( "Bearer $installationToken", "" + - "repo:${installation.account.login}/$repositoryName filename:${ManifestFileName.YAML.fileName}" + "repo:${installation.account.login}/$repositoryName filename:$MANIFEST_FILE_NAME" ) } private fun fetchManifestContents( @@ -150,7 +150,7 @@ class GitHubScanningService( ) if (content != null) { ManifestFileDTO( - path = manifestFile.path.replace("/${ManifestFileName.YAML.fileName}", ""), + path = manifestFile.path.replace("/$MANIFEST_FILE_NAME", ""), content = content ) } else { diff --git a/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt b/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt index 180f4b6..74c1908 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt @@ -2,12 +2,11 @@ package net.leanix.githubagent.services import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import net.leanix.githubagent.config.GitHubEnterpriseProperties import net.leanix.githubagent.dto.ManifestFileAction import net.leanix.githubagent.dto.ManifestFileUpdateDto import net.leanix.githubagent.dto.PushEventCommit import net.leanix.githubagent.dto.PushEventPayload -import net.leanix.githubagent.shared.ManifestFileName +import net.leanix.githubagent.shared.MANIFEST_FILE_NAME import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -15,7 +14,6 @@ import org.springframework.stereotype.Service class WebhookEventService( private val webSocketService: WebSocketService, private val gitHubGraphQLService: GitHubGraphQLService, - private val gitHubEnterpriseProperties: GitHubEnterpriseProperties, private val cachingService: CachingService, private val gitHubAuthenticationService: GitHubAuthenticationService ) { @@ -53,7 +51,6 @@ class WebhookEventService( } } - @SuppressWarnings("LongParameterList") private fun handleManifestFileChanges( headCommit: PushEventCommit, repositoryFullName: String, @@ -61,39 +58,34 @@ class WebhookEventService( repositoryName: String, installationToken: String ) { - val yamlFileName = "${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YAML.fileName}" - val ymlFileName = "${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YML.fileName}" + val addedManifestFiles = headCommit.added.filter { it.contains(MANIFEST_FILE_NAME) } + val modifiedManifestFiles = headCommit.modified.filter { it.contains(MANIFEST_FILE_NAME) } + val removedManifestFiles = headCommit.removed.filter { it.contains(MANIFEST_FILE_NAME) } - val isYAMLFileUpdated = isManifestFileUpdated(headCommit, yamlFileName) - val isYMLFileUpdated = isManifestFileUpdated(headCommit, ymlFileName) + addedManifestFiles.forEach { filePath -> + handleAddedOrModifiedManifestFile( + repositoryFullName, + owner, + repositoryName, + installationToken, + filePath, + ManifestFileAction.ADDED + ) + } - if (!isYAMLFileUpdated && isYMLFileUpdated) { - val yamlFileContent = gitHubGraphQLService.getManifestFileContent( + modifiedManifestFiles.forEach { filePath -> + handleAddedOrModifiedManifestFile( + repositoryFullName, owner, repositoryName, - yamlFileName, - installationToken + installationToken, + filePath, + ManifestFileAction.MODIFIED ) - if (yamlFileContent != null) return } - val manifestFilePath = determineManifestFilePath(isYAMLFileUpdated, isYMLFileUpdated, yamlFileName, ymlFileName) - manifestFilePath?.let { - when (it) { - in headCommit.added, in headCommit.modified -> { - handleAddedOrModifiedManifestFile( - headCommit, - repositoryFullName, - owner, - repositoryName, - installationToken, - it - ) - } - in headCommit.removed -> { - handleRemovedManifestFile(repositoryFullName) - } - } + removedManifestFiles.forEach { filePath -> + handleRemovedManifestFile(repositoryFullName) } } @@ -107,35 +99,15 @@ class WebhookEventService( return installationToken } - private fun isManifestFileUpdated(headCommit: PushEventCommit, fileName: String): Boolean { - return headCommit.added.any { it == fileName } || - headCommit.modified.any { it == fileName } || - headCommit.removed.any { it == fileName } - } - - private fun determineManifestFilePath( - isYAMLFileUpdated: Boolean, - isYMLFileUpdated: Boolean, - yamlFileName: String, - ymlFileName: String - ): String? { - return when { - isYAMLFileUpdated -> yamlFileName - isYMLFileUpdated -> ymlFileName - else -> null - } - } - @SuppressWarnings("LongParameterList") private fun handleAddedOrModifiedManifestFile( - headCommit: PushEventCommit, repositoryFullName: String, owner: String, repositoryName: String, installationToken: String, - manifestFilePath: String + manifestFilePath: String, + action: ManifestFileAction ) { - val action = if (manifestFilePath in headCommit.added) ManifestFileAction.ADDED else ManifestFileAction.MODIFIED logger.info("Manifest file $action in repository $repositoryFullName") val fileContent = gitHubGraphQLService.getManifestFileContent( owner, @@ -148,7 +120,8 @@ class WebhookEventService( ManifestFileUpdateDto( repositoryFullName, action, - fileContent + fileContent, + manifestFilePath ) ) } @@ -160,6 +133,7 @@ class WebhookEventService( ManifestFileUpdateDto( repositoryFullName, ManifestFileAction.REMOVED, + null, null ) ) diff --git a/src/main/kotlin/net/leanix/githubagent/shared/Constants.kt b/src/main/kotlin/net/leanix/githubagent/shared/Constants.kt index 4ae4046..9bbaf22 100644 --- a/src/main/kotlin/net/leanix/githubagent/shared/Constants.kt +++ b/src/main/kotlin/net/leanix/githubagent/shared/Constants.kt @@ -3,11 +3,7 @@ package net.leanix.githubagent.shared const val TOPIC_PREFIX = "/app/ghe/" const val APP_NAME_TOPIC = "appName" const val LOGS_TOPIC = "logs" - -enum class ManifestFileName(val fileName: String) { - YAML("leanix.yaml"), - YML("leanix.yml") -} +const val MANIFEST_FILE_NAME = "leanix.yml" val SUPPORTED_EVENT_TYPES = listOf( "REPOSITORY", diff --git a/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt index 754227a..5ce8d0e 100644 --- a/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt +++ b/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt @@ -5,6 +5,7 @@ import io.mockk.every import io.mockk.verify import net.leanix.githubagent.dto.ManifestFileAction import net.leanix.githubagent.dto.ManifestFileUpdateDto +import net.leanix.githubagent.shared.MANIFEST_FILE_NAME import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -74,7 +75,7 @@ class WebhookEventServiceTest { }, "head_commit": { "added": [], - "modified": ["leanix.yaml"], + "modified": ["$MANIFEST_FILE_NAME"], "removed": [] }, "installation": {"id": 1}, @@ -89,7 +90,8 @@ class WebhookEventServiceTest { ManifestFileUpdateDto( "owner/repo", ManifestFileAction.MODIFIED, - "content" + "content", + MANIFEST_FILE_NAME ) ) } @@ -118,44 +120,9 @@ class WebhookEventServiceTest { verify(exactly = 1) { webSocketService.sendMessage("/events/other", payload) } } - @Test - fun `should send updates for yaml manifest file`() { - val manifestFilePath = "leanix.yaml" - val payload = createPushEventPayload(manifestFilePath) - - webhookEventService.consumeWebhookEvent("PUSH", payload) - - verify { webSocketService.sendMessage(any(), any()) } - } - @Test fun `should send updates for yml manifest file`() { - val manifestFilePath = "leanix.yml" - val payload = createPushEventPayload(manifestFilePath) - every { gitHubGraphQLService.getManifestFileContent(any(), any(), "leanix.yaml", any()) } returns null - every { gitHubGraphQLService.getManifestFileContent(any(), any(), manifestFilePath, any()) } returns "content" - - webhookEventService.consumeWebhookEvent("PUSH", payload) - - verify { webSocketService.sendMessage(any(), any()) } - } - - @Test - fun `should ignore yml file changes if yaml file exist in repository`() { - val manifestFilePath = "leanix.yml" - val payload = createPushEventPayload(manifestFilePath) - - every { - gitHubGraphQLService.getManifestFileContent("test-owner", "test-repo", "manifest.yaml", "token") - } returns "content" - - webhookEventService.consumeWebhookEvent("PUSH", payload) - - verify(exactly = 0) { webSocketService.sendMessage(any(), any()) } - } - - private fun createPushEventPayload(manifestFileName: String): String { - return """ + val payload = """ { "ref": "refs/heads/main", "repository": { @@ -167,7 +134,7 @@ class WebhookEventServiceTest { } }, "head_commit": { - "added": ["$manifestFileName"], + "added": ["$MANIFEST_FILE_NAME"], "modified": [], "removed": [] }, @@ -176,5 +143,100 @@ class WebhookEventServiceTest { } } """ + every { gitHubGraphQLService.getManifestFileContent(any(), any(), MANIFEST_FILE_NAME, any()) } returns "content" + + webhookEventService.consumeWebhookEvent("PUSH", payload) + + verify { webSocketService.sendMessage(any(), any()) } + } + + @Test + fun `should handle manifest file in subdirectory`() { + val payload = """{ + "repository": { + "name": "repo", + "full_name": "owner/repo", + "owner": {"name": "owner"}, + "default_branch": "main" + }, + "head_commit": { + "added": ["a/b/c/$MANIFEST_FILE_NAME"], + "modified": [], + "removed": [] + }, + "installation": {"id": 1}, + "ref": "refs/heads/main" + }""" + + webhookEventService.consumeWebhookEvent("PUSH", payload) + + verify(exactly = 1) { + webSocketService.sendMessage( + "/events/manifestFile", + ManifestFileUpdateDto( + "owner/repo", + ManifestFileAction.ADDED, + "content", + "a/b/c/$MANIFEST_FILE_NAME" + ) + ) + } + } + + @Test + fun `should handle push event with multiple added and modified files`() { + val payload = """{ + "repository": { + "name": "repo", + "full_name": "owner/repo", + "owner": {"name": "owner"}, + "default_branch": "main" + }, + "head_commit": { + "added": ["custom/path/added1/$MANIFEST_FILE_NAME", "custom/path/added2/$MANIFEST_FILE_NAME"], + "modified": ["custom/path/modified/$MANIFEST_FILE_NAME"], + "removed": [] + }, + "installation": {"id": 1}, + "ref": "refs/heads/main" + }""" + + webhookEventService.consumeWebhookEvent("PUSH", payload) + + verify(exactly = 1) { + webSocketService.sendMessage( + "/events/manifestFile", + ManifestFileUpdateDto( + "owner/repo", + ManifestFileAction.ADDED, + "content", + "custom/path/added1/$MANIFEST_FILE_NAME" + ) + ) + } + + verify(exactly = 1) { + webSocketService.sendMessage( + "/events/manifestFile", + ManifestFileUpdateDto( + "owner/repo", + ManifestFileAction.ADDED, + "content", + "custom/path/added2/$MANIFEST_FILE_NAME" + ) + ) + } + + verify(exactly = 1) { + webSocketService.sendMessage( + "/events/manifestFile", + ManifestFileUpdateDto( + "owner/repo", + ManifestFileAction.MODIFIED, + "content", + "custom/path/modified/$MANIFEST_FILE_NAME" + ) + ) + } } } From eb7606bed7516bc7a751bfadfb528b1df4e15bc0 Mon Sep 17 00:00:00 2001 From: mohamedlajmileanix Date: Thu, 31 Oct 2024 11:39:17 +0100 Subject: [PATCH 2/4] CID-3105: Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index eb2075b..8e92347 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ The SAP LeanIX agent discovers self-built software in self-hosted GitHub Enterpr - `GITHUB_ENTERPRISE_BASE_URL`: The base URL of your GitHub Enterprise Server instance. - `GITHUB_APP_ID`: The ID of your GitHub App. - `PEM_FILE`: The path to your GitHub App's PEM file inside the Docker container. - - `MANIFEST_FILE_DIRECTORY`: The directory path where the manifest files are stored in each repository. Manifest files are crucial for microservice discovery as they provide essential information about the service. For more information, see [Microservice Discovery Through a Manifest File](https://docs-eam.leanix.net/reference/microservice-discovery-manifest-file) in our documentation. **If both .yaml and .yml files exist in a repository, only the .yaml file will be used.** - `WEBHOOK_SECRET`: The secret used to validate incoming webhook events from GitHub. (Optional, but recommended. [Needs to be set in the GitHub App settings first](https://docs.github.com/en/enterprise-server@3.8/webhooks/using-webhooks/validating-webhook-deliveries).) 5. **Start the agent**: To start the agent, run the following Docker command. Replace the variables in angle brackets with your actual values. @@ -38,7 +37,6 @@ The SAP LeanIX agent discovers self-built software in self-hosted GitHub Enterpr -e GITHUB_ENTERPRISE_BASE_URL= \ -e GITHUB_APP_ID= \ -e PEM_FILE=/privateKey.pem \ - -e MANIFEST_FILE_DIRECTORY= \ -e WEBHOOK_SECRET= \ leanix-github-agent ``` From d01bb99251d781c96afd8915e76e648206f0ffed Mon Sep 17 00:00:00 2001 From: mohamedlajmileanix Date: Thu, 31 Oct 2024 12:19:23 +0100 Subject: [PATCH 3/4] CID-3105: Update log message --- .../leanix/githubagent/services/WebhookEventService.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt b/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt index 74c1908..95bac9c 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt @@ -108,7 +108,14 @@ class WebhookEventService( manifestFilePath: String, action: ManifestFileAction ) { - logger.info("Manifest file $action in repository $repositoryFullName") + val location = if ('/' in manifestFilePath) { + "directory '/${manifestFilePath.substringBeforeLast('/')}'" + } else { + "root folder" + } + + logger.info("Manifest file {} in repository {} under {}", action, repositoryFullName, location) + val fileContent = gitHubGraphQLService.getManifestFileContent( owner, repositoryName, From 79fbb02496b4e60dce76b98df8c63ee8876dbf4e Mon Sep 17 00:00:00 2001 From: mohamedlajmileanix Date: Thu, 31 Oct 2024 15:19:20 +0100 Subject: [PATCH 4/4] CID-3105: addressing PR comments --- .../leanix/githubagent/services/WebhookEventServiceTest.kt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt index 5ce8d0e..7de900c 100644 --- a/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt +++ b/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt @@ -213,9 +213,6 @@ class WebhookEventServiceTest { "custom/path/added1/$MANIFEST_FILE_NAME" ) ) - } - - verify(exactly = 1) { webSocketService.sendMessage( "/events/manifestFile", ManifestFileUpdateDto( @@ -225,9 +222,6 @@ class WebhookEventServiceTest { "custom/path/added2/$MANIFEST_FILE_NAME" ) ) - } - - verify(exactly = 1) { webSocketService.sendMessage( "/events/manifestFile", ManifestFileUpdateDto(