diff --git a/README.md b/README.md index eb2075b..8e92347 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ The SAP LeanIX agent discovers self-built software in self-hosted GitHub Enterpr - `GITHUB_ENTERPRISE_BASE_URL`: The base URL of your GitHub Enterprise Server instance. - `GITHUB_APP_ID`: The ID of your GitHub App. - `PEM_FILE`: The path to your GitHub App's PEM file inside the Docker container. - - `MANIFEST_FILE_DIRECTORY`: The directory path where the manifest files are stored in each repository. Manifest files are crucial for microservice discovery as they provide essential information about the service. For more information, see [Microservice Discovery Through a Manifest File](https://docs-eam.leanix.net/reference/microservice-discovery-manifest-file) in our documentation. **If both .yaml and .yml files exist in a repository, only the .yaml file will be used.** - `WEBHOOK_SECRET`: The secret used to validate incoming webhook events from GitHub. (Optional, but recommended. [Needs to be set in the GitHub App settings first](https://docs.github.com/en/enterprise-server@3.8/webhooks/using-webhooks/validating-webhook-deliveries).) 5. **Start the agent**: To start the agent, run the following Docker command. Replace the variables in angle brackets with your actual values. @@ -38,7 +37,6 @@ The SAP LeanIX agent discovers self-built software in self-hosted GitHub Enterpr -e GITHUB_ENTERPRISE_BASE_URL= \ -e GITHUB_APP_ID= \ -e PEM_FILE=/privateKey.pem \ - -e MANIFEST_FILE_DIRECTORY= \ -e WEBHOOK_SECRET= \ leanix-github-agent ``` diff --git a/src/main/kotlin/net/leanix/githubagent/config/GitHubEnterpriseProperties.kt b/src/main/kotlin/net/leanix/githubagent/config/GitHubEnterpriseProperties.kt index bf93f3b..c1f5762 100644 --- a/src/main/kotlin/net/leanix/githubagent/config/GitHubEnterpriseProperties.kt +++ b/src/main/kotlin/net/leanix/githubagent/config/GitHubEnterpriseProperties.kt @@ -7,6 +7,5 @@ data class GitHubEnterpriseProperties( val baseUrl: String, val gitHubAppId: String, val pemFile: String, - val manifestFileDirectory: String, val webhookSecret: String ) diff --git a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFileUpdateDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFileUpdateDto.kt index 35b85d0..db0374f 100644 --- a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFileUpdateDto.kt +++ b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFileUpdateDto.kt @@ -3,7 +3,8 @@ package net.leanix.githubagent.dto data class ManifestFileUpdateDto( val repositoryFullName: String, val action: ManifestFileAction, - val manifestFileContent: String? + val manifestFileContent: String?, + val manifestFilePath: String? ) enum class ManifestFileAction { diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt index a7765c8..45554f0 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt @@ -12,7 +12,7 @@ import net.leanix.githubagent.dto.RepositoryDto import net.leanix.githubagent.dto.Trigger import net.leanix.githubagent.exceptions.JwtTokenNotFound import net.leanix.githubagent.exceptions.ManifestFileNotFoundException -import net.leanix.githubagent.shared.ManifestFileName +import net.leanix.githubagent.shared.MANIFEST_FILE_NAME import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.util.UUID @@ -141,7 +141,7 @@ class GitHubScanningService( gitHubClient.searchManifestFiles( "Bearer $installationToken", "" + - "repo:${installation.account.login}/$repositoryName filename:${ManifestFileName.YAML.fileName}" + "repo:${installation.account.login}/$repositoryName filename:$MANIFEST_FILE_NAME" ) } private fun fetchManifestContents( @@ -163,7 +163,7 @@ class GitHubScanningService( numOfManifestFilesFound++ syncLogService.sendInfoLog("Fetched manifest file ${manifestFile.path} from repository $repositoryName") ManifestFileDTO( - path = manifestFile.path.replace("/${ManifestFileName.YAML.fileName}", ""), + path = manifestFile.path.replace("/$MANIFEST_FILE_NAME", ""), content = content ) } else { diff --git a/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt b/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt index 180f4b6..95bac9c 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt @@ -2,12 +2,11 @@ package net.leanix.githubagent.services import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import net.leanix.githubagent.config.GitHubEnterpriseProperties import net.leanix.githubagent.dto.ManifestFileAction import net.leanix.githubagent.dto.ManifestFileUpdateDto import net.leanix.githubagent.dto.PushEventCommit import net.leanix.githubagent.dto.PushEventPayload -import net.leanix.githubagent.shared.ManifestFileName +import net.leanix.githubagent.shared.MANIFEST_FILE_NAME import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -15,7 +14,6 @@ import org.springframework.stereotype.Service class WebhookEventService( private val webSocketService: WebSocketService, private val gitHubGraphQLService: GitHubGraphQLService, - private val gitHubEnterpriseProperties: GitHubEnterpriseProperties, private val cachingService: CachingService, private val gitHubAuthenticationService: GitHubAuthenticationService ) { @@ -53,7 +51,6 @@ class WebhookEventService( } } - @SuppressWarnings("LongParameterList") private fun handleManifestFileChanges( headCommit: PushEventCommit, repositoryFullName: String, @@ -61,39 +58,34 @@ class WebhookEventService( repositoryName: String, installationToken: String ) { - val yamlFileName = "${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YAML.fileName}" - val ymlFileName = "${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YML.fileName}" + val addedManifestFiles = headCommit.added.filter { it.contains(MANIFEST_FILE_NAME) } + val modifiedManifestFiles = headCommit.modified.filter { it.contains(MANIFEST_FILE_NAME) } + val removedManifestFiles = headCommit.removed.filter { it.contains(MANIFEST_FILE_NAME) } - val isYAMLFileUpdated = isManifestFileUpdated(headCommit, yamlFileName) - val isYMLFileUpdated = isManifestFileUpdated(headCommit, ymlFileName) + addedManifestFiles.forEach { filePath -> + handleAddedOrModifiedManifestFile( + repositoryFullName, + owner, + repositoryName, + installationToken, + filePath, + ManifestFileAction.ADDED + ) + } - if (!isYAMLFileUpdated && isYMLFileUpdated) { - val yamlFileContent = gitHubGraphQLService.getManifestFileContent( + modifiedManifestFiles.forEach { filePath -> + handleAddedOrModifiedManifestFile( + repositoryFullName, owner, repositoryName, - yamlFileName, - installationToken + installationToken, + filePath, + ManifestFileAction.MODIFIED ) - if (yamlFileContent != null) return } - val manifestFilePath = determineManifestFilePath(isYAMLFileUpdated, isYMLFileUpdated, yamlFileName, ymlFileName) - manifestFilePath?.let { - when (it) { - in headCommit.added, in headCommit.modified -> { - handleAddedOrModifiedManifestFile( - headCommit, - repositoryFullName, - owner, - repositoryName, - installationToken, - it - ) - } - in headCommit.removed -> { - handleRemovedManifestFile(repositoryFullName) - } - } + removedManifestFiles.forEach { filePath -> + handleRemovedManifestFile(repositoryFullName) } } @@ -107,36 +99,23 @@ class WebhookEventService( return installationToken } - private fun isManifestFileUpdated(headCommit: PushEventCommit, fileName: String): Boolean { - return headCommit.added.any { it == fileName } || - headCommit.modified.any { it == fileName } || - headCommit.removed.any { it == fileName } - } - - private fun determineManifestFilePath( - isYAMLFileUpdated: Boolean, - isYMLFileUpdated: Boolean, - yamlFileName: String, - ymlFileName: String - ): String? { - return when { - isYAMLFileUpdated -> yamlFileName - isYMLFileUpdated -> ymlFileName - else -> null - } - } - @SuppressWarnings("LongParameterList") private fun handleAddedOrModifiedManifestFile( - headCommit: PushEventCommit, repositoryFullName: String, owner: String, repositoryName: String, installationToken: String, - manifestFilePath: String + manifestFilePath: String, + action: ManifestFileAction ) { - val action = if (manifestFilePath in headCommit.added) ManifestFileAction.ADDED else ManifestFileAction.MODIFIED - logger.info("Manifest file $action in repository $repositoryFullName") + val location = if ('/' in manifestFilePath) { + "directory '/${manifestFilePath.substringBeforeLast('/')}'" + } else { + "root folder" + } + + logger.info("Manifest file {} in repository {} under {}", action, repositoryFullName, location) + val fileContent = gitHubGraphQLService.getManifestFileContent( owner, repositoryName, @@ -148,7 +127,8 @@ class WebhookEventService( ManifestFileUpdateDto( repositoryFullName, action, - fileContent + fileContent, + manifestFilePath ) ) } @@ -160,6 +140,7 @@ class WebhookEventService( ManifestFileUpdateDto( repositoryFullName, ManifestFileAction.REMOVED, + null, null ) ) diff --git a/src/main/kotlin/net/leanix/githubagent/shared/Constants.kt b/src/main/kotlin/net/leanix/githubagent/shared/Constants.kt index 4ae4046..9bbaf22 100644 --- a/src/main/kotlin/net/leanix/githubagent/shared/Constants.kt +++ b/src/main/kotlin/net/leanix/githubagent/shared/Constants.kt @@ -3,11 +3,7 @@ package net.leanix.githubagent.shared const val TOPIC_PREFIX = "/app/ghe/" const val APP_NAME_TOPIC = "appName" const val LOGS_TOPIC = "logs" - -enum class ManifestFileName(val fileName: String) { - YAML("leanix.yaml"), - YML("leanix.yml") -} +const val MANIFEST_FILE_NAME = "leanix.yml" val SUPPORTED_EVENT_TYPES = listOf( "REPOSITORY", diff --git a/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt index 754227a..7de900c 100644 --- a/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt +++ b/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt @@ -5,6 +5,7 @@ import io.mockk.every import io.mockk.verify import net.leanix.githubagent.dto.ManifestFileAction import net.leanix.githubagent.dto.ManifestFileUpdateDto +import net.leanix.githubagent.shared.MANIFEST_FILE_NAME import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -74,7 +75,7 @@ class WebhookEventServiceTest { }, "head_commit": { "added": [], - "modified": ["leanix.yaml"], + "modified": ["$MANIFEST_FILE_NAME"], "removed": [] }, "installation": {"id": 1}, @@ -89,7 +90,8 @@ class WebhookEventServiceTest { ManifestFileUpdateDto( "owner/repo", ManifestFileAction.MODIFIED, - "content" + "content", + MANIFEST_FILE_NAME ) ) } @@ -118,44 +120,9 @@ class WebhookEventServiceTest { verify(exactly = 1) { webSocketService.sendMessage("/events/other", payload) } } - @Test - fun `should send updates for yaml manifest file`() { - val manifestFilePath = "leanix.yaml" - val payload = createPushEventPayload(manifestFilePath) - - webhookEventService.consumeWebhookEvent("PUSH", payload) - - verify { webSocketService.sendMessage(any(), any()) } - } - @Test fun `should send updates for yml manifest file`() { - val manifestFilePath = "leanix.yml" - val payload = createPushEventPayload(manifestFilePath) - every { gitHubGraphQLService.getManifestFileContent(any(), any(), "leanix.yaml", any()) } returns null - every { gitHubGraphQLService.getManifestFileContent(any(), any(), manifestFilePath, any()) } returns "content" - - webhookEventService.consumeWebhookEvent("PUSH", payload) - - verify { webSocketService.sendMessage(any(), any()) } - } - - @Test - fun `should ignore yml file changes if yaml file exist in repository`() { - val manifestFilePath = "leanix.yml" - val payload = createPushEventPayload(manifestFilePath) - - every { - gitHubGraphQLService.getManifestFileContent("test-owner", "test-repo", "manifest.yaml", "token") - } returns "content" - - webhookEventService.consumeWebhookEvent("PUSH", payload) - - verify(exactly = 0) { webSocketService.sendMessage(any(), any()) } - } - - private fun createPushEventPayload(manifestFileName: String): String { - return """ + val payload = """ { "ref": "refs/heads/main", "repository": { @@ -167,7 +134,7 @@ class WebhookEventServiceTest { } }, "head_commit": { - "added": ["$manifestFileName"], + "added": ["$MANIFEST_FILE_NAME"], "modified": [], "removed": [] }, @@ -176,5 +143,94 @@ class WebhookEventServiceTest { } } """ + every { gitHubGraphQLService.getManifestFileContent(any(), any(), MANIFEST_FILE_NAME, any()) } returns "content" + + webhookEventService.consumeWebhookEvent("PUSH", payload) + + verify { webSocketService.sendMessage(any(), any()) } + } + + @Test + fun `should handle manifest file in subdirectory`() { + val payload = """{ + "repository": { + "name": "repo", + "full_name": "owner/repo", + "owner": {"name": "owner"}, + "default_branch": "main" + }, + "head_commit": { + "added": ["a/b/c/$MANIFEST_FILE_NAME"], + "modified": [], + "removed": [] + }, + "installation": {"id": 1}, + "ref": "refs/heads/main" + }""" + + webhookEventService.consumeWebhookEvent("PUSH", payload) + + verify(exactly = 1) { + webSocketService.sendMessage( + "/events/manifestFile", + ManifestFileUpdateDto( + "owner/repo", + ManifestFileAction.ADDED, + "content", + "a/b/c/$MANIFEST_FILE_NAME" + ) + ) + } + } + + @Test + fun `should handle push event with multiple added and modified files`() { + val payload = """{ + "repository": { + "name": "repo", + "full_name": "owner/repo", + "owner": {"name": "owner"}, + "default_branch": "main" + }, + "head_commit": { + "added": ["custom/path/added1/$MANIFEST_FILE_NAME", "custom/path/added2/$MANIFEST_FILE_NAME"], + "modified": ["custom/path/modified/$MANIFEST_FILE_NAME"], + "removed": [] + }, + "installation": {"id": 1}, + "ref": "refs/heads/main" + }""" + + webhookEventService.consumeWebhookEvent("PUSH", payload) + + verify(exactly = 1) { + webSocketService.sendMessage( + "/events/manifestFile", + ManifestFileUpdateDto( + "owner/repo", + ManifestFileAction.ADDED, + "content", + "custom/path/added1/$MANIFEST_FILE_NAME" + ) + ) + webSocketService.sendMessage( + "/events/manifestFile", + ManifestFileUpdateDto( + "owner/repo", + ManifestFileAction.ADDED, + "content", + "custom/path/added2/$MANIFEST_FILE_NAME" + ) + ) + webSocketService.sendMessage( + "/events/manifestFile", + ManifestFileUpdateDto( + "owner/repo", + ManifestFileAction.MODIFIED, + "content", + "custom/path/modified/$MANIFEST_FILE_NAME" + ) + ) + } } }