From 536ef634668bf9e8ac92ed7a3c000432211199dc Mon Sep 17 00:00:00 2001 From: Henrique Eduardo do Amaral Date: Tue, 22 Oct 2024 10:45:11 +0200 Subject: [PATCH 1/7] cid-2908: Fix log messages removing the runId from path --- src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt | 2 +- .../kotlin/net/leanix/githubagent/services/SyncLogService.kt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt index e1f3319..b608142 100644 --- a/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt +++ b/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt @@ -6,7 +6,7 @@ data class SyncLogDto( val runId: UUID?, val trigger: Trigger, val logLevel: LogLevel, - val message: String + val message: String? ) enum class Trigger { diff --git a/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt b/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt index aff569c..bb3afa1 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt @@ -16,7 +16,7 @@ class SyncLogService( sendSyncLog(message, LOGS_TOPIC, null, LogLevel.ERROR) } - fun sendSyncLog(message: String, topic: String, trigger: Trigger?, logLevel: LogLevel) { + fun sendSyncLog(message: String? = null, topic: String = LOGS_TOPIC, trigger: Trigger?, logLevel: LogLevel) { val runId = cachingService.get("runId") as UUID val syncLogDto = SyncLogDto( runId = runId, @@ -28,6 +28,6 @@ class SyncLogService( } private fun constructTopic(topic: String): String { - return "${cachingService.get("runId")}/$topic" + return topic } } From d5e82d5ef1b3a228c7dc66b49fcdd216ee413a7d Mon Sep 17 00:00:00 2001 From: Henrique Eduardo do Amaral Date: Tue, 22 Oct 2024 10:45:34 +0200 Subject: [PATCH 2/7] cid-2908: Add full name on the DTO --- src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt index f1ea8ff..7b55086 100644 --- a/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt +++ b/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt @@ -13,4 +13,6 @@ data class RepositoryDto( val updatedAt: String, val languages: List, val topics: List, -) +) { + val fullName: String = "$organizationName/$name" +} From 61da34307279359cf6a29ca92b457316e12aaed9 Mon Sep 17 00:00:00 2001 From: Henrique Eduardo do Amaral Date: Tue, 22 Oct 2024 10:46:17 +0200 Subject: [PATCH 3/7] cid-2908: Add new method to search manifest files for each repository --- .../leanix/githubagent/client/GitHubClient.kt | 8 ++++++ .../githubagent/dto/GitHubSearchResponse.kt | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt diff --git a/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt b/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt index c69e1e4..d333087 100644 --- a/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt +++ b/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt @@ -1,6 +1,7 @@ package net.leanix.githubagent.client import net.leanix.githubagent.dto.GitHubAppResponse +import net.leanix.githubagent.dto.GitHubSearchResponse import net.leanix.githubagent.dto.Installation import net.leanix.githubagent.dto.InstallationTokenResponse import net.leanix.githubagent.dto.Organization @@ -10,6 +11,7 @@ import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestHeader +import org.springframework.web.bind.annotation.RequestParam @FeignClient(name = "githubClient", url = "\${github-enterprise.baseUrl}") interface GitHubClient { @@ -37,4 +39,10 @@ interface GitHubClient { @PathVariable("org") org: String, @RequestHeader("Authorization") token: String ): List + + @GetMapping("/api/v3/search/code") + fun searchManifestFiles( + @RequestHeader("Authorization") token: String, + @RequestParam("q") query: String, + ): GitHubSearchResponse } diff --git a/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt b/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt new file mode 100644 index 0000000..585541f --- /dev/null +++ b/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt @@ -0,0 +1,26 @@ +package net.leanix.githubagent.dto + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +@JsonIgnoreProperties(ignoreUnknown = true) +data class GitHubSearchResponse( + @JsonProperty("total_count") + val totalCount: Int, + val items: List +) + +@JsonIgnoreProperties(ignoreUnknown = true) +data class ItemResponse( + val name: String, + val path: String, + val repository: RepositoryItemResponse, + val url: String, +) + +@JsonIgnoreProperties(ignoreUnknown = true) +data class RepositoryItemResponse( + val name: String, + @JsonProperty("full_name") + val fullName: String, +) From ff4e4924d342bebc8132343976b231b62349344b Mon Sep 17 00:00:00 2001 From: Henrique Eduardo do Amaral Date: Tue, 22 Oct 2024 11:00:27 +0200 Subject: [PATCH 4/7] cid-2908: Remove not used get manifest files --- .../net/leanix/githubagent/services/GitHubGraphQLService.kt | 5 ----- src/main/resources/graphql/GetRepositories.graphql | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt index b8e8c9f..4f8a9d0 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt @@ -2,13 +2,11 @@ package net.leanix.githubagent.services import com.expediagroup.graphql.client.spring.GraphQLWebClient import kotlinx.coroutines.runBlocking -import net.leanix.githubagent.config.GitHubEnterpriseProperties import net.leanix.githubagent.dto.PagedRepositories import net.leanix.githubagent.dto.RepositoryDto import net.leanix.githubagent.exceptions.GraphQLApiException import net.leanix.githubagent.graphql.data.GetRepositories import net.leanix.githubagent.graphql.data.GetRepositoryManifestContent -import net.leanix.githubagent.shared.ManifestFileName import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import org.springframework.web.reactive.function.client.WebClient @@ -16,7 +14,6 @@ import org.springframework.web.reactive.function.client.WebClient @Component class GitHubGraphQLService( private val cachingService: CachingService, - private val gitHubEnterpriseProperties: GitHubEnterpriseProperties ) { companion object { private val logger = LoggerFactory.getLogger(GitHubGraphQLService::class.java) @@ -33,8 +30,6 @@ class GitHubGraphQLService( GetRepositories.Variables( pageCount = PAGE_COUNT, cursor = cursor, - "HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YAML.fileName}", - "HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YML.fileName}" ) ) diff --git a/src/main/resources/graphql/GetRepositories.graphql b/src/main/resources/graphql/GetRepositories.graphql index 747ddde..01bcbab 100644 --- a/src/main/resources/graphql/GetRepositories.graphql +++ b/src/main/resources/graphql/GetRepositories.graphql @@ -1,4 +1,4 @@ -query GetRepositories($pageCount: Int!, $cursor: String, $manifestYamlPath: String!, $manifestYmlPath: String!) { +query GetRepositories($pageCount: Int!, $cursor: String) { viewer { repositories(first: $pageCount, after: $cursor) { pageInfo { From 08df32bbd70d9a5139a8aad18be28b769c38c2ee Mon Sep 17 00:00:00 2001 From: Henrique Eduardo do Amaral Date: Tue, 22 Oct 2024 11:01:46 +0200 Subject: [PATCH 5/7] cid-2908: Add send manifest files to the backend --- .../githubagent/dto/ManifestFilesDTO.kt | 12 ++++ .../services/GitHubScanningService.kt | 70 ++++++++++++++++++- .../services/GitHubScanningServiceTest.kt | 53 +++++++++++++- 3 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt diff --git a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt new file mode 100644 index 0000000..0ebf8d7 --- /dev/null +++ b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt @@ -0,0 +1,12 @@ +package net.leanix.githubagent.dto + +data class ManifestFilesDTO( + val repositoryId: String, + val repositoryFullName: String, + val manifestFiles: List +) + +data class ManifestFileDTO( + val path: String, + val content: String?, +) diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt index 77ad4e4..0504dfd 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt @@ -2,8 +2,14 @@ package net.leanix.githubagent.services import net.leanix.githubagent.client.GitHubClient import net.leanix.githubagent.dto.Installation +import net.leanix.githubagent.dto.ItemResponse +import net.leanix.githubagent.dto.LogLevel +import net.leanix.githubagent.dto.ManifestFileDTO +import net.leanix.githubagent.dto.ManifestFilesDTO import net.leanix.githubagent.dto.Organization import net.leanix.githubagent.dto.OrganizationDto +import net.leanix.githubagent.dto.RepositoryDto +import net.leanix.githubagent.dto.Trigger import net.leanix.githubagent.exceptions.JwtTokenNotFound import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -24,15 +30,30 @@ class GitHubScanningService( fun scanGitHubResources() { cachingService.set("runId", UUID.randomUUID(), null) runCatching { + syncLogService.sendSyncLog( + trigger = Trigger.START_FULL_SYNC, + logLevel = LogLevel.INFO, + ) val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound() val installations = getInstallations(jwtToken.toString()) fetchAndSendOrganisationsData(installations) installations.forEach { installation -> fetchAndSendRepositoriesData(installation) + .forEach { repository -> + fetchManifestFilesAndSend(installation, repository) + } } + syncLogService.sendSyncLog( + trigger = Trigger.FINISH_FULL_SYNC, + logLevel = LogLevel.INFO, + ) }.onFailure { val message = "Error while scanning GitHub resources" - syncLogService.sendErrorLog(message) + syncLogService.sendSyncLog( + trigger = Trigger.FINISH_FULL_SYNC, + logLevel = LogLevel.ERROR, + message = message + ) cachingService.remove("runId") logger.error(message) throw it @@ -66,11 +87,12 @@ class GitHubScanningService( webSocketService.sendMessage("${cachingService.get("runId")}/organizations", organizations) } - private fun fetchAndSendRepositoriesData(installation: Installation) { + private fun fetchAndSendRepositoriesData(installation: Installation): List { val installationToken = cachingService.get("installationToken:${installation.id}").toString() var cursor: String? = null var totalRepos = 0 var page = 1 + val repositories = mutableListOf() do { val repositoriesPage = gitHubGraphQLService.getRepositories( token = installationToken, @@ -80,10 +102,54 @@ class GitHubScanningService( "${cachingService.get("runId")}/repositories", repositoriesPage.repositories ) + repositories.addAll(repositoriesPage.repositories) cursor = repositoriesPage.cursor totalRepos += repositoriesPage.repositories.size page++ } while (repositoriesPage.hasNextPage) logger.info("Fetched $totalRepos repositories data from organisation ${installation.account.login}") + return repositories + } + + private fun fetchManifestFilesAndSend(installation: Installation, repository: RepositoryDto) { + val manifestFiles = fetchManifestFiles(installation, repository.name).getOrThrow().items + val manifestFilesContents = fetchManifestContents(installation, manifestFiles, repository.name).getOrThrow() + + webSocketService.sendMessage( + "${cachingService.get("runId")}/manifestFiles", + ManifestFilesDTO( + repositoryId = repository.id, + repositoryFullName = repository.fullName, + manifestFiles = manifestFilesContents, + ) + ) + } + + private fun fetchManifestFiles(installation: Installation, repositoryName: String) = runCatching { + val installationToken = cachingService.get("installationToken:${installation.id}").toString() + gitHubClient.searchManifestFiles( + "Bearer $installationToken", + "" + + "repo:${installation.account.login}/$repositoryName filename:leanix.yaml" + ) + } + private fun fetchManifestContents( + installation: Installation, + items: List, + repositoryName: String + ) = runCatching { + val installationToken = cachingService.get("installationToken:${installation.id}").toString() + items.map { manifestFile -> + val content = gitHubGraphQLService.getManifestFileContent( + owner = installation.account.login, + repositoryName = repositoryName, + filePath = manifestFile.path, + token = installationToken + ) + ManifestFileDTO( + path = manifestFile.path.replace("/leanix.yaml", ""), + content = content + ) + } } } diff --git a/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt index 2d2c553..e83c646 100644 --- a/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt +++ b/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt @@ -5,11 +5,14 @@ import io.mockk.mockk import io.mockk.verify import net.leanix.githubagent.client.GitHubClient import net.leanix.githubagent.dto.Account +import net.leanix.githubagent.dto.GitHubSearchResponse import net.leanix.githubagent.dto.Installation import net.leanix.githubagent.dto.InstallationTokenResponse +import net.leanix.githubagent.dto.ItemResponse import net.leanix.githubagent.dto.Organization import net.leanix.githubagent.dto.PagedRepositories import net.leanix.githubagent.dto.RepositoryDto +import net.leanix.githubagent.dto.RepositoryItemResponse import net.leanix.githubagent.exceptions.JwtTokenNotFound import net.leanix.githubagent.graphql.data.enums.RepositoryVisibility import org.junit.jupiter.api.BeforeEach @@ -19,12 +22,12 @@ import java.util.UUID class GitHubScanningServiceTest { - private val gitHubClient = mockk() + private val gitHubClient = mockk(relaxUnitFun = true) private val cachingService = mockk() private val webSocketService = mockk(relaxUnitFun = true) private val gitHubGraphQLService = mockk() private val gitHubAuthenticationService = mockk() - private val syncLogService = mockk() + private val syncLogService = mockk(relaxUnitFun = true) private val gitHubScanningService = GitHubScanningService( gitHubClient, cachingService, @@ -103,7 +106,53 @@ class GitHubScanningServiceTest { hasNextPage = false, cursor = null ) + every { gitHubClient.searchManifestFiles(any(), any()) } returns GitHubSearchResponse(0, emptyList()) gitHubScanningService.scanGitHubResources() verify { webSocketService.sendMessage(eq("$runId/repositories"), any()) } } + + @Test + fun `scanGitHubResources should send repositories and manifest files over WebSocket`() { + // given + every { cachingService.get("runId") } returns runId + every { gitHubGraphQLService.getRepositories(any(), any()) } returns PagedRepositories( + repositories = listOf( + RepositoryDto( + id = "repo1", + name = "TestRepo", + organizationName = "testOrg", + description = "A test repository", + url = "https://github.com/testRepo", + archived = false, + visibility = RepositoryVisibility.PUBLIC, + updatedAt = "2024-01-01T00:00:00Z", + languages = listOf("Kotlin", "Java"), + topics = listOf("test", "example"), + ) + ), + hasNextPage = false, + cursor = null + ) + every { gitHubClient.searchManifestFiles(any(), any()) } returns GitHubSearchResponse( + 1, + listOf( + ItemResponse( + name = "leanix.yaml", + path = "dir/leanix.yaml", + repository = RepositoryItemResponse( + name = "TestRepo", + fullName = "testOrg/TestRepo" + ), + url = "http://url" + ) + ) + ) + every { gitHubGraphQLService.getManifestFileContent(any(), any(), "dir/leanix.yaml", any()) } returns "content" + + // when + gitHubScanningService.scanGitHubResources() + + // then + verify { webSocketService.sendMessage(eq("$runId/manifestFiles"), any()) } + } } From 6d2379a04cd91ef8ca1cf7c930b3abb07cfa92c3 Mon Sep 17 00:00:00 2001 From: Henrique Eduardo do Amaral Date: Mon, 28 Oct 2024 11:46:32 +0100 Subject: [PATCH 6/7] cid-2908: Change to use constant for leanix.yaml file --- .../net/leanix/githubagent/services/GitHubScanningService.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt index 0504dfd..a163424 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt @@ -11,6 +11,7 @@ import net.leanix.githubagent.dto.OrganizationDto import net.leanix.githubagent.dto.RepositoryDto import net.leanix.githubagent.dto.Trigger import net.leanix.githubagent.exceptions.JwtTokenNotFound +import net.leanix.githubagent.shared.ManifestFileName import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.util.UUID @@ -130,7 +131,7 @@ class GitHubScanningService( gitHubClient.searchManifestFiles( "Bearer $installationToken", "" + - "repo:${installation.account.login}/$repositoryName filename:leanix.yaml" + "repo:${installation.account.login}/$repositoryName filename:${ManifestFileName.YAML.fileName}" ) } private fun fetchManifestContents( @@ -147,7 +148,7 @@ class GitHubScanningService( token = installationToken ) ManifestFileDTO( - path = manifestFile.path.replace("/leanix.yaml", ""), + path = manifestFile.path.replace("/${ManifestFileName.YAML.fileName}", ""), content = content ) } From b6c57e35511d325c1090d73d46c66b7e682d9a17 Mon Sep 17 00:00:00 2001 From: Henrique Eduardo do Amaral Date: Mon, 28 Oct 2024 12:05:53 +0100 Subject: [PATCH 7/7] cid-2908: Change the manifest content to be mandatory --- .../net/leanix/githubagent/dto/ManifestFilesDTO.kt | 2 +- .../net/leanix/githubagent/exceptions/Exceptions.kt | 1 + .../githubagent/services/GitHubScanningService.kt | 13 +++++++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt index 0ebf8d7..f052c8d 100644 --- a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt +++ b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt @@ -8,5 +8,5 @@ data class ManifestFilesDTO( data class ManifestFileDTO( val path: String, - val content: String?, + val content: String, ) diff --git a/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt b/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt index 04bca83..cb64d29 100644 --- a/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt +++ b/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt @@ -13,3 +13,4 @@ class GraphQLApiException(errors: List) : RuntimeException("Errors: ${errors.joinToString(separator = "\n") { it.message }}") class WebhookSecretNotSetException : RuntimeException("Webhook secret not set") class InvalidEventSignatureException : RuntimeException("Invalid event signature") +class ManifestFileNotFoundException : RuntimeException("Manifest File Not Found") diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt index a163424..93bd6d0 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt @@ -11,6 +11,7 @@ import net.leanix.githubagent.dto.OrganizationDto import net.leanix.githubagent.dto.RepositoryDto import net.leanix.githubagent.dto.Trigger import net.leanix.githubagent.exceptions.JwtTokenNotFound +import net.leanix.githubagent.exceptions.ManifestFileNotFoundException import net.leanix.githubagent.shared.ManifestFileName import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -147,10 +148,14 @@ class GitHubScanningService( filePath = manifestFile.path, token = installationToken ) - ManifestFileDTO( - path = manifestFile.path.replace("/${ManifestFileName.YAML.fileName}", ""), - content = content - ) + if (content != null) { + ManifestFileDTO( + path = manifestFile.path.replace("/${ManifestFileName.YAML.fileName}", ""), + content = content + ) + } else { + throw ManifestFileNotFoundException() + } } } }