diff --git a/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt b/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt index c69e1e4..d333087 100644 --- a/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt +++ b/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt @@ -1,6 +1,7 @@ package net.leanix.githubagent.client import net.leanix.githubagent.dto.GitHubAppResponse +import net.leanix.githubagent.dto.GitHubSearchResponse import net.leanix.githubagent.dto.Installation import net.leanix.githubagent.dto.InstallationTokenResponse import net.leanix.githubagent.dto.Organization @@ -10,6 +11,7 @@ import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestHeader +import org.springframework.web.bind.annotation.RequestParam @FeignClient(name = "githubClient", url = "\${github-enterprise.baseUrl}") interface GitHubClient { @@ -37,4 +39,10 @@ interface GitHubClient { @PathVariable("org") org: String, @RequestHeader("Authorization") token: String ): List + + @GetMapping("/api/v3/search/code") + fun searchManifestFiles( + @RequestHeader("Authorization") token: String, + @RequestParam("q") query: String, + ): GitHubSearchResponse } diff --git a/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt b/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt new file mode 100644 index 0000000..585541f --- /dev/null +++ b/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt @@ -0,0 +1,26 @@ +package net.leanix.githubagent.dto + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +@JsonIgnoreProperties(ignoreUnknown = true) +data class GitHubSearchResponse( + @JsonProperty("total_count") + val totalCount: Int, + val items: List +) + +@JsonIgnoreProperties(ignoreUnknown = true) +data class ItemResponse( + val name: String, + val path: String, + val repository: RepositoryItemResponse, + val url: String, +) + +@JsonIgnoreProperties(ignoreUnknown = true) +data class RepositoryItemResponse( + val name: String, + @JsonProperty("full_name") + val fullName: String, +) diff --git a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt new file mode 100644 index 0000000..f052c8d --- /dev/null +++ b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt @@ -0,0 +1,12 @@ +package net.leanix.githubagent.dto + +data class ManifestFilesDTO( + val repositoryId: String, + val repositoryFullName: String, + val manifestFiles: List +) + +data class ManifestFileDTO( + val path: String, + val content: String, +) diff --git a/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt index f1ea8ff..7b55086 100644 --- a/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt +++ b/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt @@ -13,4 +13,6 @@ data class RepositoryDto( val updatedAt: String, val languages: List, val topics: List, -) +) { + val fullName: String = "$organizationName/$name" +} diff --git a/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt index e1f3319..b608142 100644 --- a/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt +++ b/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt @@ -6,7 +6,7 @@ data class SyncLogDto( val runId: UUID?, val trigger: Trigger, val logLevel: LogLevel, - val message: String + val message: String? ) enum class Trigger { diff --git a/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt b/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt index 04bca83..cb64d29 100644 --- a/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt +++ b/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt @@ -13,3 +13,4 @@ class GraphQLApiException(errors: List) : RuntimeException("Errors: ${errors.joinToString(separator = "\n") { it.message }}") class WebhookSecretNotSetException : RuntimeException("Webhook secret not set") class InvalidEventSignatureException : RuntimeException("Invalid event signature") +class ManifestFileNotFoundException : RuntimeException("Manifest File Not Found") diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt index b8e8c9f..4f8a9d0 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt @@ -2,13 +2,11 @@ package net.leanix.githubagent.services import com.expediagroup.graphql.client.spring.GraphQLWebClient import kotlinx.coroutines.runBlocking -import net.leanix.githubagent.config.GitHubEnterpriseProperties import net.leanix.githubagent.dto.PagedRepositories import net.leanix.githubagent.dto.RepositoryDto import net.leanix.githubagent.exceptions.GraphQLApiException import net.leanix.githubagent.graphql.data.GetRepositories import net.leanix.githubagent.graphql.data.GetRepositoryManifestContent -import net.leanix.githubagent.shared.ManifestFileName import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import org.springframework.web.reactive.function.client.WebClient @@ -16,7 +14,6 @@ import org.springframework.web.reactive.function.client.WebClient @Component class GitHubGraphQLService( private val cachingService: CachingService, - private val gitHubEnterpriseProperties: GitHubEnterpriseProperties ) { companion object { private val logger = LoggerFactory.getLogger(GitHubGraphQLService::class.java) @@ -33,8 +30,6 @@ class GitHubGraphQLService( GetRepositories.Variables( pageCount = PAGE_COUNT, cursor = cursor, - "HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YAML.fileName}", - "HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YML.fileName}" ) ) diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt index 77ad4e4..93bd6d0 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt @@ -2,9 +2,17 @@ package net.leanix.githubagent.services import net.leanix.githubagent.client.GitHubClient import net.leanix.githubagent.dto.Installation +import net.leanix.githubagent.dto.ItemResponse +import net.leanix.githubagent.dto.LogLevel +import net.leanix.githubagent.dto.ManifestFileDTO +import net.leanix.githubagent.dto.ManifestFilesDTO import net.leanix.githubagent.dto.Organization import net.leanix.githubagent.dto.OrganizationDto +import net.leanix.githubagent.dto.RepositoryDto +import net.leanix.githubagent.dto.Trigger import net.leanix.githubagent.exceptions.JwtTokenNotFound +import net.leanix.githubagent.exceptions.ManifestFileNotFoundException +import net.leanix.githubagent.shared.ManifestFileName import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.util.UUID @@ -24,15 +32,30 @@ class GitHubScanningService( fun scanGitHubResources() { cachingService.set("runId", UUID.randomUUID(), null) runCatching { + syncLogService.sendSyncLog( + trigger = Trigger.START_FULL_SYNC, + logLevel = LogLevel.INFO, + ) val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound() val installations = getInstallations(jwtToken.toString()) fetchAndSendOrganisationsData(installations) installations.forEach { installation -> fetchAndSendRepositoriesData(installation) + .forEach { repository -> + fetchManifestFilesAndSend(installation, repository) + } } + syncLogService.sendSyncLog( + trigger = Trigger.FINISH_FULL_SYNC, + logLevel = LogLevel.INFO, + ) }.onFailure { val message = "Error while scanning GitHub resources" - syncLogService.sendErrorLog(message) + syncLogService.sendSyncLog( + trigger = Trigger.FINISH_FULL_SYNC, + logLevel = LogLevel.ERROR, + message = message + ) cachingService.remove("runId") logger.error(message) throw it @@ -66,11 +89,12 @@ class GitHubScanningService( webSocketService.sendMessage("${cachingService.get("runId")}/organizations", organizations) } - private fun fetchAndSendRepositoriesData(installation: Installation) { + private fun fetchAndSendRepositoriesData(installation: Installation): List { val installationToken = cachingService.get("installationToken:${installation.id}").toString() var cursor: String? = null var totalRepos = 0 var page = 1 + val repositories = mutableListOf() do { val repositoriesPage = gitHubGraphQLService.getRepositories( token = installationToken, @@ -80,10 +104,58 @@ class GitHubScanningService( "${cachingService.get("runId")}/repositories", repositoriesPage.repositories ) + repositories.addAll(repositoriesPage.repositories) cursor = repositoriesPage.cursor totalRepos += repositoriesPage.repositories.size page++ } while (repositoriesPage.hasNextPage) logger.info("Fetched $totalRepos repositories data from organisation ${installation.account.login}") + return repositories + } + + private fun fetchManifestFilesAndSend(installation: Installation, repository: RepositoryDto) { + val manifestFiles = fetchManifestFiles(installation, repository.name).getOrThrow().items + val manifestFilesContents = fetchManifestContents(installation, manifestFiles, repository.name).getOrThrow() + + webSocketService.sendMessage( + "${cachingService.get("runId")}/manifestFiles", + ManifestFilesDTO( + repositoryId = repository.id, + repositoryFullName = repository.fullName, + manifestFiles = manifestFilesContents, + ) + ) + } + + private fun fetchManifestFiles(installation: Installation, repositoryName: String) = runCatching { + val installationToken = cachingService.get("installationToken:${installation.id}").toString() + gitHubClient.searchManifestFiles( + "Bearer $installationToken", + "" + + "repo:${installation.account.login}/$repositoryName filename:${ManifestFileName.YAML.fileName}" + ) + } + private fun fetchManifestContents( + installation: Installation, + items: List, + repositoryName: String + ) = runCatching { + val installationToken = cachingService.get("installationToken:${installation.id}").toString() + items.map { manifestFile -> + val content = gitHubGraphQLService.getManifestFileContent( + owner = installation.account.login, + repositoryName = repositoryName, + filePath = manifestFile.path, + token = installationToken + ) + if (content != null) { + ManifestFileDTO( + path = manifestFile.path.replace("/${ManifestFileName.YAML.fileName}", ""), + content = content + ) + } else { + throw ManifestFileNotFoundException() + } + } } } diff --git a/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt b/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt index aff569c..bb3afa1 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt @@ -16,7 +16,7 @@ class SyncLogService( sendSyncLog(message, LOGS_TOPIC, null, LogLevel.ERROR) } - fun sendSyncLog(message: String, topic: String, trigger: Trigger?, logLevel: LogLevel) { + fun sendSyncLog(message: String? = null, topic: String = LOGS_TOPIC, trigger: Trigger?, logLevel: LogLevel) { val runId = cachingService.get("runId") as UUID val syncLogDto = SyncLogDto( runId = runId, @@ -28,6 +28,6 @@ class SyncLogService( } private fun constructTopic(topic: String): String { - return "${cachingService.get("runId")}/$topic" + return topic } } diff --git a/src/main/resources/graphql/GetRepositories.graphql b/src/main/resources/graphql/GetRepositories.graphql index 747ddde..01bcbab 100644 --- a/src/main/resources/graphql/GetRepositories.graphql +++ b/src/main/resources/graphql/GetRepositories.graphql @@ -1,4 +1,4 @@ -query GetRepositories($pageCount: Int!, $cursor: String, $manifestYamlPath: String!, $manifestYmlPath: String!) { +query GetRepositories($pageCount: Int!, $cursor: String) { viewer { repositories(first: $pageCount, after: $cursor) { pageInfo { diff --git a/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt index 2d2c553..e83c646 100644 --- a/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt +++ b/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt @@ -5,11 +5,14 @@ import io.mockk.mockk import io.mockk.verify import net.leanix.githubagent.client.GitHubClient import net.leanix.githubagent.dto.Account +import net.leanix.githubagent.dto.GitHubSearchResponse import net.leanix.githubagent.dto.Installation import net.leanix.githubagent.dto.InstallationTokenResponse +import net.leanix.githubagent.dto.ItemResponse import net.leanix.githubagent.dto.Organization import net.leanix.githubagent.dto.PagedRepositories import net.leanix.githubagent.dto.RepositoryDto +import net.leanix.githubagent.dto.RepositoryItemResponse import net.leanix.githubagent.exceptions.JwtTokenNotFound import net.leanix.githubagent.graphql.data.enums.RepositoryVisibility import org.junit.jupiter.api.BeforeEach @@ -19,12 +22,12 @@ import java.util.UUID class GitHubScanningServiceTest { - private val gitHubClient = mockk() + private val gitHubClient = mockk(relaxUnitFun = true) private val cachingService = mockk() private val webSocketService = mockk(relaxUnitFun = true) private val gitHubGraphQLService = mockk() private val gitHubAuthenticationService = mockk() - private val syncLogService = mockk() + private val syncLogService = mockk(relaxUnitFun = true) private val gitHubScanningService = GitHubScanningService( gitHubClient, cachingService, @@ -103,7 +106,53 @@ class GitHubScanningServiceTest { hasNextPage = false, cursor = null ) + every { gitHubClient.searchManifestFiles(any(), any()) } returns GitHubSearchResponse(0, emptyList()) gitHubScanningService.scanGitHubResources() verify { webSocketService.sendMessage(eq("$runId/repositories"), any()) } } + + @Test + fun `scanGitHubResources should send repositories and manifest files over WebSocket`() { + // given + every { cachingService.get("runId") } returns runId + every { gitHubGraphQLService.getRepositories(any(), any()) } returns PagedRepositories( + repositories = listOf( + RepositoryDto( + id = "repo1", + name = "TestRepo", + organizationName = "testOrg", + description = "A test repository", + url = "https://github.com/testRepo", + archived = false, + visibility = RepositoryVisibility.PUBLIC, + updatedAt = "2024-01-01T00:00:00Z", + languages = listOf("Kotlin", "Java"), + topics = listOf("test", "example"), + ) + ), + hasNextPage = false, + cursor = null + ) + every { gitHubClient.searchManifestFiles(any(), any()) } returns GitHubSearchResponse( + 1, + listOf( + ItemResponse( + name = "leanix.yaml", + path = "dir/leanix.yaml", + repository = RepositoryItemResponse( + name = "TestRepo", + fullName = "testOrg/TestRepo" + ), + url = "http://url" + ) + ) + ) + every { gitHubGraphQLService.getManifestFileContent(any(), any(), "dir/leanix.yaml", any()) } returns "content" + + // when + gitHubScanningService.scanGitHubResources() + + // then + verify { webSocketService.sendMessage(eq("$runId/manifestFiles"), any()) } + } }