Merge pull request #30 from leanix/feature/cid-2908-send-multi-manife…

…st-files Feature/cid 2908 send multi manifest files
leanix · Oct 28, 2024 · bfde0ee · bfde0ee
2 parents 7d0a6db + b6c57e3
commit bfde0ee
Show file tree

Hide file tree

Showing 11 changed files with 179 additions and 14 deletions.
diff --git a/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt b/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt
@@ -1,6 +1,7 @@
 package net.leanix.githubagent.client
 
 import net.leanix.githubagent.dto.GitHubAppResponse
+import net.leanix.githubagent.dto.GitHubSearchResponse
 import net.leanix.githubagent.dto.Installation
 import net.leanix.githubagent.dto.InstallationTokenResponse
 import net.leanix.githubagent.dto.Organization
@@ -10,6 +11,7 @@ import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.PathVariable
 import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestHeader
+import org.springframework.web.bind.annotation.RequestParam
 
 @FeignClient(name = "githubClient", url = "\${github-enterprise.baseUrl}")
 interface GitHubClient {
@@ -37,4 +39,10 @@ interface GitHubClient {
         @PathVariable("org") org: String,
         @RequestHeader("Authorization") token: String
     ): List<Repository>
+
+    @GetMapping("/api/v3/search/code")
+    fun searchManifestFiles(
+        @RequestHeader("Authorization") token: String,
+        @RequestParam("q") query: String,
+    ): GitHubSearchResponse
 }
diff --git a/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt b/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt
@@ -0,0 +1,26 @@
+package net.leanix.githubagent.dto
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonProperty
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class GitHubSearchResponse(
+    @JsonProperty("total_count")
+    val totalCount: Int,
+    val items: List<ItemResponse>
+)
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class ItemResponse(
+    val name: String,
+    val path: String,
+    val repository: RepositoryItemResponse,
+    val url: String,
+)
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class RepositoryItemResponse(
+    val name: String,
+    @JsonProperty("full_name")
+    val fullName: String,
+)
diff --git a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt
@@ -0,0 +1,12 @@
+package net.leanix.githubagent.dto
+
+data class ManifestFilesDTO(
+    val repositoryId: String,
+    val repositoryFullName: String,
+    val manifestFiles: List<ManifestFileDTO>
+)
+
+data class ManifestFileDTO(
+    val path: String,
+    val content: String,
+)
diff --git a/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt
@@ -13,4 +13,6 @@ data class RepositoryDto(
     val updatedAt: String,
     val languages: List<String>,
     val topics: List<String>,
-)
+) {
+    val fullName: String = "$organizationName/$name"
+}
diff --git a/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt
@@ -6,7 +6,7 @@ data class SyncLogDto(
     val runId: UUID?,
     val trigger: Trigger,
     val logLevel: LogLevel,
-    val message: String
+    val message: String?
 )
 
 enum class Trigger {

diff --git a/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt b/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt
@@ -13,3 +13,4 @@ class GraphQLApiException(errors: List<GraphQLClientError>) :
     RuntimeException("Errors: ${errors.joinToString(separator = "\n") { it.message }}")
 class WebhookSecretNotSetException : RuntimeException("Webhook secret not set")
 class InvalidEventSignatureException : RuntimeException("Invalid event signature")
+class ManifestFileNotFoundException : RuntimeException("Manifest File Not Found")
diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt
@@ -2,21 +2,18 @@ package net.leanix.githubagent.services
 
 import com.expediagroup.graphql.client.spring.GraphQLWebClient
 import kotlinx.coroutines.runBlocking
-import net.leanix.githubagent.config.GitHubEnterpriseProperties
 import net.leanix.githubagent.dto.PagedRepositories
 import net.leanix.githubagent.dto.RepositoryDto
 import net.leanix.githubagent.exceptions.GraphQLApiException
 import net.leanix.githubagent.graphql.data.GetRepositories
 import net.leanix.githubagent.graphql.data.GetRepositoryManifestContent
-import net.leanix.githubagent.shared.ManifestFileName
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Component
 import org.springframework.web.reactive.function.client.WebClient
 
 @Component
 class GitHubGraphQLService(
     private val cachingService: CachingService,
-    private val gitHubEnterpriseProperties: GitHubEnterpriseProperties
 ) {
     companion object {
         private val logger = LoggerFactory.getLogger(GitHubGraphQLService::class.java)
@@ -33,8 +30,6 @@ class GitHubGraphQLService(
             GetRepositories.Variables(
                 pageCount = PAGE_COUNT,
                 cursor = cursor,
-                "HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YAML.fileName}",
-                "HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YML.fileName}"
             )
         )
 

diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt
@@ -2,9 +2,17 @@ package net.leanix.githubagent.services
 
 import net.leanix.githubagent.client.GitHubClient
 import net.leanix.githubagent.dto.Installation
+import net.leanix.githubagent.dto.ItemResponse
+import net.leanix.githubagent.dto.LogLevel
+import net.leanix.githubagent.dto.ManifestFileDTO
+import net.leanix.githubagent.dto.ManifestFilesDTO
 import net.leanix.githubagent.dto.Organization
 import net.leanix.githubagent.dto.OrganizationDto
+import net.leanix.githubagent.dto.RepositoryDto
+import net.leanix.githubagent.dto.Trigger
 import net.leanix.githubagent.exceptions.JwtTokenNotFound
+import net.leanix.githubagent.exceptions.ManifestFileNotFoundException
+import net.leanix.githubagent.shared.ManifestFileName
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Service
 import java.util.UUID
@@ -24,15 +32,30 @@ class GitHubScanningService(
     fun scanGitHubResources() {
         cachingService.set("runId", UUID.randomUUID(), null)
         runCatching {
+            syncLogService.sendSyncLog(
+                trigger = Trigger.START_FULL_SYNC,
+                logLevel = LogLevel.INFO,
+            )
             val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound()
             val installations = getInstallations(jwtToken.toString())
             fetchAndSendOrganisationsData(installations)
             installations.forEach { installation ->
                 fetchAndSendRepositoriesData(installation)
+                    .forEach { repository ->
+                        fetchManifestFilesAndSend(installation, repository)
+                    }
             }
+            syncLogService.sendSyncLog(
+                trigger = Trigger.FINISH_FULL_SYNC,
+                logLevel = LogLevel.INFO,
+            )
         }.onFailure {
             val message = "Error while scanning GitHub resources"
-            syncLogService.sendErrorLog(message)
+            syncLogService.sendSyncLog(
+                trigger = Trigger.FINISH_FULL_SYNC,
+                logLevel = LogLevel.ERROR,
+                message = message
+            )
             cachingService.remove("runId")
             logger.error(message)
             throw it
@@ -66,11 +89,12 @@ class GitHubScanningService(
         webSocketService.sendMessage("${cachingService.get("runId")}/organizations", organizations)
     }
 
-    private fun fetchAndSendRepositoriesData(installation: Installation) {
+    private fun fetchAndSendRepositoriesData(installation: Installation): List<RepositoryDto> {
         val installationToken = cachingService.get("installationToken:${installation.id}").toString()
         var cursor: String? = null
         var totalRepos = 0
         var page = 1
+        val repositories = mutableListOf<RepositoryDto>()
         do {
             val repositoriesPage = gitHubGraphQLService.getRepositories(
                 token = installationToken,
@@ -80,10 +104,58 @@ class GitHubScanningService(
                 "${cachingService.get("runId")}/repositories",
                 repositoriesPage.repositories
             )
+            repositories.addAll(repositoriesPage.repositories)
             cursor = repositoriesPage.cursor
             totalRepos += repositoriesPage.repositories.size
             page++
         } while (repositoriesPage.hasNextPage)
         logger.info("Fetched $totalRepos repositories data from organisation ${installation.account.login}")
+        return repositories
+    }
+
+    private fun fetchManifestFilesAndSend(installation: Installation, repository: RepositoryDto) {
+        val manifestFiles = fetchManifestFiles(installation, repository.name).getOrThrow().items
+        val manifestFilesContents = fetchManifestContents(installation, manifestFiles, repository.name).getOrThrow()
+
+        webSocketService.sendMessage(
+            "${cachingService.get("runId")}/manifestFiles",
+            ManifestFilesDTO(
+                repositoryId = repository.id,
+                repositoryFullName = repository.fullName,
+                manifestFiles = manifestFilesContents,
+            )
+        )
+    }
+
+    private fun fetchManifestFiles(installation: Installation, repositoryName: String) = runCatching {
+        val installationToken = cachingService.get("installationToken:${installation.id}").toString()
+        gitHubClient.searchManifestFiles(
+            "Bearer $installationToken",
+            "" +
+                "repo:${installation.account.login}/$repositoryName filename:${ManifestFileName.YAML.fileName}"
+        )
+    }
+    private fun fetchManifestContents(
+        installation: Installation,
+        items: List<ItemResponse>,
+        repositoryName: String
+    ) = runCatching {
+        val installationToken = cachingService.get("installationToken:${installation.id}").toString()
+        items.map { manifestFile ->
+            val content = gitHubGraphQLService.getManifestFileContent(
+                owner = installation.account.login,
+                repositoryName = repositoryName,
+                filePath = manifestFile.path,
+                token = installationToken
+            )
+            if (content != null) {
+                ManifestFileDTO(
+                    path = manifestFile.path.replace("/${ManifestFileName.YAML.fileName}", ""),
+                    content = content
+                )
+            } else {
+                throw ManifestFileNotFoundException()
+            }
+        }
     }
 }
diff --git a/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt b/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt
@@ -16,7 +16,7 @@ class SyncLogService(
         sendSyncLog(message, LOGS_TOPIC, null, LogLevel.ERROR)
     }
 
-    fun sendSyncLog(message: String, topic: String, trigger: Trigger?, logLevel: LogLevel) {
+    fun sendSyncLog(message: String? = null, topic: String = LOGS_TOPIC, trigger: Trigger?, logLevel: LogLevel) {
         val runId = cachingService.get("runId") as UUID
         val syncLogDto = SyncLogDto(
             runId = runId,
@@ -28,6 +28,6 @@ class SyncLogService(
     }
 
     private fun constructTopic(topic: String): String {
-        return "${cachingService.get("runId")}/$topic"
+        return topic
     }
 }
diff --git a/src/main/resources/graphql/GetRepositories.graphql b/src/main/resources/graphql/GetRepositories.graphql
@@ -1,4 +1,4 @@
-query GetRepositories($pageCount: Int!, $cursor: String, $manifestYamlPath: String!, $manifestYmlPath: String!) {
+query GetRepositories($pageCount: Int!, $cursor: String) {
     viewer {
         repositories(first: $pageCount, after: $cursor) {
             pageInfo {

diff --git a/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt
@@ -5,11 +5,14 @@ import io.mockk.mockk
 import io.mockk.verify
 import net.leanix.githubagent.client.GitHubClient
 import net.leanix.githubagent.dto.Account
+import net.leanix.githubagent.dto.GitHubSearchResponse
 import net.leanix.githubagent.dto.Installation
 import net.leanix.githubagent.dto.InstallationTokenResponse
+import net.leanix.githubagent.dto.ItemResponse
 import net.leanix.githubagent.dto.Organization
 import net.leanix.githubagent.dto.PagedRepositories
 import net.leanix.githubagent.dto.RepositoryDto
+import net.leanix.githubagent.dto.RepositoryItemResponse
 import net.leanix.githubagent.exceptions.JwtTokenNotFound
 import net.leanix.githubagent.graphql.data.enums.RepositoryVisibility
 import org.junit.jupiter.api.BeforeEach
@@ -19,12 +22,12 @@ import java.util.UUID
 
 class GitHubScanningServiceTest {
 
-    private val gitHubClient = mockk<GitHubClient>()
+    private val gitHubClient = mockk<GitHubClient>(relaxUnitFun = true)
     private val cachingService = mockk<CachingService>()
     private val webSocketService = mockk<WebSocketService>(relaxUnitFun = true)
     private val gitHubGraphQLService = mockk<GitHubGraphQLService>()
     private val gitHubAuthenticationService = mockk<GitHubAuthenticationService>()
-    private val syncLogService = mockk<SyncLogService>()
+    private val syncLogService = mockk<SyncLogService>(relaxUnitFun = true)
     private val gitHubScanningService = GitHubScanningService(
         gitHubClient,
         cachingService,
@@ -103,7 +106,53 @@ class GitHubScanningServiceTest {
             hasNextPage = false,
             cursor = null
         )
+        every { gitHubClient.searchManifestFiles(any(), any()) } returns GitHubSearchResponse(0, emptyList())
         gitHubScanningService.scanGitHubResources()
         verify { webSocketService.sendMessage(eq("$runId/repositories"), any()) }
     }
+
+    @Test
+    fun `scanGitHubResources should send repositories and manifest files over WebSocket`() {
+        // given
+        every { cachingService.get("runId") } returns runId
+        every { gitHubGraphQLService.getRepositories(any(), any()) } returns PagedRepositories(
+            repositories = listOf(
+                RepositoryDto(
+                    id = "repo1",
+                    name = "TestRepo",
+                    organizationName = "testOrg",
+                    description = "A test repository",
+                    url = "https://github.com/testRepo",
+                    archived = false,
+                    visibility = RepositoryVisibility.PUBLIC,
+                    updatedAt = "2024-01-01T00:00:00Z",
+                    languages = listOf("Kotlin", "Java"),
+                    topics = listOf("test", "example"),
+                )
+            ),
+            hasNextPage = false,
+            cursor = null
+        )
+        every { gitHubClient.searchManifestFiles(any(), any()) } returns GitHubSearchResponse(
+            1,
+            listOf(
+                ItemResponse(
+                    name = "leanix.yaml",
+                    path = "dir/leanix.yaml",
+                    repository = RepositoryItemResponse(
+                        name = "TestRepo",
+                        fullName = "testOrg/TestRepo"
+                    ),
+                    url = "http://url"
+                )
+            )
+        )
+        every { gitHubGraphQLService.getManifestFileContent(any(), any(), "dir/leanix.yaml", any()) } returns "content"
+
+        // when
+        gitHubScanningService.scanGitHubResources()
+
+        // then
+        verify { webSocketService.sendMessage(eq("$runId/manifestFiles"), any()) }
+    }
 }