diff --git a/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt b/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt
index c69e1e4..d333087 100644
--- a/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt
+++ b/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt
@@ -1,6 +1,7 @@
package net.leanix.githubagent.client
import net.leanix.githubagent.dto.GitHubAppResponse
+import net.leanix.githubagent.dto.GitHubSearchResponse
import net.leanix.githubagent.dto.Installation
import net.leanix.githubagent.dto.InstallationTokenResponse
import net.leanix.githubagent.dto.Organization
@@ -10,6 +11,7 @@ import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.PathVariable
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestHeader
+import org.springframework.web.bind.annotation.RequestParam
@FeignClient(name = "githubClient", url = "\${github-enterprise.baseUrl}")
interface GitHubClient {
@@ -37,4 +39,10 @@ interface GitHubClient {
@PathVariable("org") org: String,
@RequestHeader("Authorization") token: String
): List<Repository>
+
+ @GetMapping("/api/v3/search/code")
+ fun searchManifestFiles(
+ @RequestHeader("Authorization") token: String,
+ @RequestParam("q") query: String,
+ ): GitHubSearchResponse
}
diff --git a/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt b/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt
new file mode 100644
index 0000000..585541f
--- /dev/null
+++ b/src/main/kotlin/net/leanix/githubagent/dto/GitHubSearchResponse.kt
@@ -0,0 +1,26 @@
+package net.leanix.githubagent.dto
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonProperty
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class GitHubSearchResponse(
+ @JsonProperty("total_count")
+ val totalCount: Int,
+ val items: List<ItemResponse>
+)
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class ItemResponse(
+ val name: String,
+ val path: String,
+ val repository: RepositoryItemResponse,
+ val url: String,
+)
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+data class RepositoryItemResponse(
+ val name: String,
+ @JsonProperty("full_name")
+ val fullName: String,
+)
diff --git a/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt
new file mode 100644
index 0000000..f052c8d
--- /dev/null
+++ b/src/main/kotlin/net/leanix/githubagent/dto/ManifestFilesDTO.kt
@@ -0,0 +1,12 @@
+package net.leanix.githubagent.dto
+
+data class ManifestFilesDTO(
+ val repositoryId: String,
+ val repositoryFullName: String,
+ val manifestFiles: List<ManifestFileDTO>
+)
+
+data class ManifestFileDTO(
+ val path: String,
+ val content: String,
+)
diff --git a/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt
index f1ea8ff..7b55086 100644
--- a/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt
+++ b/src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt
@@ -13,4 +13,6 @@ data class RepositoryDto(
val updatedAt: String,
val languages: List<String>,
val topics: List<String>,
-)
+) {
+ val fullName: String = "$organizationName/$name"
+}
diff --git a/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt b/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt
index e1f3319..b608142 100644
--- a/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt
+++ b/src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt
@@ -6,7 +6,7 @@ data class SyncLogDto(
val runId: UUID?,
val trigger: Trigger,
val logLevel: LogLevel,
- val message: String
+ val message: String?
)
enum class Trigger {
diff --git a/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt b/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt
index 04bca83..cb64d29 100644
--- a/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt
+++ b/src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt
@@ -13,3 +13,4 @@ class GraphQLApiException(errors: List<GraphQLClientError>) :
RuntimeException("Errors: ${errors.joinToString(separator = "\n") { it.message }}")
class WebhookSecretNotSetException : RuntimeException("Webhook secret not set")
class InvalidEventSignatureException : RuntimeException("Invalid event signature")
+class ManifestFileNotFoundException : RuntimeException("Manifest File Not Found")
diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt
index b8e8c9f..4f8a9d0 100644
--- a/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt
+++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt
@@ -2,13 +2,11 @@ package net.leanix.githubagent.services
import com.expediagroup.graphql.client.spring.GraphQLWebClient
import kotlinx.coroutines.runBlocking
-import net.leanix.githubagent.config.GitHubEnterpriseProperties
import net.leanix.githubagent.dto.PagedRepositories
import net.leanix.githubagent.dto.RepositoryDto
import net.leanix.githubagent.exceptions.GraphQLApiException
import net.leanix.githubagent.graphql.data.GetRepositories
import net.leanix.githubagent.graphql.data.GetRepositoryManifestContent
-import net.leanix.githubagent.shared.ManifestFileName
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Component
import org.springframework.web.reactive.function.client.WebClient
@@ -16,7 +14,6 @@ import org.springframework.web.reactive.function.client.WebClient
@Component
class GitHubGraphQLService(
private val cachingService: CachingService,
- private val gitHubEnterpriseProperties: GitHubEnterpriseProperties
) {
companion object {
private val logger = LoggerFactory.getLogger(GitHubGraphQLService::class.java)
@@ -33,8 +30,6 @@ class GitHubGraphQLService(
GetRepositories.Variables(
pageCount = PAGE_COUNT,
cursor = cursor,
- "HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YAML.fileName}",
- "HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YML.fileName}"
)
)
diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt
index 77ad4e4..93bd6d0 100644
--- a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt
+++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt
@@ -2,9 +2,17 @@ package net.leanix.githubagent.services
import net.leanix.githubagent.client.GitHubClient
import net.leanix.githubagent.dto.Installation
+import net.leanix.githubagent.dto.ItemResponse
+import net.leanix.githubagent.dto.LogLevel
+import net.leanix.githubagent.dto.ManifestFileDTO
+import net.leanix.githubagent.dto.ManifestFilesDTO
import net.leanix.githubagent.dto.Organization
import net.leanix.githubagent.dto.OrganizationDto
+import net.leanix.githubagent.dto.RepositoryDto
+import net.leanix.githubagent.dto.Trigger
import net.leanix.githubagent.exceptions.JwtTokenNotFound
+import net.leanix.githubagent.exceptions.ManifestFileNotFoundException
+import net.leanix.githubagent.shared.ManifestFileName
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import java.util.UUID
@@ -24,15 +32,30 @@ class GitHubScanningService(
fun scanGitHubResources() {
cachingService.set("runId", UUID.randomUUID(), null)
runCatching {
+ syncLogService.sendSyncLog(
+ trigger = Trigger.START_FULL_SYNC,
+ logLevel = LogLevel.INFO,
+ )
val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound()
val installations = getInstallations(jwtToken.toString())
fetchAndSendOrganisationsData(installations)
installations.forEach { installation ->
fetchAndSendRepositoriesData(installation)
+ .forEach { repository ->
+ fetchManifestFilesAndSend(installation, repository)
+ }
}
+ syncLogService.sendSyncLog(
+ trigger = Trigger.FINISH_FULL_SYNC,
+ logLevel = LogLevel.INFO,
+ )
}.onFailure {
val message = "Error while scanning GitHub resources"
- syncLogService.sendErrorLog(message)
+ syncLogService.sendSyncLog(
+ trigger = Trigger.FINISH_FULL_SYNC,
+ logLevel = LogLevel.ERROR,
+ message = message
+ )
cachingService.remove("runId")
logger.error(message)
throw it
@@ -66,11 +89,12 @@ class GitHubScanningService(
webSocketService.sendMessage("${cachingService.get("runId")}/organizations", organizations)
}
- private fun fetchAndSendRepositoriesData(installation: Installation) {
+ private fun fetchAndSendRepositoriesData(installation: Installation): List<RepositoryDto> {
val installationToken = cachingService.get("installationToken:${installation.id}").toString()
var cursor: String? = null
var totalRepos = 0
var page = 1
+ val repositories = mutableListOf<RepositoryDto>()
do {
val repositoriesPage = gitHubGraphQLService.getRepositories(
token = installationToken,
@@ -80,10 +104,58 @@ class GitHubScanningService(
"${cachingService.get("runId")}/repositories",
repositoriesPage.repositories
)
+ repositories.addAll(repositoriesPage.repositories)
cursor = repositoriesPage.cursor
totalRepos += repositoriesPage.repositories.size
page++
} while (repositoriesPage.hasNextPage)
logger.info("Fetched $totalRepos repositories data from organisation ${installation.account.login}")
+ return repositories
+ }
+
+ private fun fetchManifestFilesAndSend(installation: Installation, repository: RepositoryDto) {
+ val manifestFiles = fetchManifestFiles(installation, repository.name).getOrThrow().items
+ val manifestFilesContents = fetchManifestContents(installation, manifestFiles, repository.name).getOrThrow()
+
+ webSocketService.sendMessage(
+ "${cachingService.get("runId")}/manifestFiles",
+ ManifestFilesDTO(
+ repositoryId = repository.id,
+ repositoryFullName = repository.fullName,
+ manifestFiles = manifestFilesContents,
+ )
+ )
+ }
+
+ private fun fetchManifestFiles(installation: Installation, repositoryName: String) = runCatching {
+ val installationToken = cachingService.get("installationToken:${installation.id}").toString()
+ gitHubClient.searchManifestFiles(
+ "Bearer $installationToken",
+ "" +
+ "repo:${installation.account.login}/$repositoryName filename:${ManifestFileName.YAML.fileName}"
+ )
+ }
+ private fun fetchManifestContents(
+ installation: Installation,
+ items: List<ItemResponse>,
+ repositoryName: String
+ ) = runCatching {
+ val installationToken = cachingService.get("installationToken:${installation.id}").toString()
+ items.map { manifestFile ->
+ val content = gitHubGraphQLService.getManifestFileContent(
+ owner = installation.account.login,
+ repositoryName = repositoryName,
+ filePath = manifestFile.path,
+ token = installationToken
+ )
+ if (content != null) {
+ ManifestFileDTO(
+ path = manifestFile.path.replace("/${ManifestFileName.YAML.fileName}", ""),
+ content = content
+ )
+ } else {
+ throw ManifestFileNotFoundException()
+ }
+ }
}
}
diff --git a/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt b/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt
index aff569c..bb3afa1 100644
--- a/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt
+++ b/src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt
@@ -16,7 +16,7 @@ class SyncLogService(
sendSyncLog(message, LOGS_TOPIC, null, LogLevel.ERROR)
}
- fun sendSyncLog(message: String, topic: String, trigger: Trigger?, logLevel: LogLevel) {
+ fun sendSyncLog(message: String? = null, topic: String = LOGS_TOPIC, trigger: Trigger?, logLevel: LogLevel) {
val runId = cachingService.get("runId") as UUID
val syncLogDto = SyncLogDto(
runId = runId,
@@ -28,6 +28,6 @@ class SyncLogService(
}
private fun constructTopic(topic: String): String {
- return "${cachingService.get("runId")}/$topic"
+ return topic
}
}
diff --git a/src/main/resources/graphql/GetRepositories.graphql b/src/main/resources/graphql/GetRepositories.graphql
index 747ddde..01bcbab 100644
--- a/src/main/resources/graphql/GetRepositories.graphql
+++ b/src/main/resources/graphql/GetRepositories.graphql
@@ -1,4 +1,4 @@
-query GetRepositories($pageCount: Int!, $cursor: String, $manifestYamlPath: String!, $manifestYmlPath: String!) {
+query GetRepositories($pageCount: Int!, $cursor: String) {
viewer {
repositories(first: $pageCount, after: $cursor) {
pageInfo {
diff --git a/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt
index 2d2c553..e83c646 100644
--- a/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt
+++ b/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt
@@ -5,11 +5,14 @@ import io.mockk.mockk
import io.mockk.verify
import net.leanix.githubagent.client.GitHubClient
import net.leanix.githubagent.dto.Account
+import net.leanix.githubagent.dto.GitHubSearchResponse
import net.leanix.githubagent.dto.Installation
import net.leanix.githubagent.dto.InstallationTokenResponse
+import net.leanix.githubagent.dto.ItemResponse
import net.leanix.githubagent.dto.Organization
import net.leanix.githubagent.dto.PagedRepositories
import net.leanix.githubagent.dto.RepositoryDto
+import net.leanix.githubagent.dto.RepositoryItemResponse
import net.leanix.githubagent.exceptions.JwtTokenNotFound
import net.leanix.githubagent.graphql.data.enums.RepositoryVisibility
import org.junit.jupiter.api.BeforeEach
@@ -19,12 +22,12 @@ import java.util.UUID
class GitHubScanningServiceTest {
- private val gitHubClient = mockk<GitHubClient>()
+ private val gitHubClient = mockk<GitHubClient>(relaxUnitFun = true)
private val cachingService = mockk<CachingService>()
private val webSocketService = mockk<WebSocketService>(relaxUnitFun = true)
private val gitHubGraphQLService = mockk<GitHubGraphQLService>()
private val gitHubAuthenticationService = mockk<GitHubAuthenticationService>()
- private val syncLogService = mockk<SyncLogService>()
+ private val syncLogService = mockk<SyncLogService>(relaxUnitFun = true)
private val gitHubScanningService = GitHubScanningService(
gitHubClient,
cachingService,
@@ -103,7 +106,53 @@ class GitHubScanningServiceTest {
hasNextPage = false,
cursor = null
)
+ every { gitHubClient.searchManifestFiles(any(), any()) } returns GitHubSearchResponse(0, emptyList())
gitHubScanningService.scanGitHubResources()
verify { webSocketService.sendMessage(eq("$runId/repositories"), any()) }
}
+
+ @Test
+ fun `scanGitHubResources should send repositories and manifest files over WebSocket`() {
+ // given
+ every { cachingService.get("runId") } returns runId
+ every { gitHubGraphQLService.getRepositories(any(), any()) } returns PagedRepositories(
+ repositories = listOf(
+ RepositoryDto(
+ id = "repo1",
+ name = "TestRepo",
+ organizationName = "testOrg",
+ description = "A test repository",
+ url = "https://github.com/testRepo",
+ archived = false,
+ visibility = RepositoryVisibility.PUBLIC,
+ updatedAt = "2024-01-01T00:00:00Z",
+ languages = listOf("Kotlin", "Java"),
+ topics = listOf("test", "example"),
+ )
+ ),
+ hasNextPage = false,
+ cursor = null
+ )
+ every { gitHubClient.searchManifestFiles(any(), any()) } returns GitHubSearchResponse(
+ 1,
+ listOf(
+ ItemResponse(
+ name = "leanix.yaml",
+ path = "dir/leanix.yaml",
+ repository = RepositoryItemResponse(
+ name = "TestRepo",
+ fullName = "testOrg/TestRepo"
+ ),
+ url = "http://url"
+ )
+ )
+ )
+ every { gitHubGraphQLService.getManifestFileContent(any(), any(), "dir/leanix.yaml", any()) } returns "content"
+
+ // when
+ gitHubScanningService.scanGitHubResources()
+
+ // then
+ verify { webSocketService.sendMessage(eq("$runId/manifestFiles"), any()) }
+ }
}