Skip to content

Commit

Permalink
CID-2911: merge main
Browse files Browse the repository at this point in the history
  • Loading branch information
mohamedlajmileanix committed Jan 14, 2025
2 parents 7f56f31 + b793587 commit 5ca9182
Show file tree
Hide file tree
Showing 13 changed files with 334 additions and 44 deletions.
18 changes: 16 additions & 2 deletions src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,17 @@ interface GitHubClient {
): GitHubAppResponse

@GetMapping("/api/v3/app/installations")
fun getInstallations(@RequestHeader("Authorization") jwt: String): List<Installation>
fun getInstallations(
@RequestHeader("Authorization") jwt: String,
@RequestParam("per_page", defaultValue = "30") perPage: Int,
@RequestParam("page", defaultValue = "1") page: Int
): List<Installation>

@GetMapping("/api/v3/app/installations/{installationId}")
fun getInstallation(
@PathVariable("installationId") installationId: Long,
@RequestHeader("Authorization") jwt: String
): Installation

@PostMapping("/api/v3/app/installations/{installationId}/access_tokens")
fun createInstallationToken(
Expand All @@ -37,7 +47,11 @@ interface GitHubClient {
): InstallationTokenResponse

@GetMapping("/api/v3/organizations")
fun getOrganizations(@RequestHeader("Authorization") token: String): List<Organization>
fun getOrganizations(
@RequestHeader("Authorization") jwt: String,
@RequestParam("per_page", defaultValue = "30") perPage: Int,
@RequestParam("since", defaultValue = "1") since: Int
): List<Organization>

@GetMapping("/api/v3/orgs/{org}/repos")
fun getRepositories(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ data class InstallationTokenResponse(
@JsonIgnoreProperties(ignoreUnknown = true)
data class Installation(
@JsonProperty("id") val id: Long,
@JsonProperty("account") val account: Account
@JsonProperty("account") val account: Account,
@JsonProperty("permissions") val permissions: Map<String, String>,
@JsonProperty("events") val events: List<String>
)

@JsonIgnoreProperties(ignoreUnknown = true)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package net.leanix.githubagent.services

import net.leanix.githubagent.client.GitHubClient
import net.leanix.githubagent.dto.Installation
import net.leanix.githubagent.dto.Organization
import org.springframework.stereotype.Service

@Service
class GitHubAPIService(
private val gitHubClient: GitHubClient,
) {

companion object {
private const val PAGE_SIZE = 30 // Maximum allowed by GitHub API is 100
}

fun getPaginatedInstallations(jwtToken: String): List<Installation> {
val installations = mutableListOf<Installation>()
var page = 1
var currentInstallations: List<Installation>

do {
currentInstallations = gitHubClient.getInstallations("Bearer $jwtToken", PAGE_SIZE, page)
if (currentInstallations.isNotEmpty()) installations.addAll(currentInstallations) else break
page++
} while (currentInstallations.size == PAGE_SIZE)
return installations
}

fun getPaginatedOrganizations(installationToken: String): List<Organization> {
val organizations = mutableListOf<Organization>()
var since = 1
var currentOrganizations: List<Organization>

do {
currentOrganizations = gitHubClient.getOrganizations("Bearer $installationToken", PAGE_SIZE, since)
if (currentOrganizations.isNotEmpty()) organizations.addAll(currentOrganizations) else break
since = currentOrganizations.last().id
} while (currentOrganizations.size == PAGE_SIZE)
return organizations
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import net.leanix.githubagent.client.GitHubClient
import net.leanix.githubagent.config.GitHubEnterpriseProperties
import net.leanix.githubagent.dto.Installation
import net.leanix.githubagent.exceptions.FailedToCreateJWTException
import net.leanix.githubagent.exceptions.JwtTokenNotFound
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.slf4j.LoggerFactory
import org.springframework.core.io.ResourceLoader
Expand All @@ -26,7 +27,8 @@ class GitHubAuthenticationService(
private val githubEnterpriseProperties: GitHubEnterpriseProperties,
private val resourceLoader: ResourceLoader,
private val gitHubEnterpriseService: GitHubEnterpriseService,
private val gitHubClient: GitHubClient
private val gitHubClient: GitHubClient,
private val gitHubAPIService: GitHubAPIService,
) {

companion object {
Expand All @@ -38,11 +40,9 @@ class GitHubAuthenticationService(

fun refreshTokens() {
generateAndCacheJwtToken()
val jwtToken = cachingService.get("jwtToken")
generateAndCacheInstallationTokens(
gitHubClient.getInstallations("Bearer $jwtToken"),
jwtToken.toString()
)
val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound()
val installations = gitHubAPIService.getPaginatedInstallations(jwtToken.toString())
generateAndCacheInstallationTokens(installations, jwtToken.toString())
}

fun generateAndCacheJwtToken() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
package net.leanix.githubagent.services

import net.leanix.githubagent.client.GitHubClient
import net.leanix.githubagent.dto.GitHubAppResponse
import net.leanix.githubagent.exceptions.GitHubAppInsufficientPermissionsException
import net.leanix.githubagent.exceptions.UnableToConnectToGitHubEnterpriseException
import net.leanix.githubagent.shared.GITHUB_APP_LABEL
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service

@Service
class GitHubEnterpriseService(private val githubClient: GitHubClient) {
class GitHubEnterpriseService(
private val githubClient: GitHubClient,
private val syncLogService: SyncLogService,
) {

companion object {
val expectedPermissions = listOf("administration", "contents", "metadata")
Expand All @@ -19,29 +22,34 @@ class GitHubEnterpriseService(private val githubClient: GitHubClient) {
fun verifyJwt(jwt: String) {
runCatching {
val githubApp = getGitHubApp(jwt)
validateGithubAppResponse(githubApp)
validateEnabledPermissionsAndEvents(GITHUB_APP_LABEL, githubApp.permissions, githubApp.events)
logger.info("Authenticated as GitHub App: '${githubApp.slug}'")
}.onFailure {
logger.error("Failed to verify JWT token", it)
when (it) {
is GitHubAppInsufficientPermissionsException -> throw it
else -> throw UnableToConnectToGitHubEnterpriseException("Failed to verify JWT token")
is GitHubAppInsufficientPermissionsException -> {
logger.error(it.message)
syncLogService.sendErrorLog(it.message!!)
}
else -> {
logger.error("Failed to verify JWT token", it)
throw UnableToConnectToGitHubEnterpriseException("Failed to verify JWT token")
}
}
}
}

fun validateGithubAppResponse(response: GitHubAppResponse) {
val missingPermissions = expectedPermissions.filterNot { response.permissions.containsKey(it) }
val missingEvents = expectedEvents.filterNot { response.events.contains(it) }
fun validateEnabledPermissionsAndEvents(type: String, permissions: Map<String, String>, events: List<String>) {
val missingPermissions = expectedPermissions.filterNot { permissions.containsKey(it) }
val missingEvents = expectedEvents.filterNot { events.contains(it) }

if (missingPermissions.isNotEmpty() || missingEvents.isNotEmpty()) {
var message = "GitHub App is missing the following "
var message = "$type missing the following "
if (missingPermissions.isNotEmpty()) {
message = message.plus("permissions: $missingPermissions")
}
if (missingEvents.isNotEmpty()) {
if (missingPermissions.isNotEmpty()) {
message = message.plus(", and the following")
message = message.plus(", and the following ")
}
message = message.plus("events: $missingEvents")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ import net.leanix.githubagent.dto.Organization
import net.leanix.githubagent.dto.OrganizationDto
import net.leanix.githubagent.dto.RateLimitType
import net.leanix.githubagent.dto.RepositoryDto
import net.leanix.githubagent.exceptions.GitHubAppInsufficientPermissionsException
import net.leanix.githubagent.exceptions.JwtTokenNotFound
import net.leanix.githubagent.exceptions.ManifestFileNotFoundException
import net.leanix.githubagent.handler.RateLimitHandler
import net.leanix.githubagent.shared.INSTALLATION_LABEL
import net.leanix.githubagent.shared.MANIFEST_FILE_NAME
import net.leanix.githubagent.shared.fileNameMatchRegex
import net.leanix.githubagent.shared.generateFullPath
Expand All @@ -27,6 +29,8 @@ class GitHubScanningService(
private val gitHubAuthenticationService: GitHubAuthenticationService,
private val syncLogService: SyncLogService,
private val rateLimitHandler: RateLimitHandler,
private val gitHubEnterpriseService: GitHubEnterpriseService,
private val gitHubAPIService: GitHubAPIService,
) {

private val logger = LoggerFactory.getLogger(GitHubScanningService::class.java)
Expand All @@ -36,17 +40,36 @@ class GitHubScanningService(
val installations = getInstallations(jwtToken.toString())
fetchAndSendOrganisationsData(installations)
installations.forEach { installation ->
fetchAndSendRepositoriesData(installation)
.forEach { repository ->
fetchManifestFilesAndSend(installation, repository)
kotlin.runCatching {
gitHubEnterpriseService.validateEnabledPermissionsAndEvents(
INSTALLATION_LABEL,
installation.permissions,
installation.events
)
fetchAndSendRepositoriesData(installation)
.forEach { repository ->
fetchManifestFilesAndSend(installation, repository)
}
syncLogService.sendInfoLog("Finished initial full scan for organization ${installation.account.login}.")
}.onFailure {
val message = "Failed to scan organization ${installation.account.login}."
when (it) {
is GitHubAppInsufficientPermissionsException -> {
syncLogService.sendErrorLog("$message ${it.message}")
logger.error("$message ${it.message}")
}
else -> {
syncLogService.sendErrorLog(message)
logger.error(message, it)
}
}
syncLogService.sendInfoLog("Finished initial full scan for organization ${installation.account.login}.")
}
}
syncLogService.sendInfoLog("Finished full scan for all available organizations.")
}

private fun getInstallations(jwtToken: String): List<Installation> {
val installations = gitHubClient.getInstallations("Bearer $jwtToken")
val installations = gitHubAPIService.getPaginatedInstallations(jwtToken)
gitHubAuthenticationService.generateAndCacheInstallationTokens(installations, jwtToken)
return installations
}
Expand All @@ -61,7 +84,7 @@ class GitHubScanningService(
}
val installationToken = cachingService.get("installationToken:${installations.first().id}")
val organizations = rateLimitHandler.executeWithRateLimitHandler(RateLimitType.REST) {
gitHubClient.getOrganizations("Bearer $installationToken")
gitHubAPIService.getPaginatedOrganizations(installationToken.toString())
.map { organization ->
if (installations.find { it.account.login == organization.login } != null) {
OrganizationDto(organization.id, organization.login, true)
Expand Down Expand Up @@ -105,6 +128,7 @@ class GitHubScanningService(
}

fun fetchManifestFilesAndSend(installation: Installation, repository: RepositoryDto) {
if (repository.archived) return
val manifestFiles = fetchManifestFiles(installation, repository.name).getOrThrow().items
val manifestFilesContents = fetchManifestContents(
installation,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ package net.leanix.githubagent.services

import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import net.leanix.githubagent.dto.Account
import net.leanix.githubagent.dto.Installation
import net.leanix.githubagent.client.GitHubClient
import net.leanix.githubagent.dto.InstallationEventPayload
import net.leanix.githubagent.dto.ManifestFileAction
import net.leanix.githubagent.dto.ManifestFileUpdateDto
import net.leanix.githubagent.dto.PushEventCommit
import net.leanix.githubagent.dto.PushEventPayload
import net.leanix.githubagent.exceptions.JwtTokenNotFound
import net.leanix.githubagent.shared.INSTALLATION_LABEL
import net.leanix.githubagent.shared.MANIFEST_FILE_NAME
import net.leanix.githubagent.shared.fileNameMatchRegex
import net.leanix.githubagent.shared.generateFullPath
Expand All @@ -24,7 +25,9 @@ class WebhookEventService(
private val gitHubAuthenticationService: GitHubAuthenticationService,
private val gitHubScanningService: GitHubScanningService,
private val syncLogService: SyncLogService,
@Value("\${webhookEventService.waitingTime}") private val waitingTime: Long
@Value("\${webhookEventService.waitingTime}") private val waitingTime: Long,
private val gitHubClient: GitHubClient,
private val gitHubEnterpriseService: GitHubEnterpriseService
) {

private val logger = LoggerFactory.getLogger(WebhookEventService::class.java)
Expand Down Expand Up @@ -77,9 +80,15 @@ class WebhookEventService(
}
syncLogService.sendFullScanStart(installationEventPayload.installation.account.login)
kotlin.runCatching {
val installation = Installation(
val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound()
val installation = gitHubClient.getInstallation(
installationEventPayload.installation.id.toLong(),
Account(installationEventPayload.installation.account.login)
"Bearer $jwtToken"
)
gitHubEnterpriseService.validateEnabledPermissionsAndEvents(
INSTALLATION_LABEL,
installation.permissions,
installation.events
)
gitHubAuthenticationService.refreshTokens()
gitHubScanningService.fetchAndSendOrganisationsData(listOf(installation))
Expand Down
3 changes: 3 additions & 0 deletions src/main/kotlin/net/leanix/githubagent/shared/Constants.kt
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,6 @@ val SUPPORTED_EVENT_TYPES = listOf(
)

val fileNameMatchRegex = Regex("/?$MANIFEST_FILE_NAME\$")

const val GITHUB_APP_LABEL = "GitHub App"
const val INSTALLATION_LABEL = "Installation"
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package net.leanix.githubagent.services

import io.mockk.every
import io.mockk.mockk
import net.leanix.githubagent.client.GitHubClient
import net.leanix.githubagent.dto.Account
import net.leanix.githubagent.dto.Installation
import net.leanix.githubagent.dto.Organization
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test

class GitHubAPIServiceTest {

private val gitHubClient = mockk<GitHubClient>()
private val gitHubAPIService = GitHubAPIService(gitHubClient)

private val permissions = mapOf("administration" to "read", "contents" to "read", "metadata" to "read")
private val events = listOf("label", "public", "repository", "push")

@Test
fun `test getPaginatedInstallations with one page`() {
val jwtToken = "test-jwt-token"
val installationsPage1 = listOf(
Installation(1, Account("test-account"), permissions, events),
Installation(2, Account("test-account"), permissions, events)
)

every { gitHubClient.getInstallations(any(), any(), any()) } returns installationsPage1

val installations = gitHubAPIService.getPaginatedInstallations(jwtToken)
assertEquals(2, installations.size)
assertEquals(installationsPage1, installations)
}

@Test
fun `test getPaginatedInstallations with multiple pages`() {
val jwtToken = "test-jwt-token"
val perPage = 30
val totalInstallations = 100
val installations = (1..totalInstallations).map {
Installation(it.toLong(), Account("test-account-$it"), permissions, events)
}
val pages = installations.chunked(perPage)

every { gitHubClient.getInstallations(any(), any(), any()) } returnsMany pages + listOf(emptyList())

val result = gitHubAPIService.getPaginatedInstallations(jwtToken)
assertEquals(totalInstallations, result.size)
assertEquals(installations, result)
}

@Test
fun `test getPaginatedOrganizations with one page`() {
val installationToken = "test-installation-token"
val organizationsPage1 = listOf(
Organization("org-1", 1),
Organization("org-2", 2)
)

every { gitHubClient.getOrganizations(any(), any(), any()) } returns organizationsPage1

val organizations = gitHubAPIService.getPaginatedOrganizations(installationToken)
assertEquals(2, organizations.size)
assertEquals(organizationsPage1, organizations)
}

@Test
fun `test getPaginatedOrganizations with multiple pages`() {
val installationToken = "test-installation-token"
val perPage = 30
val totalOrganizations = 100
val organizations = (1..totalOrganizations).map { Organization("org-$it", it) }
val pages = organizations.chunked(perPage)

every { gitHubClient.getOrganizations(any(), any(), any()) } returnsMany pages + listOf(emptyList())

val result = gitHubAPIService.getPaginatedOrganizations(installationToken)
assertEquals(totalOrganizations, result.size)
assertEquals(organizations, result)
}
}
Loading

0 comments on commit 5ca9182

Please sign in to comment.