Skip to content

Commit

Permalink
Merge pull request #18 from leanix/feature/CID-2874/support-yaml-and-…
Browse files Browse the repository at this point in the history
…yml-manifest-files

CID-2874: Support .yaml and .yml extensions for manifest file
  • Loading branch information
mohamedlajmileanix authored Aug 14, 2024
2 parents 5b33c79 + 2d4d205 commit 99f2a8a
Show file tree
Hide file tree
Showing 10 changed files with 204 additions and 49 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ The SAP LeanIX agent discovers self-built software in self-hosted GitHub Enterpr
- `GITHUB_ENTERPRISE_BASE_URL`: The base URL of your GitHub Enterprise Server instance.
- `GITHUB_APP_ID`: The ID of your GitHub App.
- `PEM_FILE`: The path to your GitHub App's PEM file inside the Docker container.
- `MANIFEST_FILE_DIRECTORY`: The directory path where the manifest files are stored in each repository. Manifest files are crucial for microservice discovery as they provide essential information about the service. For more information, see [Microservice Discovery Through a Manifest File](https://docs-eam.leanix.net/reference/microservice-discovery-manifest-file) in our documentation.
- `MANIFEST_FILE_DIRECTORY`: The directory path where the manifest files are stored in each repository. Manifest files are crucial for microservice discovery as they provide essential information about the service. For more information, see [Microservice Discovery Through a Manifest File](https://docs-eam.leanix.net/reference/microservice-discovery-manifest-file) in our documentation. **If both .yaml and .yml files exist in a repository, only the .yaml file will be used.**
- `WEBHOOK_SECRET`: The secret used to validate incoming webhook events from GitHub. (Optional, but recommended. [Needs to be set in the GitHub App settings first](https://docs.github.com/en/enterprise-server@3.8/webhooks/using-webhooks/validating-webhook-deliveries).)

5. **Start the agent**: To start the agent, run the following Docker command. Replace the variables in angle brackets with your actual values.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package net.leanix.githubagent.dto
data class ManifestFileUpdateDto(
val repositoryFullName: String,
val action: ManifestFileAction,
val manifestContent: String?
val manifestFileContent: String?
)

enum class ManifestFileAction {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import net.leanix.githubagent.exceptions.GraphQLApiException
import net.leanix.githubagent.graphql.data.GetRepositories
import net.leanix.githubagent.graphql.data.GetRepositoryManifestContent
import net.leanix.githubagent.graphql.data.getrepositories.Blob
import net.leanix.githubagent.shared.MANIFEST_FILE_NAME
import net.leanix.githubagent.shared.ManifestFileName
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Component
import org.springframework.web.reactive.function.client.WebClient
Expand All @@ -34,7 +34,8 @@ class GitHubGraphQLService(
GetRepositories.Variables(
pageCount = PAGE_COUNT,
cursor = cursor,
expression = "HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}$MANIFEST_FILE_NAME"
"HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YAML.fileName}",
"HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YML.fileName}"
)
)

Expand All @@ -61,7 +62,13 @@ class GitHubGraphQLService(
updatedAt = it.updatedAt,
languages = it.languages!!.nodes!!.map { language -> language!!.name },
topics = it.repositoryTopics.nodes!!.map { topic -> topic!!.topic.name },
manifestFileContent = (it.`object` as Blob?)?.text
manifestFileContent = if (it.manifestYaml != null) {
(it.manifestYaml as Blob).text.toString()
} else if (it.manifestYml != null) {
(it.manifestYml as Blob).text.toString()
} else {
null
}
)
}
)
Expand All @@ -73,14 +80,14 @@ class GitHubGraphQLService(
repositoryName: String,
filePath: String,
token: String
): String {
): String? {
val client = buildGitHubGraphQLClient(token)

val query = GetRepositoryManifestContent(
GetRepositoryManifestContent.Variables(
owner = owner,
repositoryName = repositoryName,
filePath = filePath
manifestFilePath = "HEAD:$filePath"
)
)

Expand All @@ -93,7 +100,7 @@ class GitHubGraphQLService(
throw GraphQLApiException(result.errors!!)
} else {
(
result.data!!.repository!!.`object`
result.data!!.repository!!.manifestFile
as net.leanix.githubagent.graphql.`data`.getrepositorymanifestcontent.Blob
).text.toString()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ class GitHubScanningService(
val installations = getInstallations(jwtToken.toString())
fetchAndSendOrganisationsData(installations)
installations.forEach { installation ->
logger.info("Fetching repositories for organisation ${installation.account.login}")
fetchAndSendRepositoriesData(installation)
}
}.onFailure {
Expand Down Expand Up @@ -74,7 +73,6 @@ class GitHubScanningService(
token = installationToken,
cursor = cursor
)
logger.info("Sending page $page of repositories")
webSocketService.sendMessage(
"${cachingService.get("runId")}/repositories",
repositoriesPage.repositories
Expand All @@ -83,6 +81,6 @@ class GitHubScanningService(
totalRepos += repositoriesPage.repositories.size
page++
} while (repositoriesPage.hasNextPage)
logger.info("Fetched $totalRepos repositories")
logger.info("Fetched $totalRepos repositories data from organisation ${installation.account.login}")
}
}
140 changes: 111 additions & 29 deletions src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ import com.fasterxml.jackson.module.kotlin.readValue
import net.leanix.githubagent.config.GitHubEnterpriseProperties
import net.leanix.githubagent.dto.ManifestFileAction
import net.leanix.githubagent.dto.ManifestFileUpdateDto
import net.leanix.githubagent.dto.PushEventCommit
import net.leanix.githubagent.dto.PushEventPayload
import net.leanix.githubagent.shared.MANIFEST_FILE_NAME
import net.leanix.githubagent.shared.ManifestFileName
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service

Expand Down Expand Up @@ -37,49 +38,130 @@ class WebhookEventService(
val repositoryName = pushEventPayload.repository.name
val repositoryFullName = pushEventPayload.repository.fullName
val headCommit = pushEventPayload.headCommit
val organizationName = pushEventPayload.repository.owner.name
val owner = pushEventPayload.repository.owner.name

var installationToken = cachingService.get("installationToken:${pushEventPayload.installation.id}")?.toString()
if (installationToken == null) {
gitHubAuthenticationService.refreshTokens()
installationToken = cachingService.get("installationToken:${pushEventPayload.installation.id}")?.toString()
require(installationToken != null) { "Installation token not found/ expired" }
}
val installationToken = getInstallationToken(pushEventPayload.installation.id)

if (pushEventPayload.ref == "refs/heads/${pushEventPayload.repository.defaultBranch}") {
when {
MANIFEST_FILE_NAME in headCommit.added -> {
logger.info("Manifest file added to repository $repositoryFullName")
val fileContent = getManifestFileContent(organizationName, repositoryName, installationToken)
sendManifestData(repositoryFullName, ManifestFileAction.ADDED, fileContent)
}
MANIFEST_FILE_NAME in headCommit.modified -> {
logger.info("Manifest file modified in repository $repositoryFullName")
val fileContent = getManifestFileContent(organizationName, repositoryName, installationToken)
sendManifestData(repositoryFullName, ManifestFileAction.MODIFIED, fileContent)
handleManifestFileChanges(
headCommit,
repositoryFullName,
owner,
repositoryName,
installationToken
)
}
}

@SuppressWarnings("LongParameterList")
private fun handleManifestFileChanges(
headCommit: PushEventCommit,
repositoryFullName: String,
owner: String,
repositoryName: String,
installationToken: String
) {
val yamlFileName = "${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YAML.fileName}"
val ymlFileName = "${gitHubEnterpriseProperties.manifestFileDirectory}${ManifestFileName.YML.fileName}"

val isYAMLFileUpdated = isManifestFileUpdated(headCommit, yamlFileName)
val isYMLFileUpdated = isManifestFileUpdated(headCommit, ymlFileName)

if (!isYAMLFileUpdated && isYMLFileUpdated) {
val yamlFileContent = gitHubGraphQLService.getManifestFileContent(
owner,
repositoryName,
yamlFileName,
installationToken
)
if (yamlFileContent != null) return
}

val manifestFilePath = determineManifestFilePath(isYAMLFileUpdated, isYMLFileUpdated, yamlFileName, ymlFileName)
manifestFilePath?.let {
when (it) {
in headCommit.added, in headCommit.modified -> {
handleAddedOrModifiedManifestFile(
headCommit,
repositoryFullName,
owner,
repositoryName,
installationToken,
it
)
}
MANIFEST_FILE_NAME in headCommit.removed -> {
logger.info("Manifest file removed from repository $repositoryFullName")
sendManifestData(repositoryFullName, ManifestFileAction.REMOVED, null)
in headCommit.removed -> {
handleRemovedManifestFile(repositoryFullName)
}
}
}
}

private fun getManifestFileContent(organizationName: String, repositoryName: String, token: String): String {
return gitHubGraphQLService.getManifestFileContent(
owner = organizationName,
private fun getInstallationToken(installationId: Int): String {
var installationToken = cachingService.get("installationToken:$installationId")?.toString()
if (installationToken == null) {
gitHubAuthenticationService.refreshTokens()
installationToken = cachingService.get("installationToken:$installationId")?.toString()
require(installationToken != null) { "Installation token not found/ expired" }
}
return installationToken
}

private fun isManifestFileUpdated(headCommit: PushEventCommit, fileName: String): Boolean {
return headCommit.added.any { it == fileName } ||
headCommit.modified.any { it == fileName } ||
headCommit.removed.any { it == fileName }
}

private fun determineManifestFilePath(
isYAMLFileUpdated: Boolean,
isYMLFileUpdated: Boolean,
yamlFileName: String,
ymlFileName: String
): String? {
return when {
isYAMLFileUpdated -> yamlFileName
isYMLFileUpdated -> ymlFileName
else -> null
}
}

@SuppressWarnings("LongParameterList")
private fun handleAddedOrModifiedManifestFile(
headCommit: PushEventCommit,
repositoryFullName: String,
owner: String,
repositoryName: String,
installationToken: String,
manifestFilePath: String
) {
val action = if (manifestFilePath in headCommit.added) ManifestFileAction.ADDED else ManifestFileAction.MODIFIED
logger.info("Manifest file $action in repository $repositoryFullName")
val fileContent = gitHubGraphQLService.getManifestFileContent(
owner,
repositoryName,
"HEAD:${gitHubEnterpriseProperties.manifestFileDirectory}$MANIFEST_FILE_NAME",
token
manifestFilePath,
installationToken
)
webSocketService.sendMessage(
"/events/manifestFile",
ManifestFileUpdateDto(
repositoryFullName,
action,
fileContent
)
)
}

private fun sendManifestData(repositoryFullName: String, action: ManifestFileAction, manifestContent: String?) {
logger.info("Sending manifest file update event for repository $repositoryFullName")
private fun handleRemovedManifestFile(repositoryFullName: String) {
logger.info("Manifest file ${ManifestFileAction.REMOVED} from repository $repositoryFullName")
webSocketService.sendMessage(
"/events/manifestFile",
ManifestFileUpdateDto(repositoryFullName, action, manifestContent)
ManifestFileUpdateDto(
repositoryFullName,
ManifestFileAction.REMOVED,
null
)
)
}
}
5 changes: 4 additions & 1 deletion src/main/kotlin/net/leanix/githubagent/shared/Constants.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ package net.leanix.githubagent.shared

const val TOPIC_PREFIX = "/app/ghe/"

const val MANIFEST_FILE_NAME = "leanix.yaml"
enum class ManifestFileName(val fileName: String) {
YAML("leanix.yaml"),
YML("leanix.yml")
}

val SUPPORTED_EVENT_TYPES = listOf(
"REPOSITORY",
Expand Down
10 changes: 8 additions & 2 deletions src/main/resources/graphql/GetRepositories.graphql
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
query GetRepositories($pageCount: Int!, $cursor: String, $expression: String!) {
query GetRepositories($pageCount: Int!, $cursor: String, $manifestYamlPath: String!, $manifestYmlPath: String!) {
viewer {
repositories(first: $pageCount, after: $cursor) {
pageInfo {
Expand Down Expand Up @@ -28,7 +28,13 @@ query GetRepositories($pageCount: Int!, $cursor: String, $expression: String!) {
}
}
}
object(expression: $expression) {
manifestYaml: object(expression: $manifestYamlPath) {
__typename
... on Blob {
text
}
}
manifestYml: object(expression: $manifestYmlPath) {
__typename
... on Blob {
text
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
query GetRepositoryManifestContent($owner: String!, $repositoryName: String!, $filePath: String!) {
query GetRepositoryManifestContent($owner: String!, $repositoryName: String!, $manifestFilePath: String!) {
repository(owner: $owner, name: $repositoryName) {
object(expression: $filePath) {
manifestFile: object(expression: $manifestFilePath) {
__typename
... on Blob {
text
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ class GitHubScanningServiceTest {
updatedAt = "2024-01-01T00:00:00Z",
languages = listOf("Kotlin", "Java"),
topics = listOf("test", "example"),
manifestFileContent = "dependencies { implementation 'com.example:example-lib:1.0.0' }"
manifestFileContent = "content"
)
),
hasNextPage = false,
Expand Down
Loading

0 comments on commit 99f2a8a

Please sign in to comment.