Feature/cid 2745/get list of repositories and manifest files and send data to backend

README.md

@@ -27,6 +27,7 @@ SAP LeanIX agent to discover self built software in self-hosted GitHub Enterpris
     - `GITHUB_ENTERPRISE_BASE_URL`: The base URL of your GitHub Enterprise Server instance.
     - `GITHUB_APP_ID`: The ID of your GitHub App.
     - `PEM_FILE`: The path to your GitHub App's PEM file inside the Docker container.
+    - `MANIFEST_FILE_DIRECTORY`: The directory path where the manifest files are located in every repository. For more details on the manifest file: [Microservice Discovery Through a Manifest File](https://docs-eam.leanix.net/reference/microservice-discovery-manifest-file)
 
 5. **Start the Agent**: Run the Docker command to start the agent. Replace `<variable>` with your actual values:
 
@@ -36,6 +37,7 @@ SAP LeanIX agent to discover self built software in self-hosted GitHub Enterpris
     -e GITHUB_ENTERPRISE_BASE_URL=<github_enterprise_base_url> \
     -e GITHUB_APP_ID=<github_app_id> \
     -e PEM_FILE=/privateKey.pem \
+    -e MANIFEST_FILE_DIRECTORY=<manifest_file_directory> \
     leanix-github-agent
     ```
 

build.gradle.kts

@@ -1,6 +1,9 @@
+import com.expediagroup.graphql.plugin.gradle.tasks.GraphQLGenerateClientTask
+
 plugins {
-    id("org.springframework.boot") version "3.3.1"
+    id("org.springframework.boot") version "3.2.5"
     id("io.spring.dependency-management") version "1.1.5"
+    id("com.expediagroup.graphql") version "7.0.2"
     id("io.gitlab.arturbosch.detekt") version "1.23.4"
     kotlin("jvm") version "1.9.21"
     kotlin("plugin.spring") version "1.9.21"
@@ -37,6 +40,7 @@ dependencies {
     implementation("io.github.resilience4j:resilience4j-spring-boot3:2.1.0")
     implementation("org.jetbrains.kotlin:kotlin-reflect")
     implementation("com.fasterxml.jackson.core:jackson-annotations:2.17.1")
+    implementation("com.expediagroup:graphql-kotlin-spring-client:7.0.2")
 
     // Dependencies for generating JWT token
     implementation("io.jsonwebtoken:jjwt-impl:0.11.2")
@@ -48,6 +52,12 @@ dependencies {
     }
 }
 
+val generateGitHubGraphQLClient by tasks.creating(GraphQLGenerateClientTask::class)  {
+    packageName.set("net.leanix.githubbroker.connector.adapter.graphql.data")
+    schemaFile = file("${project.projectDir}/src/main/resources/schemas/schema.docs-enterprise.graphql")
+    queryFileDirectory = file("${project.projectDir}/src/main/resources/graphql")
+}
+
 configurations.all {
     resolutionStrategy {
         eachDependency {

src/main/kotlin/net/leanix/githubagent/config/GitHubEnterpriseProperties.kt

@@ -7,4 +7,5 @@ data class GitHubEnterpriseProperties(
     val baseUrl: String,
     val gitHubAppId: String,
     val pemFile: String,
+    val manifestFilePath: String,
 )

src/main/kotlin/net/leanix/githubagent/dto/GitHubResponsesDto.kt

@@ -26,6 +26,7 @@ data class Account(
 data class Organization(
     @JsonProperty("login") val login: String,
     @JsonProperty("id") val id: Int,
+    @JsonProperty("node_id") val nodeId: String,
 )
 
 @JsonIgnoreProperties(ignoreUnknown = true)

src/main/kotlin/net/leanix/githubagent/dto/OrganizationDto.kt

@@ -2,6 +2,7 @@ package net.leanix.githubagent.dto
 
 class OrganizationDto(
     val id: Int,
+    val nodeId: String,
     val name: String,
     val installed: Boolean
 )

src/main/kotlin/net/leanix/githubagent/dto/PagedRepositories.kt

@@ -0,0 +1,7 @@
+package net.leanix.githubagent.dto
+
+data class PagedRepositories(
+    val repositories: List<RepositoryDto>,
+    val hasNextPage: Boolean,
+    val cursor: String? = null
+)

src/main/kotlin/net/leanix/githubagent/dto/RepositoryDto.kt

@@ -0,0 +1,17 @@
+package net.leanix.githubagent.dto
+
+data class RepositoryDto(
+    val id: String,
+    val name: String,
+    val description: String?,
+    val url: String,
+    val organization: RepositoryOrganizationDto,
+    val languages: List<String>,
+    val topics: List<String>,
+    val manifest: String?,
+)
+
+class RepositoryOrganizationDto(
+    val id: String,
+    val name: String
+)

src/main/kotlin/net/leanix/githubagent/exceptions/Exceptions.kt

@@ -1,9 +1,13 @@
 package net.leanix.githubagent.exceptions
 
+import com.expediagroup.graphql.client.types.GraphQLClientError
+
 class GitHubEnterpriseConfigurationMissingException(properties: String) : RuntimeException(
     "Github Enterprise properties '$properties' are not set"
 )
 class GitHubAppInsufficientPermissionsException(message: String) : RuntimeException(message)
 class FailedToCreateJWTException(message: String) : RuntimeException(message)
 class UnableToConnectToGitHubEnterpriseException(message: String) : RuntimeException(message)
 class JwtTokenNotFound : RuntimeException("JWT token not found")
+class GraphQLApiException(errors: List<GraphQLClientError>) :
+    RuntimeException("Errors: ${errors.joinToString(separator = "\n") { it.message }}")

src/main/kotlin/net/leanix/githubagent/runners/PostStartupRunner.kt

@@ -17,8 +17,8 @@ class PostStartupRunner(
 ) : ApplicationRunner {
 
     override fun run(args: ApplicationArguments?) {
-        githubAuthenticationService.generateJwtToken()
         webSocketService.initSession()
+        githubAuthenticationService.generateJwtToken()
         gitHubScanningService.scanGitHubResources()
     }
 }

src/main/kotlin/net/leanix/githubagent/services/GitHubGraphQLService.kt

@@ -0,0 +1,78 @@
+package net.leanix.githubagent.services
+
+import com.expediagroup.graphql.client.spring.GraphQLWebClient
+import kotlinx.coroutines.runBlocking
+import net.leanix.githubagent.config.GitHubEnterpriseProperties
+import net.leanix.githubagent.dto.PagedRepositories
+import net.leanix.githubagent.dto.RepositoryDto
+import net.leanix.githubagent.dto.RepositoryOrganizationDto
+import net.leanix.githubagent.exceptions.GraphQLApiException
+import net.leanix.githubbroker.connector.adapter.graphql.data.GetRepositories
+import net.leanix.githubbroker.connector.adapter.graphql.data.getrepositories.Blob
+import org.slf4j.LoggerFactory
+import org.springframework.stereotype.Component
+import org.springframework.web.reactive.function.client.WebClient
+
+@Component
+class GitHubGraphQLService(
+    private val cachingService: CachingService,
+    private val gitHubEnterpriseProperties: GitHubEnterpriseProperties
+) {
+    companion object {
+        private val logger = LoggerFactory.getLogger(GitHubGraphQLService::class.java)
+        private const val PAGE_COUNT = 100
+        private const val MANIFEST_FILE_NAME = "leanix.yaml"
+    }
+
+    fun getRepositories(
+        token: String,
+        cursor: String? = null
+    ): PagedRepositories {
+        val client = buildGitHubGraphQLClient(token)
+
+        val query = GetRepositories(
+            GetRepositories.Variables(
+                pageCount = PAGE_COUNT,
+                cursor = cursor,
+                expression = "HEAD:${gitHubEnterpriseProperties.manifestFilePath}$MANIFEST_FILE_NAME"
+            )
+        )
+
+        val result = runBlocking {
+            client.execute(query)
+        }
+
+        return if (result.errors != null && result.errors!!.isNotEmpty()) {
+            logger.error("Error getting repositories: ${result.errors}")
+            throw GraphQLApiException(result.errors!!)
+        } else {
+            PagedRepositories(
+                hasNextPage = result.data!!.viewer.repositories.pageInfo.hasNextPage,
+                cursor = result.data!!.viewer.repositories.pageInfo.endCursor,
+                repositories = result.data!!.viewer.repositories.nodes!!.map {
+                    RepositoryDto(
+                        id = it!!.id,
+                        name = it.name,
+                        description = it.description,
+                        url = it.url,
+                        organization = RepositoryOrganizationDto(
+                            id = it.owner.id,
+                            name = it.owner.login
+                        ),
+                        languages = it.languages!!.nodes!!.map { language -> language!!.name },
+                        topics = it.repositoryTopics.nodes!!.map { topic -> topic!!.topic.name },
+                        manifest = (it.`object` as Blob?)?.text
+                    )
+                }
+            )
+        }
+    }
+
+    private fun buildGitHubGraphQLClient(
+        token: String
+    ) =
+        GraphQLWebClient(
+            url = "${cachingService.get("baseUrl")}/api/graphql",
+            builder = WebClient.builder().defaultHeaders { it.setBearerAuth(token) }
+        )
+}

src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt

@@ -11,16 +11,20 @@ import org.springframework.stereotype.Service
 class GitHubScanningService(
     private val gitHubClient: GitHubClient,
     private val cachingService: CachingService,
-    private val webSocketService: WebSocketService
+    private val webSocketService: WebSocketService,
+    private val gitHubGraphQLService: GitHubGraphQLService
 ) {
     private val logger = LoggerFactory.getLogger(GitHubScanningService::class.java)
 
     fun scanGitHubResources() {
         runCatching {
             val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound()
             val installations = getInstallations(jwtToken.toString())
-            val organizations = generateOrganizations(installations)
-            webSocketService.sendMessage("/app/ghe/organizations", organizations)
+            fetchAndBroadcastOrganisationsData(installations)
+            installations.forEach { installation ->
+                logger.info("Fetching repositories for organisation ${installation.account.login}")
+                fetchAndBroadcastRepositoriesData(installation)
+            }
         }.onFailure {
             logger.error("Error while scanning GitHub resources")
             throw it
@@ -43,17 +47,37 @@ class GitHubScanningService(
         }
     }
 
-    private fun generateOrganizations(
+    private fun fetchAndBroadcastOrganisationsData(
         installations: List<Installation>
-    ): List<OrganizationDto> {
+    ) {
         val installationToken = cachingService.get("installationToken:${installations.first().id}")
+        println(installationToken)
         val organizations = gitHubClient.getOrganizations("Bearer $installationToken")
-        return organizations.map { organization ->
-            if (installations.find { it.account.login == organization.login } != null) {
-                OrganizationDto(organization.id, organization.login, true)
-            } else {
-                OrganizationDto(organization.id, organization.login, false)
+            .map { organization ->
+                if (installations.find { it.account.login == organization.login } != null) {
+                    OrganizationDto(organization.id, organization.nodeId, organization.login, true)
+                } else {
+                    OrganizationDto(organization.id, organization.nodeId, organization.login, false)
+                }
             }
-        }
+        webSocketService.sendMessage("/app/ghe/organizations", organizations)
+    }
+
+    private fun fetchAndBroadcastRepositoriesData(installation: Installation) {
+        val installationToken = cachingService.get("installationToken:${installation.id}").toString()
+        var cursor: String? = null
+        var totalRepos = 0
+        var page = 1
+        do {
+            val repositoriesPage = gitHubGraphQLService.getRepositories(
+                token = installationToken,
+                cursor = cursor
+            )
+            webSocketService.sendMessage("/app/ghe/repositories", repositoriesPage.repositories)
+            cursor = repositoriesPage.cursor
+            totalRepos += repositoriesPage.repositories.size
+            page++
+        } while (repositoriesPage.hasNextPage)
+        logger.info("Fetched $totalRepos repositories")
     }
 }

src/main/resources/application.yaml

@@ -2,6 +2,7 @@ github-enterprise:
   baseUrl: ${GITHUB_ENTERPRISE_BASE_URL:}
   githubAppId: ${GITHUB_APP_ID:}
   pemFile: ${PEM_FILE:}
+  manifestFilePath: ${MANIFEST_FILE_DIRECTORY:}
 leanix:
   base-url: https://${LEANIX_DOMAIN}/services
   ws-base-url: wss://${LEANIX_DOMAIN}/services

src/main/resources/graphql/GetRepositories.graphql

@@ -0,0 +1,38 @@
+query GetRepositories($pageCount: Int!, $cursor: String, $expression: String!) {
+    viewer {
+        repositories(first: $pageCount, after: $cursor) {
+            pageInfo {
+                endCursor
+                hasNextPage
+            }
+            nodes {
+                id
+                name
+                description
+                url
+                owner{
+                    id
+                  login
+                }
+                languages(first: 10) {
+                    nodes {
+                        name
+                    }
+                }
+                repositoryTopics(first:10) {
+                    nodes {
+                        topic {
+                            name
+                        }
+                    }
+                }
+                object(expression: $expression) {
+                    __typename
+                    ... on Blob {
+                        text
+                    }
+                }
+            }
+        }
+    }
+}