From f98c5dbf0cdd3f9a227bb80be10d7bb4dd7029a5 Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Fri, 21 Apr 2023 12:12:35 +0200 Subject: [PATCH 01/13] #100: Github CI Indexer * adding github.js provider and GitHubCIIndexer.js * adding functionality to github.js to request all current pipelines. --- lib/core/provider/github.js | 71 +++++++++++++++++ lib/indexers/ci/CIIndexer.js | 8 +- lib/indexers/ci/GitHubCIIndexer.js | 120 +++++++++++++++++++++++++++++ lib/indexers/ci/index.js | 1 + lib/paginator.js | 11 ++- lib/url-providers/index.js | 1 + 6 files changed, 202 insertions(+), 10 deletions(-) create mode 100644 lib/core/provider/github.js create mode 100644 lib/indexers/ci/GitHubCIIndexer.js diff --git a/lib/core/provider/github.js b/lib/core/provider/github.js new file mode 100644 index 00000000..cd42e2ab --- /dev/null +++ b/lib/core/provider/github.js @@ -0,0 +1,71 @@ +'use strict'; + +const _ = require('lodash'); +const urlJoin = require('url-join'); +const log = require('debug')('github'); +const Paginator = require('../../paginator.js'); +const { response } = require('express'); +const { Octokit } = require('@octokit/rest'); + +class GitHub { + constructor(options) { + this.baseUrl = options.baseUrl; + this.privateToken = options.privateToken; + this.requestTimeout = options.requestTimeout; + this.count = 0; + this.stopping = false; + this.github = new Octokit({ + baseUrl: 'https://api.github.com', + auth: this.privateToken, + }); + } + + getPipelines(projectId) { + log('getPipelines(%o)', projectId); + return this.paginatedRequest( + this.github.rest.actions.listRepoWorkflows({ + owner: projectId.split('/')[0], + repo: projectId.split('/')[1], + }) + ); + } + + getPipeline(projectId, pipelineId) { + log('getPipeline(%o, %o)', projectId, pipelineId); + return null; + } + + getPipelineJobs(projectId, pipelineId) { + log('getPipelineJobs(%o,%o)', projectId, pipelineId); + return null; + } + + isStopping() { + return this.stopping; + } + + stop() { + this.stopping = true; + } + + paginatedRequest(rq) { + return new Paginator( + (page, per_page) => { + if (this.stopping) { + return Promise.resolve({ + data: { workflows: [] }, + }); + } + return rq; + }, + (resp) => { + return resp.data.workflows || []; + }, + (resp) => { + return (this.count = parseInt(resp.data.total_count, 10)); + } + ); + } +} + +module.exports = GitHub; diff --git a/lib/indexers/ci/CIIndexer.js b/lib/indexers/ci/CIIndexer.js index 056aad1c..7761019d 100644 --- a/lib/indexers/ci/CIIndexer.js +++ b/lib/indexers/ci/CIIndexer.js @@ -48,11 +48,11 @@ class CIIndexer { (!existingBuild || new Date(existingBuild.updatedAt).getTime() < new Date(pipeline.updatedAt).getTime()) ) { log(`Processing build #${pipeline.id} [${persistCount + omitCount}]`); - return Promise.join( + return Promise.all([ this.controller.getPipeline(projectId, pipeline.id), - this.controller.getPipelineJobs(projectId, pipeline.id) - ) - .spread(this.buildMapper) + this.controller.getPipelineJobs(projectId, pipeline.id), + ]) + .then((results) => this.buildMapper(results[0], results[1])) .then(() => persistCount++); } else { log(`Skipping build #${pipeline.id} [${persistCount + omitCount}]`); diff --git a/lib/indexers/ci/GitHubCIIndexer.js b/lib/indexers/ci/GitHubCIIndexer.js new file mode 100644 index 00000000..14ca6134 --- /dev/null +++ b/lib/indexers/ci/GitHubCIIndexer.js @@ -0,0 +1,120 @@ +'use strict'; + +const Build = require('../../models/Build.js'); +const FindFiles = require('file-regex'); +const UrlProvider = require('../../url-providers'); +const log = require('debug')('importer:github-ci-indexer'); +const ConfigurationError = require('../../errors/ConfigurationError.js'); +const CIIndexer = require('./CIIndexer'); +const moment = require('moment'); +const { Octokit } = require('@octokit/rest'); +const GitHub = require('../../core/provider/github'); + +class GitHubCIIndexer { + constructor(repository, progressReporter) { + this.repo = repository; + this.reporter = progressReporter; + this.stopping = false; + } + + async configure(config) { + this.urlProvider = await UrlProvider.getCiUrlProvider(this.repo, this.reporter); + + this.github = new Octokit({ + baseUrl: 'https://api.github.com', + auth: config?.auth?.token, + }); + + //prerequisites to permit the use of travis + if (!this.urlProvider || !config) { + throw new ConfigurationError('GitHub/Octokit cannot be configured!'); + } + + const currentBranch = await this.repo.getCurrentBranch(); + const repoName = (await this.repo.getOriginUrl()).substring('https://github.com'.length + 1).slice(0, -'.git'.length); + + log(`fetching branch data from ${currentBranch}[${currentBranch}]`); + this.controller = new GitHub({ + baseUrl: 'https://api.github.com', + privateToken: config?.auth?.token, + requestTimeout: config.timeout, + }); + + this.indexer = new CIIndexer(this.reporter, this.controller, repoName, async (pipeline, jobs) => { + jobs = jobs || []; + log( + `create build ${JSON.stringify({ + id: pipeline.id, + number: pipeline.number, + sha: pipeline.commit.sha, + status: convertState(pipeline.state), + updatedAt: moment(pipeline.updated_at).toISOString(), + startedAt: moment(pipeline.started_at).toISOString(), + finishedAt: moment(pipeline.finished_at).toISOString(), + committedAt: moment(pipeline.commit.committed_at).toISOString(), + })}` + ); + const username = (pipeline.created_by || {}).login; + const userFullName = username !== undefined ? (await this.github.users.getByUsername({ username: username })).data.name : ''; + return Build.persist({ + id: pipeline.id, + sha: pipeline.commit.sha, + ref: pipeline.commit.ref, + status: convertState(pipeline.state), + tag: pipeline.tag, + user: username, + userFullName: userFullName, + createdAt: moment(pipeline.created_at || (jobs.length > 0 ? jobs[0].created_at : pipeline.started_at)).toISOString(), + updatedAt: moment(pipeline.updated_at).toISOString(), + startedAt: moment(pipeline.started_at).toISOString(), + finishedAt: moment(pipeline.finished_at).toISOString(), + committedAt: moment(pipeline.commit.committed_at).toISOString(), + duration: pipeline.duration, + jobs: jobs.map((job) => ({ + id: job.id, + name: username, + status: convertState(job.state), + stage: job.stage, + createdAt: moment(job.created_at).toISOString(), + finishedAt: moment(job.finished_at).toISOString(), + webUrl: this.urlProvider.getJobUrl(job.id), + })), + webUrl: this.urlProvider.getPipelineUrl(pipeline.id), + }); + }); + } + + async index() { + try { + return await this.indexer.index(); + } catch (e) { + console.log('Travis Indexer Failed! - ' + e.message); + } + } + + isStopping() { + return this.stopping; + } + + stop() { + if (this.indexer && !this.indexer.isStopping()) { + this.indexer.stop(); + } + + if (this.controller && !this.controller.isStopping()) { + this.controller.stop(); + } + this.stopping = true; + } +} + +const convertState = (state) => { + switch (state) { + case 'passed': + return 'success'; + default: + return state; + } +}; + +module.exports = GitHubCIIndexer; diff --git a/lib/indexers/ci/index.js b/lib/indexers/ci/index.js index 24147560..083146ef 100755 --- a/lib/indexers/ci/index.js +++ b/lib/indexers/ci/index.js @@ -7,6 +7,7 @@ let provider; const components = { travis: require('./TravisCIIndexer'), gitlab: require('./GitLabCIIndexer'), + github: require('./GitHubCIIndexer'), }; /** diff --git a/lib/paginator.js b/lib/paginator.js index fd133bda..b3a4462b 100755 --- a/lib/paginator.js +++ b/lib/paginator.js @@ -49,7 +49,7 @@ Paginator.prototype.execute = function (perPage = this.defaultPageSize) { this.once('page', (firstPage) => { try { - return this.getCount(firstPage).then((count) => { + return Promise.resolve(this.getCount(firstPage)).then((count) => { countHolder.count = count; log(`Determined total item count: ${count}`); return this.emit('count', count); @@ -143,10 +143,9 @@ function each(array, fn, i = 0) { return Promise.resolve(array); } else { const item = array[i]; - return Promise.try(() => fn(item, i)).then((ret) => { - if (ret !== false) { - return each(array, fn, i + 1); - } - }); + const ret = fn(item, i); + if (ret !== false) { + return each(array, fn, i + 1); + } } } diff --git a/lib/url-providers/index.js b/lib/url-providers/index.js index c8c1b86d..b18ebd06 100755 --- a/lib/url-providers/index.js +++ b/lib/url-providers/index.js @@ -14,6 +14,7 @@ const VcsComponents = { const CiComponents = { gitlab: require('./GitLabUrlProvider'), travis: require('./TravisCIUrlProvider'), + github: require('./GitHubUrlProvider'), }; module.exports = { From ed9e3af672eb45e5b244b57602beb23747fbbc5b Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Fri, 28 Apr 2023 13:01:14 +0200 Subject: [PATCH 02/13] #100: Github CI Indexer * adding full functionality for GitHub Ci indexer * improving CI Builds chart to also display canceled jobs --- foxx/schema.js | 13 ++--- lib/core/provider/github.js | 37 +++++++++--- lib/indexers/ci/GitHubCIIndexer.js | 56 +++++++++++-------- lib/indexers/ci/TravisCIIndexer.js | 4 +- .../components/LegendCompact/LegendCompact.js | 16 +++++- .../ciBuilds/chart/chart.js | 19 +++++-- .../ciBuilds/sagas/index.js | 1 - 7 files changed, 93 insertions(+), 53 deletions(-) diff --git a/foxx/schema.js b/foxx/schema.js index 3c097c98..fb4b3aa1 100755 --- a/foxx/schema.js +++ b/foxx/schema.js @@ -230,7 +230,6 @@ const queryType = new gql.GraphQLObjectType({ // //TODO: RETURN MERGE(build, { stats: MERGE(countsByStatus) })`; // return q; - if (args.since && args.until) { return aql` FOR build IN ${builds} @@ -239,10 +238,8 @@ const queryType = new gql.GraphQLObjectType({ FILTER DATE_TIMESTAMP(build.createdAt) >= DATE_TIMESTAMP(${args.since}) FILTER DATE_TIMESTAMP(build.createdAt) <= DATE_TIMESTAMP(${args.until}) LET countsByStatus = ( - FOR other IN ${builds} - FILTER other.finishedAt <= build.createdAt - COLLECT status = other.status WITH COUNT INTO statusCount - RETURN { [status]: statusCount } + COLLECT status = build.status WITH COUNT INTO statusCount + RETURN { [status]: statusCount } ) RETURN MERGE(build, { stats: MERGE(countsByStatus) })`; } @@ -251,10 +248,8 @@ const queryType = new gql.GraphQLObjectType({ SORT build.createdAt ASC ${limit} LET countsByStatus = ( - FOR other IN ${builds} - FILTER other.finishedAt <= build.createdAt - COLLECT status = other.status WITH COUNT INTO statusCount - RETURN { [status]: statusCount } + COLLECT status = build.status WITH COUNT INTO statusCount + RETURN { [status]: statusCount } ) RETURN MERGE(build, { stats: MERGE(countsByStatus) })`; }, diff --git a/lib/core/provider/github.js b/lib/core/provider/github.js index cd42e2ab..c98f1e6a 100644 --- a/lib/core/provider/github.js +++ b/lib/core/provider/github.js @@ -22,22 +22,39 @@ class GitHub { getPipelines(projectId) { log('getPipelines(%o)', projectId); - return this.paginatedRequest( - this.github.rest.actions.listRepoWorkflows({ + return this.paginatedRequest((page) => { + return this.github.rest.actions.listWorkflowRunsForRepo({ owner: projectId.split('/')[0], repo: projectId.split('/')[1], - }) - ); + page: page, + }); + }); } getPipeline(projectId, pipelineId) { log('getPipeline(%o, %o)', projectId, pipelineId); - return null; + return this.github.rest.actions + .getWorkflowRun({ + owner: projectId.split('/')[0], + repo: projectId.split('/')[1], + run_id: pipelineId, + }) + .then((workflowRun) => { + return workflowRun.data; + }); } getPipelineJobs(projectId, pipelineId) { log('getPipelineJobs(%o,%o)', projectId, pipelineId); - return null; + return this.github.rest.actions + .listJobsForWorkflowRun({ + owner: projectId.split('/')[0], + repo: projectId.split('/')[1], + run_id: pipelineId, + }) + .then((jobs) => { + return jobs.data.jobs; + }); } isStopping() { @@ -53,13 +70,15 @@ class GitHub { (page, per_page) => { if (this.stopping) { return Promise.resolve({ - data: { workflows: [] }, + data: { + workflow_runs: [], + }, }); } - return rq; + return rq(page); }, (resp) => { - return resp.data.workflows || []; + return resp.data.workflow_runs || []; }, (resp) => { return (this.count = parseInt(resp.data.total_count, 10)); diff --git a/lib/indexers/ci/GitHubCIIndexer.js b/lib/indexers/ci/GitHubCIIndexer.js index 14ca6134..1b60e5ab 100644 --- a/lib/indexers/ci/GitHubCIIndexer.js +++ b/lib/indexers/ci/GitHubCIIndexer.js @@ -45,41 +45,49 @@ class GitHubCIIndexer { log( `create build ${JSON.stringify({ id: pipeline.id, - number: pipeline.number, - sha: pipeline.commit.sha, - status: convertState(pipeline.state), + number: pipeline.run_number, + sha: pipeline.head_commit.sha, + status: convertState(pipeline.status), updatedAt: moment(pipeline.updated_at).toISOString(), - startedAt: moment(pipeline.started_at).toISOString(), - finishedAt: moment(pipeline.finished_at).toISOString(), - committedAt: moment(pipeline.commit.committed_at).toISOString(), + startedAt: moment(pipeline.run_started_at).toISOString(), + finishedAt: moment(pipeline.updated_at).toISOString(), + committedAt: moment(pipeline.head_commit.timestamp).toISOString(), })}` ); - const username = (pipeline.created_by || {}).login; + const username = (pipeline.actor || {}).login; const userFullName = username !== undefined ? (await this.github.users.getByUsername({ username: username })).data.name : ''; + let status = 'canceled'; + let lastStartedAt = pipeline.run_started_at; + let lastFinishedAt = pipeline.updated_at; + if (jobs.length > 0) { + status = convertState(jobs[jobs.length - 1].conclusion); + lastStartedAt = jobs[jobs.length - 1].created_at; + lastFinishedAt = jobs[jobs.length - 1].completed_at; + } return Build.persist({ id: pipeline.id, - sha: pipeline.commit.sha, - ref: pipeline.commit.ref, - status: convertState(pipeline.state), - tag: pipeline.tag, + sha: pipeline.head_sha, + ref: pipeline.head_commit.id, + status: status, + tag: pipeline.display_title, user: username, - userFullName: userFullName, + userFullName: userFullName !== null ? userFullName : username, createdAt: moment(pipeline.created_at || (jobs.length > 0 ? jobs[0].created_at : pipeline.started_at)).toISOString(), updatedAt: moment(pipeline.updated_at).toISOString(), - startedAt: moment(pipeline.started_at).toISOString(), - finishedAt: moment(pipeline.finished_at).toISOString(), - committedAt: moment(pipeline.commit.committed_at).toISOString(), - duration: pipeline.duration, + startedAt: moment(pipeline.run_started_at).toISOString(), + finishedAt: moment(lastFinishedAt).toISOString(), + committedAt: moment(pipeline.head_commit.committed_at).toISOString(), + duration: moment(lastFinishedAt).unix() - moment(lastStartedAt).unix(), jobs: jobs.map((job) => ({ id: job.id, name: username, - status: convertState(job.state), - stage: job.stage, + status: job.conclusion, + stage: job.conclusion, createdAt: moment(job.created_at).toISOString(), - finishedAt: moment(job.finished_at).toISOString(), - webUrl: this.urlProvider.getJobUrl(job.id), + finishedAt: moment(job.completed_at).toISOString(), + webUrl: job.html_url, })), - webUrl: this.urlProvider.getPipelineUrl(pipeline.id), + webUrl: pipeline.html_url, }); }); } @@ -88,7 +96,7 @@ class GitHubCIIndexer { try { return await this.indexer.index(); } catch (e) { - console.log('Travis Indexer Failed! - ' + e.message); + console.log('GithubCI Indexer Failed! - ' + e.message); } } @@ -110,8 +118,8 @@ class GitHubCIIndexer { const convertState = (state) => { switch (state) { - case 'passed': - return 'success'; + case 'failure': + return 'failed'; default: return state; } diff --git a/lib/indexers/ci/TravisCIIndexer.js b/lib/indexers/ci/TravisCIIndexer.js index c0dc67ac..c40421e9 100644 --- a/lib/indexers/ci/TravisCIIndexer.js +++ b/lib/indexers/ci/TravisCIIndexer.js @@ -62,7 +62,7 @@ class TravisCIIndexer { id: pipeline.id, sha: pipeline.commit.sha, ref: pipeline.commit.ref, - status: convertState(pipeline.state), + status: pipeline.state, tag: pipeline.tag, user: username, userFullName: userFullName, @@ -75,7 +75,7 @@ class TravisCIIndexer { jobs: jobs.map((job) => ({ id: job.id, name: username, - status: convertState(job.state), + status: job.state, stage: job.stage, createdAt: moment(job.created_at).toISOString(), finishedAt: moment(job.finished_at).toISOString(), diff --git a/ui/src/components/LegendCompact/LegendCompact.js b/ui/src/components/LegendCompact/LegendCompact.js index aa5f7ba6..9bb140d9 100644 --- a/ui/src/components/LegendCompact/LegendCompact.js +++ b/ui/src/components/LegendCompact/LegendCompact.js @@ -15,8 +15,20 @@ export default class LegendCompact extends React.Component { render() { const rects = []; if (this.props.color2) { - rects.push(); - rects.push(); + if (this.props.color3) { + rects.push(); + rects.push( + + ); + rects.push( + + ); + } else { + rects.push(); + rects.push( + + ); + } } else { rects.push(); } diff --git a/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js b/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js index 14b4a1ae..affe1b01 100644 --- a/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js +++ b/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js @@ -40,7 +40,7 @@ export default class CIBuilds extends React.Component { {this.state.ciChartData !== undefined && this.state.ciChartData.length > 0 ? ( - + ); @@ -118,12 +118,14 @@ export default class CIBuilds extends React.Component { //Iterate through time buckets const currTimestamp = curr.toDate().getTime(); const nextTimestamp = next.toDate().getTime(); - const obj = { date: currTimestamp, succeeded: 0, failed: 0 }; //Save date of time bucket, create object + const obj = { date: currTimestamp, succeeded: 0, canceled: 0, failed: 0 }; //Save date of time bucket, create object for (; i < builds.length && Date.parse(builds[i].createdAt) < nextTimestamp; i++) { //Iterate through commits that fall into this time bucket const buildDate = Date.parse(builds[i].createdAt); if (buildDate >= currTimestamp && buildDate < nextTimestamp) { + console.log(builds[i]); obj.succeeded += builds[i].stats.success || 0; + obj.canceled += builds[i].stats.canceled || -0.001; obj.failed += builds[i].stats.failed || -0.001; //-0.001 for stack layout to realize it belongs on the bottom } } @@ -134,12 +136,17 @@ export default class CIBuilds extends React.Component { const ciChartData = []; const ciScale = [0, 0]; _.each(data, function (build) { - ciChartData.push({ date: build.date, Succeeded: build.succeeded, Failed: build.failed > 0 ? build.failed * -1 : 0 }); + ciChartData.push({ + date: build.date, + Succeeded: build.succeeded, + Canceled: build.canceled > 0 ? build.canceled * -1 : 0, + Failed: build.failed > 0 ? build.failed * -1 : 0, + }); if (ciScale[1] < build.succeeded) { ciScale[1] = build.succeeded; } - if (ciScale[0] > build.failed * -1) { - ciScale[0] = build.failed * -1; + if (ciScale[0] > (build.failed + build.canceled) * -1) { + ciScale[0] = (build.failed + build.canceled) * -1; } }); diff --git a/ui/src/visualizations/VisualizationComponents/ciBuilds/sagas/index.js b/ui/src/visualizations/VisualizationComponents/ciBuilds/sagas/index.js index 6f6cf4b4..ca5c582b 100644 --- a/ui/src/visualizations/VisualizationComponents/ciBuilds/sagas/index.js +++ b/ui/src/visualizations/VisualizationComponents/ciBuilds/sagas/index.js @@ -79,7 +79,6 @@ export const fetchBuildsData = fetchFactory( .then((results) => { const filteredBuilds = results[0]; const builds = results[1]; - return { otherCount: 0, filteredBuilds, From 3d0a85eb9886d9f7e9179d9ceec0e7ff393ef259 Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Fri, 28 Apr 2023 13:24:07 +0200 Subject: [PATCH 03/13] #100: Github CI Indexer * removing unnecessary log --- .../VisualizationComponents/ciBuilds/chart/chart.js | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js b/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js index affe1b01..006334a5 100644 --- a/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js +++ b/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js @@ -123,7 +123,6 @@ export default class CIBuilds extends React.Component { //Iterate through commits that fall into this time bucket const buildDate = Date.parse(builds[i].createdAt); if (buildDate >= currTimestamp && buildDate < nextTimestamp) { - console.log(builds[i]); obj.succeeded += builds[i].stats.success || 0; obj.canceled += builds[i].stats.canceled || -0.001; obj.failed += builds[i].stats.failed || -0.001; //-0.001 for stack layout to realize it belongs on the bottom From cd94bf7bc6d307ed4c60c89a4f55cd3b1e09bd9b Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Wed, 3 May 2023 10:57:03 +0200 Subject: [PATCH 04/13] #100: Github CI Indexer * updated README.md to mention the new github indexer --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index afc74c5d..27edbd0a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Binocular ===== -[![Build Status](https://travis-ci.org/INSO-TUWien/Binocular.svg?branch=development)](https://travis-ci.org/INSO-TUWien/Binocular) +[![Build Binocular](https://github.com/INSO-TUWien/Binocular/actions/workflows/build-bincoular.yml/badge.svg?branch=develop)](https://github.com/INSO-TUWien/Binocular/actions/workflows/build-bincoular.yml) Binocular is a tool for visualizing data from various software-engineering tools. It works as a command-line tool run from a git-repository. When @@ -63,7 +63,7 @@ json. Should only be used if the repository could not detect the indexers automatically. - `its`: Holds the name of the issue tracking system indexer,for instance, gitlab or github - `ci`: Since the CI indexer importer is searching for the corresponding file in the repository, it can be necessary to specify the - correct indexer like, for example, travis. + correct indexer like, for example, gitlab, github or travis. A sample configuration file looks like this: @@ -88,7 +88,7 @@ A sample configuration file looks like this: }, "indexers": { "its": "github", - "ci": "travis" + "ci": "github" } } ``` From b5584e25458f419dcff1a6a9f178436b08b6ef95 Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Thu, 4 May 2023 10:10:42 +0200 Subject: [PATCH 05/13] #100: Github CI Indexer * changing that CI indexer defaults to github actions --- lib/importer/GenericImporter.js | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/lib/importer/GenericImporter.js b/lib/importer/GenericImporter.js index 5172c17b..8a356fff 100644 --- a/lib/importer/GenericImporter.js +++ b/lib/importer/GenericImporter.js @@ -26,10 +26,6 @@ module.exports = async (repo, componentType, componentFactory, reporter, context const typePath = componentType !== 'CI' ? 'indexers.its' : 'indexers.ci'; let type = _.toLower(config.get(typePath, 'auto')); - if (type === 'auto' && componentType === 'CI') { - type = await detectCIProvider(repo); - } - if (type === 'auto') { type = await detectRepoProvider(repo, componentType); } @@ -72,19 +68,3 @@ async function detectRepoProvider(repo, componentType) { throw new IllegalArgumentError(`Unable to auto-detect ${componentType}. Please configure it manually`); } } - -/** - * analyse the repository to check the corresponding remote provider - * - * @param repo contains the information of the repository - * @returns {Promise} - */ -// eslint-disable-next-line no-unused-vars -async function detectCIProvider(repo) { - log('Detecting CI from files in the repo...'); - if (await FindFiles(await repo.getRoot(), /.travis\.ya?ml$/)) { - return 'travis'; - } - - return 'auto'; -} From 61e9fa53d937e9f59597a37accb993cb7061af6a Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Thu, 4 May 2023 10:31:39 +0200 Subject: [PATCH 06/13] #100: Github CI Indexer * adding full error log to GitHubCIIndexer.js for testing --- lib/indexers/ci/GitHubCIIndexer.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/indexers/ci/GitHubCIIndexer.js b/lib/indexers/ci/GitHubCIIndexer.js index 1b60e5ab..ed7f3ffc 100644 --- a/lib/indexers/ci/GitHubCIIndexer.js +++ b/lib/indexers/ci/GitHubCIIndexer.js @@ -96,7 +96,8 @@ class GitHubCIIndexer { try { return await this.indexer.index(); } catch (e) { - console.log('GithubCI Indexer Failed! - ' + e.message); + console.log('GitHubCI Indexer Failed! - ' + e.message); + console.log(e); } } From a074cdece145f81666cc116d7c9a088416fe96dc Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Thu, 4 May 2023 10:38:34 +0200 Subject: [PATCH 07/13] #100: Github CI Indexer * removing full error log to GitHubCIIndexer.js for testing * adding test log --- lib/core/provider/github.js | 1 + lib/indexers/ci/GitHubCIIndexer.js | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/provider/github.js b/lib/core/provider/github.js index c98f1e6a..3902bddd 100644 --- a/lib/core/provider/github.js +++ b/lib/core/provider/github.js @@ -21,6 +21,7 @@ class GitHub { } getPipelines(projectId) { + console.log('get Runs for ' + projectId); log('getPipelines(%o)', projectId); return this.paginatedRequest((page) => { return this.github.rest.actions.listWorkflowRunsForRepo({ diff --git a/lib/indexers/ci/GitHubCIIndexer.js b/lib/indexers/ci/GitHubCIIndexer.js index ed7f3ffc..dd1ed9d4 100644 --- a/lib/indexers/ci/GitHubCIIndexer.js +++ b/lib/indexers/ci/GitHubCIIndexer.js @@ -97,7 +97,6 @@ class GitHubCIIndexer { return await this.indexer.index(); } catch (e) { console.log('GitHubCI Indexer Failed! - ' + e.message); - console.log(e); } } From 0057fd62da7592f048fe13b88a5c8023b68af237 Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Thu, 4 May 2023 10:44:07 +0200 Subject: [PATCH 08/13] #100: Github CI Indexer * adding test log --- lib/core/provider/github.js | 1 - lib/indexers/ci/CIIndexer.js | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/provider/github.js b/lib/core/provider/github.js index 3902bddd..c98f1e6a 100644 --- a/lib/core/provider/github.js +++ b/lib/core/provider/github.js @@ -21,7 +21,6 @@ class GitHub { } getPipelines(projectId) { - console.log('get Runs for ' + projectId); log('getPipelines(%o)', projectId); return this.paginatedRequest((page) => { return this.github.rest.actions.listWorkflowRunsForRepo({ diff --git a/lib/indexers/ci/CIIndexer.js b/lib/indexers/ci/CIIndexer.js index 7761019d..f8e9f48b 100644 --- a/lib/indexers/ci/CIIndexer.js +++ b/lib/indexers/ci/CIIndexer.js @@ -6,6 +6,7 @@ const log = require('debug')('idx:ci:indexer'); class CIIndexer { constructor(progressReporter, controller, projectId, createBuildArtifactHandler) { + console.log('project id: '+projectId); this.reporter = progressReporter; this.controller = controller; this.projectId = projectId; From 5bd729ddea57bcce82d16504f4264544b7ef2553 Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Thu, 4 May 2023 10:44:41 +0200 Subject: [PATCH 09/13] #100: Github CI Indexer * fixing eslint error --- lib/indexers/ci/CIIndexer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/indexers/ci/CIIndexer.js b/lib/indexers/ci/CIIndexer.js index f8e9f48b..f0971fe9 100644 --- a/lib/indexers/ci/CIIndexer.js +++ b/lib/indexers/ci/CIIndexer.js @@ -6,7 +6,7 @@ const log = require('debug')('idx:ci:indexer'); class CIIndexer { constructor(progressReporter, controller, projectId, createBuildArtifactHandler) { - console.log('project id: '+projectId); + console.log('project id: ' + projectId); this.reporter = progressReporter; this.controller = controller; this.projectId = projectId; From e000db634159d228aec7427fbaf06b610b4ce826 Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Thu, 4 May 2023 10:54:29 +0200 Subject: [PATCH 10/13] #100: Github CI Indexer * adding test log --- lib/indexers/ci/CIIndexer.js | 1 - lib/indexers/ci/GitHubCIIndexer.js | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/indexers/ci/CIIndexer.js b/lib/indexers/ci/CIIndexer.js index f0971fe9..7761019d 100644 --- a/lib/indexers/ci/CIIndexer.js +++ b/lib/indexers/ci/CIIndexer.js @@ -6,7 +6,6 @@ const log = require('debug')('idx:ci:indexer'); class CIIndexer { constructor(progressReporter, controller, projectId, createBuildArtifactHandler) { - console.log('project id: ' + projectId); this.reporter = progressReporter; this.controller = controller; this.projectId = projectId; diff --git a/lib/indexers/ci/GitHubCIIndexer.js b/lib/indexers/ci/GitHubCIIndexer.js index dd1ed9d4..29d844b3 100644 --- a/lib/indexers/ci/GitHubCIIndexer.js +++ b/lib/indexers/ci/GitHubCIIndexer.js @@ -31,7 +31,10 @@ class GitHubCIIndexer { } const currentBranch = await this.repo.getCurrentBranch(); - const repoName = (await this.repo.getOriginUrl()).substring('https://github.com'.length + 1).slice(0, -'.git'.length); + const originUrl = await this.repo.getOriginUrl(); + console.log('originUrl ' + originUrl); + const repoName = originUrl.substring('https://github.com'.length + 1).slice(0, -'.git'.length); + console.log('repoName ' + repoName); log(`fetching branch data from ${currentBranch}[${currentBranch}]`); this.controller = new GitHub({ From be5f83bc6e2ca3d8ab669799e82c3ccae86729d1 Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Thu, 4 May 2023 11:02:00 +0200 Subject: [PATCH 11/13] #100: Github CI Indexer * adding condition to just remove '.git' if the originUrl actually includes it --- lib/indexers/ci/GitHubCIIndexer.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/indexers/ci/GitHubCIIndexer.js b/lib/indexers/ci/GitHubCIIndexer.js index 29d844b3..fc9e8bf1 100644 --- a/lib/indexers/ci/GitHubCIIndexer.js +++ b/lib/indexers/ci/GitHubCIIndexer.js @@ -33,7 +33,10 @@ class GitHubCIIndexer { const currentBranch = await this.repo.getCurrentBranch(); const originUrl = await this.repo.getOriginUrl(); console.log('originUrl ' + originUrl); - const repoName = originUrl.substring('https://github.com'.length + 1).slice(0, -'.git'.length); + let repoName = originUrl.substring('https://github.com'.length + 1); + if (repoName.endsWith('.git')) { + repoName = repoName.slice(0, -'.git'.length); + } console.log('repoName ' + repoName); log(`fetching branch data from ${currentBranch}[${currentBranch}]`); From 704b2dce9ae925bc4b4ed1376bbd8b0440b9b738 Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Thu, 4 May 2023 11:26:22 +0200 Subject: [PATCH 12/13] #100: Github CI Indexer * adding cancelled stats for localDB getBuildData function * removing test logs --- lib/indexers/ci/GitHubCIIndexer.js | 3 +-- ui/src/database/localDB/builds.js | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/indexers/ci/GitHubCIIndexer.js b/lib/indexers/ci/GitHubCIIndexer.js index fc9e8bf1..3dd2b93c 100644 --- a/lib/indexers/ci/GitHubCIIndexer.js +++ b/lib/indexers/ci/GitHubCIIndexer.js @@ -32,12 +32,11 @@ class GitHubCIIndexer { const currentBranch = await this.repo.getCurrentBranch(); const originUrl = await this.repo.getOriginUrl(); - console.log('originUrl ' + originUrl); + let repoName = originUrl.substring('https://github.com'.length + 1); if (repoName.endsWith('.git')) { repoName = repoName.slice(0, -'.git'.length); } - console.log('repoName ' + repoName); log(`fetching branch data from ${currentBranch}[${currentBranch}]`); this.controller = new GitHub({ diff --git a/ui/src/database/localDB/builds.js b/ui/src/database/localDB/builds.js index 490c1c59..60b14d1e 100644 --- a/ui/src/database/localDB/builds.js +++ b/ui/src/database/localDB/builds.js @@ -28,6 +28,8 @@ export default class Builds { stats.success = 1; } else if (build.status === 'failed' || build.status === 'errored') { stats.failed = 1; + } else if (build.status === 'cancelled') { + stats.cancelled = 1; } build.stats = stats; From 0e95af8c5081332809bac554bab4461f9d0254ca Mon Sep 17 00:00:00 2001 From: Maximilian Zenz Date: Thu, 4 May 2023 12:31:24 +0200 Subject: [PATCH 13/13] #100: Github CI Indexer * fixing writing of cancelled to be coherent with github * fixing that cancelled builds didn't show in offline export --- foxx/types/build.js | 4 ++-- ui/src/database/localDB/builds.js | 7 +++---- ui/src/database/serverDB/builds.js | 8 ++++---- .../ciBuilds/chart/chart.js | 14 +++++++------- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/foxx/types/build.js b/foxx/types/build.js index 1eb3c82d..9ecebfc3 100755 --- a/foxx/types/build.js +++ b/foxx/types/build.js @@ -15,7 +15,7 @@ const BuildStatus = new gql.GraphQLEnumType({ success: {}, pending: {}, running: {}, - canceled: {}, + cancelled: {}, skipped: {}, created: {}, started: {}, @@ -138,7 +138,7 @@ module.exports = new gql.GraphQLObjectType({ pending: { type: gql.GraphQLInt, }, - canceled: { + cancelled: { type: gql.GraphQLInt, }, }, diff --git a/ui/src/database/localDB/builds.js b/ui/src/database/localDB/builds.js index 60b14d1e..6f04b03c 100644 --- a/ui/src/database/localDB/builds.js +++ b/ui/src/database/localDB/builds.js @@ -19,7 +19,7 @@ export default class Builds { static getBuildData(db, commitSpan, significantSpan) { // add stats object to each build return findAll(db, 'builds').then((res) => { - const emptyStats = { success: 0, failed: 0, pending: 0, canceled: 0 }; + const emptyStats = { success: 0, failed: 0, pending: 0, cancelled: 0 }; return res.docs.map((build) => { const stats = Object.assign({}, emptyStats); @@ -33,7 +33,6 @@ export default class Builds { } build.stats = stats; - return build; }); }); @@ -48,7 +47,7 @@ export default class Builds { success: 0, failed: 0, pending: 0, - canceled: 0, + cancelled: 0, }, }, ]; @@ -62,7 +61,7 @@ export default class Builds { date: new Date(next), stats: _.defaults( { - total: (build.stats.success || 0) + (build.stats.failed || 0) + (build.stats.pending || 0) + (build.stats.canceled || 0), + total: (build.stats.success || 0) + (build.stats.failed || 0) + (build.stats.pending || 0) + (build.stats.cancelled || 0), }, build.stats ), diff --git a/ui/src/database/serverDB/builds.js b/ui/src/database/serverDB/builds.js index 1b971bb0..a2f43167 100644 --- a/ui/src/database/serverDB/builds.js +++ b/ui/src/database/serverDB/builds.js @@ -26,7 +26,7 @@ export default class Builds { success failed pending - canceled + cancelled } } } @@ -52,7 +52,7 @@ export default class Builds { success: 0, failed: 0, pending: 0, - canceled: 0, + cancelled: 0, }, }, ]; @@ -74,7 +74,7 @@ export default class Builds { success failed pending - canceled + cancelled } } } @@ -92,7 +92,7 @@ export default class Builds { date: new Date(next), stats: _.defaults( { - total: (build.stats.success || 0) + (build.stats.failed || 0) + (build.stats.pending || 0) + (build.stats.canceled || 0), + total: (build.stats.success || 0) + (build.stats.failed || 0) + (build.stats.pending || 0) + (build.stats.cancelled || 0), }, build.stats ), diff --git a/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js b/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js index 006334a5..50f2a019 100644 --- a/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js +++ b/ui/src/visualizations/VisualizationComponents/ciBuilds/chart/chart.js @@ -40,7 +40,7 @@ export default class CIBuilds extends React.Component { {this.state.ciChartData !== undefined && this.state.ciChartData.length > 0 ? ( - + ); @@ -118,13 +118,13 @@ export default class CIBuilds extends React.Component { //Iterate through time buckets const currTimestamp = curr.toDate().getTime(); const nextTimestamp = next.toDate().getTime(); - const obj = { date: currTimestamp, succeeded: 0, canceled: 0, failed: 0 }; //Save date of time bucket, create object + const obj = { date: currTimestamp, succeeded: 0, cancelled: 0, failed: 0 }; //Save date of time bucket, create object for (; i < builds.length && Date.parse(builds[i].createdAt) < nextTimestamp; i++) { //Iterate through commits that fall into this time bucket const buildDate = Date.parse(builds[i].createdAt); if (buildDate >= currTimestamp && buildDate < nextTimestamp) { obj.succeeded += builds[i].stats.success || 0; - obj.canceled += builds[i].stats.canceled || -0.001; + obj.cancelled += builds[i].stats.cancelled || -0.001; obj.failed += builds[i].stats.failed || -0.001; //-0.001 for stack layout to realize it belongs on the bottom } } @@ -138,14 +138,14 @@ export default class CIBuilds extends React.Component { ciChartData.push({ date: build.date, Succeeded: build.succeeded, - Canceled: build.canceled > 0 ? build.canceled * -1 : 0, + Cancelled: build.cancelled > 0 ? build.cancelled * -1 : 0, Failed: build.failed > 0 ? build.failed * -1 : 0, }); if (ciScale[1] < build.succeeded) { ciScale[1] = build.succeeded; } - if (ciScale[0] > (build.failed + build.canceled) * -1) { - ciScale[0] = (build.failed + build.canceled) * -1; + if (ciScale[0] > (build.failed + build.cancelled) * -1) { + ciScale[0] = (build.failed + build.cancelled) * -1; } });