From ac3b5ae21f3570b0a9baeb4d2cab4c8ab6fb3234 Mon Sep 17 00:00:00 2001 From: Nyzabes <96738332+Nyzabes@users.noreply.github.com> Date: Thu, 31 Aug 2023 13:34:51 +0200 Subject: [PATCH] DB: new commits-builds connection - instead of the `sha`/`ref` attributes in the builds collection, the commits-builds connection is used. - #145 --- binocular.js | 18 ++++++++++++++ foxx/types/build.js | 13 ++++------ foxx/types/commit.js | 9 ++++--- lib/models/Build.js | 9 +++++++ lib/models/CommitBuildConnection.js | 7 ++++++ ui/src/database/localDB.js | 5 +++- ui/src/database/localDB/builds.js | 24 ++++++++++++++++--- ui/src/database/serverDB/builds.js | 4 +++- .../code-expertise/sagas/helper.js | 2 +- .../language-module-river/chart/chart.js | 2 +- .../sagas/getBuildData.js | 4 +++- 11 files changed, 75 insertions(+), 22 deletions(-) create mode 100644 lib/models/CommitBuildConnection.js diff --git a/binocular.js b/binocular.js index 202929e9..404f99e8 100755 --- a/binocular.js +++ b/binocular.js @@ -53,6 +53,7 @@ const BranchFileConnection = require('./lib/models/BranchFileConnection'); const BranchFileFileConnection = require('./lib/models/BranchFileFileConnection.js'); const CommitFileStakeholderConnection = require('./lib/models/CommitFileStakeholderConnection.js'); const CommitFileConnection = require('./lib/models/CommitFileConnection.js'); +const CommitBuildConnection = require('./lib/models/CommitBuildConnection.js'); const ConfigurationError = require('./lib/errors/ConfigurationError'); const DatabaseError = require('./lib/errors/DatabaseError'); const GateWayService = require('./lib/gateway-service'); @@ -330,6 +331,7 @@ async function indexing(indexers, context, reporter, gateway, indexingThread) { // This is why we connect them here and then delete the temporary references in the collections themselves // (like the `mentions` field in issues). await connectIssuesAndCommits(); + await connectCommitsAndBuilds(); threadLog(indexingThread, 'Indexing finished'); } catch (error) { @@ -494,6 +496,7 @@ function ensureDb(repo, context) { Module.ensureCollection(), MergeRequest.ensureCollection(), CommitFileConnection.ensureCollection(), + CommitBuildConnection.ensureCollection(), LanguageFileConnection.ensureCollection(), CommitStakeholderConnection.ensureCollection(), IssueStakeholderConnection.ensureCollection(), @@ -609,3 +612,18 @@ async function connectIssuesAndCommits() { //remove the temporary `mentions` attribute since we have the connections now await Issue.deleteMentionsAttribute(); } + +async function connectCommitsAndBuilds() { + const builds = await Build.findAll(); + const commits = await Commit.findAll(); + + for (const build of builds) { + if (!build.sha) continue; + const commit = commits.filter((c) => c.sha === build.sha); + if (commit && commit[0]) { + commit[0].connect(build); + } + } + + await Build.deleteShaRefAttributes(); +} diff --git a/foxx/types/build.js b/foxx/types/build.js index 2b83f054..cdc14a8a 100755 --- a/foxx/types/build.js +++ b/foxx/types/build.js @@ -6,6 +6,7 @@ const db = arangodb.db; const aql = arangodb.aql; const Timestamp = require('./Timestamp.js'); const commits = db._collection('commits'); +const commitsToBuilds = db._collection('commits-builds'); const BuildStatus = new gql.GraphQLEnumType({ name: 'BuildStatus', @@ -69,16 +70,10 @@ module.exports = new gql.GraphQLObjectType({ type: new gql.GraphQLNonNull(gql.GraphQLString), resolve: (e) => e._key, }, - sha: { - type: gql.GraphQLString, - description: 'Sha of the commit that triggered the build', - }, + //TODO is this necessary? beforeSha: { type: gql.GraphQLString, }, - ref: { - type: gql.GraphQLString, - }, status: { type: BuildStatus, description: 'Status of the build', @@ -156,8 +151,8 @@ module.exports = new gql.GraphQLObjectType({ return db ._query( aql` - FOR commit IN ${commits} - FILTER commit.sha == ${build.sha} + FOR commit, edge + IN OUTBOUND ${build} ${commitsToBuilds} RETURN commit` ) .toArray()[0]; diff --git a/foxx/types/commit.js b/foxx/types/commit.js index 3eb74a33..ca171e1f 100755 --- a/foxx/types/commit.js +++ b/foxx/types/commit.js @@ -6,10 +6,10 @@ const db = arangodb.db; const aql = arangodb.aql; const commitsToFiles = db._collection('commits-files'); const commitsToFilesToStakeholders = db._collection('commits-files-stakeholders'); -const builds = db._collection('builds'); const commitsToStakeholders = db._collection('commits-stakeholders'); const commitsToLanguages = db._collection('commits-languages'); const CommitsToModules = db._collection('commits-modules'); +const commitsToBuilds = db._collection('commits-builds'); const paginated = require('./paginated.js'); const Timestamp = require('./Timestamp.js'); @@ -147,10 +147,9 @@ module.exports = new gql.GraphQLObjectType({ return db ._query( aql` - FOR build - IN ${builds} - FILTER build.sha == ${commit.sha} - RETURN build` + FOR build, edge + IN INBOUND ${commit} ${commitsToBuilds} + RETURN build` ) .toArray(); }, diff --git a/lib/models/Build.js b/lib/models/Build.js index 67139848..3b9c8fc8 100755 --- a/lib/models/Build.js +++ b/lib/models/Build.js @@ -2,6 +2,7 @@ const _ = require('lodash'); const Model = require('./Model.js'); +const aql = require('arangojs').aql; const Build = Model.define('Build', { attributes: [ @@ -35,4 +36,12 @@ Build.persist = function (_buildData) { return Build.ensureById(buildData.id, buildData, { ignoreUnknownAttributes: true }); }; +Build.deleteShaRefAttributes = async function () { + return Build.rawDb.query( + aql` + FOR b IN builds + REPLACE b WITH UNSET(b, "sha", "ref") IN builds` + ); +}; + module.exports = Build; diff --git a/lib/models/CommitBuildConnection.js b/lib/models/CommitBuildConnection.js new file mode 100644 index 00000000..76147309 --- /dev/null +++ b/lib/models/CommitBuildConnection.js @@ -0,0 +1,7 @@ +'use strict'; + +const Connection = require('./Connection'); +const Commit = require('./Commit'); +const Build = require('./Build'); + +module.exports = Connection.define(Commit, Build); diff --git a/ui/src/database/localDB.js b/ui/src/database/localDB.js index c163f1a5..73fa9f6a 100644 --- a/ui/src/database/localDB.js +++ b/ui/src/database/localDB.js @@ -24,6 +24,7 @@ import branchesFilesFiles from '../../db_export/branches-files-files.json'; import builds from '../../db_export/builds.json'; import commitsCommits from '../../db_export/commits-commits.json'; import commitsFiles from '../../db_export/commits-files.json'; +import commitsBuilds from '../../db_export/commits-builds.json'; import commitsFilesStakeholders from '../../db_export/commits-files-stakeholders.json'; import commitsLanguages from '../../db_export/commits-languages.json'; import commitsModules from '../../db_export/commits-modules.json'; @@ -47,6 +48,7 @@ const relations = { 'commits-commits': commitsCommits, 'commits-files': commitsFiles, 'commits-files-stakeholders': commitsFilesStakeholders, + 'commits-builds': commitsBuilds, 'commits-languages': commitsLanguages, 'commits-modules': commitsModules, 'commits-stakeholders': commitsStakeholders, @@ -124,7 +126,7 @@ export default class LocalDB { } static getBuildData(commitSpan, significantSpan) { - return Builds.getBuildData(db, commitSpan, significantSpan); + return Builds.getBuildData(db, tripleStore, commitSpan, significantSpan); } static getIssueData(issueSpan, significantSpan) { @@ -242,6 +244,7 @@ export default class LocalDB { database.commits_commits = relations['commits-commits']; database.commits_files = relations['commits-files']; database.commits_files_stakeholders = relations['commits-files-stakeholders']; + database.commits_builds = relations['commits-builds']; database.commits_languages = relations['commits-languages']; database.commits_modules = relations['commits-modules']; database.commits_stakeholders = relations['commits-stakeholders']; diff --git a/ui/src/database/localDB/builds.js b/ui/src/database/localDB/builds.js index 6f04b03c..6df703c4 100644 --- a/ui/src/database/localDB/builds.js +++ b/ui/src/database/localDB/builds.js @@ -15,10 +15,19 @@ function findAll(database, collection) { }); } +function findCommitBuildConnections(relations) { + return relations.find({ + selector: { _id: { $regex: new RegExp('^commits-builds/.*') } }, + }); +} + export default class Builds { - static getBuildData(db, commitSpan, significantSpan) { + static getBuildData(db, relations, commitSpan, significantSpan) { // add stats object to each build - return findAll(db, 'builds').then((res) => { + return findAll(db, 'builds').then(async (res) => { + const allCommits = (await findAll(db, 'commits')).docs; + const commitBuildConnections = (await findCommitBuildConnections(relations)).docs; + const emptyStats = { success: 0, failed: 0, pending: 0, cancelled: 0 }; return res.docs.map((build) => { @@ -31,8 +40,17 @@ export default class Builds { } else if (build.status === 'cancelled') { stats.cancelled = 1; } - build.stats = stats; + build.commit = { sha: null }; + + const relevantConnection = commitBuildConnections.filter((cb) => cb.from === build._id); + if (relevantConnection.length !== 0) { + const relevantCommit = allCommits.filter((c) => c._id === relevantConnection[0].to); + if (relevantCommit.length !== 0) { + build.commit.sha = relevantCommit[0].sha; + } + } + return build; }); }); diff --git a/ui/src/database/serverDB/builds.js b/ui/src/database/serverDB/builds.js index eae1d7a7..188d82cb 100644 --- a/ui/src/database/serverDB/builds.js +++ b/ui/src/database/serverDB/builds.js @@ -20,7 +20,6 @@ export default class Builds { count data { id - sha status webUrl createdAt @@ -31,6 +30,9 @@ export default class Builds { pending cancelled } + commit { + sha + } } } }`, diff --git a/ui/src/visualizations/code-expertise/sagas/helper.js b/ui/src/visualizations/code-expertise/sagas/helper.js index 8e64757f..328702a3 100644 --- a/ui/src/visualizations/code-expertise/sagas/helper.js +++ b/ui/src/visualizations/code-expertise/sagas/helper.js @@ -186,7 +186,7 @@ export function addBuildData(relevantCommits, builds) { return relevantCommits.map((commit) => { const resultCommit = commit; resultCommit['build'] = null; - const relevantBuilds = builds.filter((build) => build.sha === commit.sha); + const relevantBuilds = builds.filter((build) => build.commit && build.commit.sha === commit.sha); if (relevantBuilds.length > 0) { resultCommit['build'] = relevantBuilds[0].status; resultCommit['buildUrl'] = relevantBuilds[0].webUrl; diff --git a/ui/src/visualizations/legacy/language-module-river/chart/chart.js b/ui/src/visualizations/legacy/language-module-river/chart/chart.js index 988d1027..00b64a17 100644 --- a/ui/src/visualizations/legacy/language-module-river/chart/chart.js +++ b/ui/src/visualizations/legacy/language-module-river/chart/chart.js @@ -230,7 +230,7 @@ export default class LanguageModuleRiver extends React.Component { * @returns [RiverData] get a list of all aggregated commits */ createAggregatedRiverData(commits, props, granularity) { - const jobs = _.flatMap(props.builds, (build) => build.jobs.map((job) => Object.assign(job, { sha: build.sha }))); + const jobs = _.flatMap(props.builds, (build) => build.jobs.map((job) => Object.assign(job, { sha: build.commit.sha }))); const groupedBuilds = _.groupBy(jobs, 'sha'); // aggregate all commits referring to a given commit stream and the corresponding timespan aggregation diff --git a/ui/src/visualizations/legacy/language-module-river/sagas/getBuildData.js b/ui/src/visualizations/legacy/language-module-river/sagas/getBuildData.js index 3dacda38..6d69af08 100644 --- a/ui/src/visualizations/legacy/language-module-river/sagas/getBuildData.js +++ b/ui/src/visualizations/legacy/language-module-river/sagas/getBuildData.js @@ -20,7 +20,9 @@ const getBuildsPage = (page, perPage) => { page perPage data { - sha + commit{ + sha + } jobs { status }