DB: new commits-builds connection

- instead of the `sha`/`ref` attributes in the builds collection, the commits-builds connection is used.
- #145

binocular.js

@@ -53,6 +53,7 @@ const BranchFileConnection = require('./lib/models/BranchFileConnection');
 const BranchFileFileConnection = require('./lib/models/BranchFileFileConnection.js');
 const CommitFileStakeholderConnection = require('./lib/models/CommitFileStakeholderConnection.js');
 const CommitFileConnection = require('./lib/models/CommitFileConnection.js');
+const CommitBuildConnection = require('./lib/models/CommitBuildConnection.js');
 const ConfigurationError = require('./lib/errors/ConfigurationError');
 const DatabaseError = require('./lib/errors/DatabaseError');
 const GateWayService = require('./lib/gateway-service');
@@ -330,6 +331,7 @@ async function indexing(indexers, context, reporter, gateway, indexingThread) {
     // This is why we connect them here and then delete the temporary references in the collections themselves
     // (like the `mentions` field in issues).
     await connectIssuesAndCommits();
+    await connectCommitsAndBuilds();
 
     threadLog(indexingThread, 'Indexing finished');
   } catch (error) {
@@ -494,6 +496,7 @@ function ensureDb(repo, context) {
         Module.ensureCollection(),
         MergeRequest.ensureCollection(),
         CommitFileConnection.ensureCollection(),
+        CommitBuildConnection.ensureCollection(),
         LanguageFileConnection.ensureCollection(),
         CommitStakeholderConnection.ensureCollection(),
         IssueStakeholderConnection.ensureCollection(),
@@ -609,3 +612,18 @@ async function connectIssuesAndCommits() {
   //remove the temporary `mentions` attribute since we have the connections now
   await Issue.deleteMentionsAttribute();
 }
+
+async function connectCommitsAndBuilds() {
+  const builds = await Build.findAll();
+  const commits = await Commit.findAll();
+
+  for (const build of builds) {
+    if (!build.sha) continue;
+    const commit = commits.filter((c) => c.sha === build.sha);
+    if (commit && commit[0]) {
+      commit[0].connect(build);
+    }
+  }
+
+  await Build.deleteShaRefAttributes();
+}

foxx/types/build.js

@@ -6,6 +6,7 @@ const db = arangodb.db;
 const aql = arangodb.aql;
 const Timestamp = require('./Timestamp.js');
 const commits = db._collection('commits');
+const commitsToBuilds = db._collection('commits-builds');
 
 const BuildStatus = new gql.GraphQLEnumType({
   name: 'BuildStatus',
@@ -69,16 +70,10 @@ module.exports = new gql.GraphQLObjectType({
         type: new gql.GraphQLNonNull(gql.GraphQLString),
         resolve: (e) => e._key,
       },
-      sha: {
-        type: gql.GraphQLString,
-        description: 'Sha of the commit that triggered the build',
-      },
+      //TODO is this necessary?
       beforeSha: {
         type: gql.GraphQLString,
       },
-      ref: {
-        type: gql.GraphQLString,
-      },
       status: {
         type: BuildStatus,
         description: 'Status of the build',
@@ -156,8 +151,8 @@ module.exports = new gql.GraphQLObjectType({
           return db
             ._query(
               aql`
-            FOR commit IN ${commits}
-              FILTER commit.sha == ${build.sha}
+              FOR commit, edge
+              IN OUTBOUND ${build} ${commitsToBuilds}
               RETURN commit`
             )
             .toArray()[0];

foxx/types/commit.js

@@ -6,10 +6,10 @@ const db = arangodb.db;
 const aql = arangodb.aql;
 const commitsToFiles = db._collection('commits-files');
 const commitsToFilesToStakeholders = db._collection('commits-files-stakeholders');
-const builds = db._collection('builds');
 const commitsToStakeholders = db._collection('commits-stakeholders');
 const commitsToLanguages = db._collection('commits-languages');
 const CommitsToModules = db._collection('commits-modules');
+const commitsToBuilds = db._collection('commits-builds');
 const paginated = require('./paginated.js');
 const Timestamp = require('./Timestamp.js');
 
@@ -147,10 +147,9 @@ module.exports = new gql.GraphQLObjectType({
           return db
             ._query(
               aql`
-              FOR build
-              IN ${builds}
-              FILTER build.sha == ${commit.sha}
-                RETURN build`
+              FOR build, edge
+              IN INBOUND ${commit} ${commitsToBuilds}
+              RETURN build`
             )
             .toArray();
         },

lib/models/Build.js

@@ -2,6 +2,7 @@
 
 const _ = require('lodash');
 const Model = require('./Model.js');
+const aql = require('arangojs').aql;
 
 const Build = Model.define('Build', {
   attributes: [
@@ -35,4 +36,12 @@ Build.persist = function (_buildData) {
   return Build.ensureById(buildData.id, buildData, { ignoreUnknownAttributes: true });
 };
 
+Build.deleteShaRefAttributes = async function () {
+  return Build.rawDb.query(
+    aql`
+    FOR b IN builds
+    REPLACE b WITH UNSET(b, "sha", "ref") IN builds`
+  );
+};
+
 module.exports = Build;

lib/models/CommitBuildConnection.js

@@ -0,0 +1,7 @@
+'use strict';
+
+const Connection = require('./Connection');
+const Commit = require('./Commit');
+const Build = require('./Build');
+
+module.exports = Connection.define(Commit, Build);

ui/src/database/localDB.js

@@ -24,6 +24,7 @@ import branchesFilesFiles from '../../db_export/branches-files-files.json';
 import builds from '../../db_export/builds.json';
 import commitsCommits from '../../db_export/commits-commits.json';
 import commitsFiles from '../../db_export/commits-files.json';
+import commitsBuilds from '../../db_export/commits-builds.json';
 import commitsFilesStakeholders from '../../db_export/commits-files-stakeholders.json';
 import commitsLanguages from '../../db_export/commits-languages.json';
 import commitsModules from '../../db_export/commits-modules.json';
@@ -47,6 +48,7 @@ const relations = {
   'commits-commits': commitsCommits,
   'commits-files': commitsFiles,
   'commits-files-stakeholders': commitsFilesStakeholders,
+  'commits-builds': commitsBuilds,
   'commits-languages': commitsLanguages,
   'commits-modules': commitsModules,
   'commits-stakeholders': commitsStakeholders,
@@ -124,7 +126,7 @@ export default class LocalDB {
   }
 
   static getBuildData(commitSpan, significantSpan) {
-    return Builds.getBuildData(db, commitSpan, significantSpan);
+    return Builds.getBuildData(db, tripleStore, commitSpan, significantSpan);
   }
 
   static getIssueData(issueSpan, significantSpan) {
@@ -242,6 +244,7 @@ export default class LocalDB {
     database.commits_commits = relations['commits-commits'];
     database.commits_files = relations['commits-files'];
     database.commits_files_stakeholders = relations['commits-files-stakeholders'];
+    database.commits_builds = relations['commits-builds'];
     database.commits_languages = relations['commits-languages'];
     database.commits_modules = relations['commits-modules'];
     database.commits_stakeholders = relations['commits-stakeholders'];

ui/src/database/localDB/builds.js

@@ -15,10 +15,19 @@ function findAll(database, collection) {
   });
 }
 
+function findCommitBuildConnections(relations) {
+  return relations.find({
+    selector: { _id: { $regex: new RegExp('^commits-builds/.*') } },
+  });
+}
+
 export default class Builds {
-  static getBuildData(db, commitSpan, significantSpan) {
+  static getBuildData(db, relations, commitSpan, significantSpan) {
     // add stats object to each build
-    return findAll(db, 'builds').then((res) => {
+    return findAll(db, 'builds').then(async (res) => {
+      const allCommits = (await findAll(db, 'commits')).docs;
+      const commitBuildConnections = (await findCommitBuildConnections(relations)).docs;
+
       const emptyStats = { success: 0, failed: 0, pending: 0, cancelled: 0 };
 
       return res.docs.map((build) => {
@@ -31,8 +40,17 @@ export default class Builds {
         } else if (build.status === 'cancelled') {
           stats.cancelled = 1;
         }
-
         build.stats = stats;
+        build.commit = { sha: null };
+
+        const relevantConnection = commitBuildConnections.filter((cb) => cb.from === build._id);
+        if (relevantConnection.length !== 0) {
+          const relevantCommit = allCommits.filter((c) => c._id === relevantConnection[0].to);
+          if (relevantCommit.length !== 0) {
+            build.commit.sha = relevantCommit[0].sha;
+          }
+        }
+
         return build;
       });
     });

ui/src/database/serverDB/builds.js

@@ -20,7 +20,6 @@ export default class Builds {
               count
               data {
                 id
-                sha
                 status
                 webUrl
                 createdAt
@@ -31,6 +30,9 @@ export default class Builds {
                   pending
                   cancelled
                 }
+                commit {
+                  sha
+                }
               }
             }
           }`,

ui/src/visualizations/code-expertise/sagas/helper.js

@@ -186,7 +186,7 @@ export function addBuildData(relevantCommits, builds) {
   return relevantCommits.map((commit) => {
     const resultCommit = commit;
     resultCommit['build'] = null;
-    const relevantBuilds = builds.filter((build) => build.sha === commit.sha);
+    const relevantBuilds = builds.filter((build) => build.commit && build.commit.sha === commit.sha);
     if (relevantBuilds.length > 0) {
       resultCommit['build'] = relevantBuilds[0].status;
       resultCommit['buildUrl'] = relevantBuilds[0].webUrl;

ui/src/visualizations/legacy/language-module-river/chart/chart.js

@@ -230,7 +230,7 @@ export default class LanguageModuleRiver extends React.Component {
    * @returns [RiverData] get a list of all aggregated commits
    */
   createAggregatedRiverData(commits, props, granularity) {
-    const jobs = _.flatMap(props.builds, (build) => build.jobs.map((job) => Object.assign(job, { sha: build.sha })));
+    const jobs = _.flatMap(props.builds, (build) => build.jobs.map((job) => Object.assign(job, { sha: build.commit.sha })));
     const groupedBuilds = _.groupBy(jobs, 'sha');
 
     // aggregate all commits referring to a given commit stream and the corresponding timespan aggregation

ui/src/visualizations/legacy/language-module-river/sagas/getBuildData.js

@@ -20,7 +20,9 @@ const getBuildsPage = (page, perPage) => {
         page
         perPage
         data {
-          sha
+          commit{
+            sha
+          }
           jobs {
             status
           }