diff --git a/binocular.js b/binocular.js index 7b12140b..534dd9f0 100755 --- a/binocular.js +++ b/binocular.js @@ -318,7 +318,7 @@ async function indexing(indexers, context, reporter, gateway, indexingThread) { threadLog(indexingThread, 'All indexers stopped!'); return; } - Promise.all([Commit.deduceStakeholders(), Issue.deduceStakeholders()]).then(() => { + Issue.deduceStakeholders().then(() => { threadLog(indexingThread, 'Indexing finished'); projectStructureHelper.checkProjectStructureAndFix(); }); diff --git a/foxx/schema.js b/foxx/schema.js index f5f213e9..69499108 100755 --- a/foxx/schema.js +++ b/foxx/schema.js @@ -201,6 +201,7 @@ const queryType = new gql.GraphQLObjectType({ ${limit} RETURN stakeholder`, }), + //TODO use stakeholders collection here committers: { type: new gql.GraphQLList(gql.GraphQLString), resolve: () => { diff --git a/foxx/types/commit.js b/foxx/types/commit.js index fbdbf1a0..3eb74a33 100755 --- a/foxx/types/commit.js +++ b/foxx/types/commit.js @@ -38,6 +38,17 @@ module.exports = new gql.GraphQLObjectType({ signature: { type: gql.GraphQLString, description: "The commit author's signature", + resolve(commit) { + return db + ._query( + aql` + FOR stakeholder, edge + IN INBOUND ${commit} ${commitsToStakeholders} + return stakeholder.gitSignature + ` + ) + .toArray()[0]; + }, }, branch: { type: gql.GraphQLString, diff --git a/lib/indexers/vcs/GitIndexer.js b/lib/indexers/vcs/GitIndexer.js index 07655ddd..9fa384be 100755 --- a/lib/indexers/vcs/GitIndexer.js +++ b/lib/indexers/vcs/GitIndexer.js @@ -112,15 +112,14 @@ class GitIndexer { } log('Processing', this.counter.commits.total, 'commits'); + //persist commits + //also creates (and connects) stakeholder objects from commit signature for (const commit of commits) { await processCommit .bind(this)(commit, currentBranch) .catch({ stop: true }, () => null); } - //create stakeholder objects - await createStakeholders(); - //create branch-file connections //in the process, check which files have been renamed and store these in the branch-file-file connection await createBranchFileConnections(this.repo); @@ -451,14 +450,6 @@ function adjustStatistic(key, list, newElements) { return emitCount; } -async function createStakeholders() { - const commitsDAO = await Commit.findAll(); - const signatures = _.uniq(commitsDAO.map((c) => c.data.signature)); - for (const stakeholder of signatures) { - await Stakeholder.ensureByGitSignature(stakeholder); - } -} - async function createBranchFileConnections(repo) { //find all branch file connections so we can skip them const existingBranchFileConnections = await BranchFileConnection.findAll(); diff --git a/lib/models/Commit.js b/lib/models/Commit.js index d01b196c..b50f77c4 100644 --- a/lib/models/Commit.js +++ b/lib/models/Commit.js @@ -46,16 +46,16 @@ Commit.persist = async function (repo, nCommit, urlProvider) { } // create new commit and link it to its parent commits + const authorSignature = nCommit.commit.author.name + ' <' + nCommit.commit.author.email + '>'; return Commit.create( { sha, - signature: nCommit.commit.author.name + ' <' + nCommit.commit.author.email + '>', date: new Date(nCommit.commit.author.timestamp * 1000), message: nCommit.commit.message, webUrl: urlProvider ? urlProvider.getCommitUrl(sha) : '', branch: nCommit.commit.branch, history: history, - parents: parents, + parents: parents, //TODO can be removed since commit is connected to parents through the commits-commits connection stats: { additions: 0, deletions: 0, @@ -63,6 +63,7 @@ Commit.persist = async function (repo, nCommit, urlProvider) { }, { isNew: true } ).then(function (commit) { + //connect commit to parents Promise.resolve( parents.split(',').map((parentSha) => { if (parentSha === '') { @@ -71,7 +72,14 @@ Commit.persist = async function (repo, nCommit, urlProvider) { return Commit.findById(parentSha).then((parentCommit) => commit.connect(parentCommit)); }) ); - return commit; + + //connect commit to author + const Stakeholder = require('./Stakeholder.js'); + return Stakeholder.ensureByGitSignature(authorSignature).then((results) => { + const stakeholder = results[0]; + commit.connect(stakeholder); + return commit; + }); }); }); }); @@ -276,48 +284,4 @@ Commit.prototype.processTree = function (repo, nCommit, currentBranch, urlProvid ); }; -/** - * create a connection for each stakeholder and all of their commits - * - * @returns {*} - */ -Commit.deduceStakeholders = async function () { - const CommitStakeholderConnection = require('./CommitStakeholderConnection.js'); - const Stakeholder = require('./Stakeholder.js'); - - // walk through all commits - return Promise.resolve( - Commit.rawDb.query( - aql` - FOR commit IN ${Commit.collection} - LET stakeholders = (FOR stakeholder - IN - INBOUND commit ${CommitStakeholderConnection.collection} - RETURN stakeholder) - FILTER LENGTH(stakeholders) == 0 - COLLECT sig = commit.signature INTO commitsPerSignature = commit - RETURN { - "signature": sig, - "commits": commitsPerSignature - }` - ) - ) - .then((cursor) => cursor.all()) - .then((signatures) => - signatures.forEach((signature) => { - // try to get an already existing stakeholder with that signature - return Stakeholder.ensureByGitSignature(signature.signature).then((results) => { - const stakeholder = results[0]; - // walk over all commits with that signature - return signature.commits.map((rawCommit) => - Promise.resolve(rawCommit, (commit) => { - // assign the commit to the stakeholder - return Commit.parse(commit).connect(stakeholder); - }) - ); - }); - }) - ); -}; - module.exports = Commit; diff --git a/test/backend/commit.test.js b/test/backend/commit.test.js index 6256859e..99c40574 100644 --- a/test/backend/commit.test.js +++ b/test/backend/commit.test.js @@ -12,6 +12,7 @@ const File = require('../../lib/models/File'); const Language = require('../../lib/models/Language'); const Hunk = require('../../lib/models/Hunk'); const LanguageFileConnection = require('../../lib/models/LanguageFileConnection'); +const CommitStakeholderConnection = require('../../lib/models/CommitStakeholderConnection.js'); const config = require('../../lib/config.js').get(); const ctx = require('../../lib/context'); @@ -45,6 +46,7 @@ describe('commit', function () { await Hunk.ensureCollection(); await Language.ensureCollection(); await LanguageFileConnection.ensureCollection(); + await CommitStakeholderConnection.ensureCollection(); await fake.file(repo, 'test.js', testFile); await helpers.commit(repo, ['test.js'], bob, 'Commit1'); diff --git a/ui/src/database/localDB.js b/ui/src/database/localDB.js index 571a6811..ecf93f07 100644 --- a/ui/src/database/localDB.js +++ b/ui/src/database/localDB.js @@ -116,11 +116,11 @@ export default class LocalDB { } static getCommitData(commitSpan, significantSpan) { - return Commits.getCommitData(db, commitSpan, significantSpan); + return Commits.getCommitData(db, tripleStore, commitSpan, significantSpan); } static getCommitDataForSha(sha) { - return Commits.getCommitDataForSha(db, sha); + return Commits.getCommitDataForSha(db, tripleStore, sha); } static getBuildData(commitSpan, significantSpan) { @@ -128,11 +128,11 @@ export default class LocalDB { } static getIssueData(issueSpan, significantSpan) { - return Issues.getIssueData(db, issueSpan, significantSpan); + return Issues.getIssueData(db, tripleStore, issueSpan, significantSpan); } static getCommitsForIssue(iid) { - return Issues.getCommitsForIssue(db, iid); + return Issues.getCommitsForIssue(db, tripleStore, iid); } static getMergeRequestData(mergeRequestSpan, significantSpan) { @@ -168,7 +168,7 @@ export default class LocalDB { } static getCommitDataOwnershipRiver(commitSpan, significantSpan, granularity, interval, excludeMergeCommits) { - return Commits.getCommitDataOwnershipRiver(db, commitSpan, significantSpan, granularity, interval, excludeMergeCommits); + return Commits.getCommitDataOwnershipRiver(db, tripleStore, commitSpan, significantSpan, granularity, interval, excludeMergeCommits); } static getBuildDataOwnershipRiver(commitSpan, significantSpan, granularity, interval) { @@ -180,11 +180,11 @@ export default class LocalDB { } static getRelatedCommitDataOwnershipRiver(issue) { - return Commits.getRelatedCommitDataOwnershipRiver(db, issue); + return Commits.getRelatedCommitDataOwnershipRiver(db, tripleStore, issue); } static getCommitDateHistogram(granularity, dateField, since, until) { - return Commits.getCommitDateHistogram(db, granularity, dateField, since, until); + return Commits.getCommitDateHistogram(db, tripleStore, granularity, dateField, since, until); } static issueImpactQuery(iid, since, until) { diff --git a/ui/src/database/localDB/commits.js b/ui/src/database/localDB/commits.js index 6fda1802..2c71fc74 100644 --- a/ui/src/database/localDB/commits.js +++ b/ui/src/database/localDB/commits.js @@ -8,12 +8,37 @@ import moment from 'moment/moment'; PouchDB.plugin(PouchDBFind); PouchDB.plugin(PouchDBAdapterMemory); +//add stakeholder to commit +async function preprocessCommit(commit, database, commitStakeholderRelations) { + const relevantRelation = commitStakeholderRelations.filter((r) => r.to === commit._id)[0]; + if (!relevantRelation) { + console.log('Error in localDB: commit: no stakeholder found for commit ' + commit.sha); + return commit; + } + + const author = (await findID(database, relevantRelation.from)).docs[0]; + if (!author) { + console.log('Error in localDB: commit: no stakeholder found with ID ' + relevantRelation.from); + return commit; + } + + return _.assign(commit, { signature: author.gitSignature }); +} + // find all of given collection (example _id field for e.g. issues looks like 'issues/{issue_id}') function findAll(database, collection) { return database.find({ selector: { _id: { $regex: new RegExp(`^${collection}/.*`) } }, }); } +async function findAllCommits(database, relations) { + let commits = await database.find({ + selector: { _id: { $regex: new RegExp('^commits/.*') } }, + }); + const relevantRelations = (await findCommitStakeholderConnections(relations)).docs; + commits.docs = await Promise.all(commits.docs.map((c) => preprocessCommit(c, database, relevantRelations))); + return commits; +} function bulkGet(database, ids) { const idsObjects = ids.map((id) => { @@ -30,10 +55,15 @@ function findIssue(database, iid) { }); } -function findCommit(database, sha) { - return database.find({ +async function findCommit(database, relations, sha) { + let commit = await database.find({ selector: { _id: { $regex: new RegExp('^commits/.*') }, sha: { $eq: sha } }, }); + if (commit.docs && commit.docs[0]) { + const commitStakeholderConnections = (await findCommitStakeholderConnections(relations)).docs; + commit.docs[0] = await preprocessCommit(commit.docs[0], database, commitStakeholderConnections); + } + return commit; } function findID(database, id) { @@ -54,6 +84,12 @@ function findFileCommitConnections(relations) { }); } +function findCommitStakeholderConnections(relations) { + return relations.find({ + selector: { _id: { $regex: new RegExp('^commits-stakeholders/.*') } }, + }); +} + function findFileCommitStakeholderConnections(relations) { return relations.find({ selector: { _id: { $regex: new RegExp('^commits-files-stakeholders/.*') } }, @@ -61,12 +97,12 @@ function findFileCommitStakeholderConnections(relations) { } export default class Commits { - static getCommitData(db, commitSpan, significantSpan) { + static getCommitData(db, relations, commitSpan, significantSpan) { // return all commits, filtering according to parameters can be added in the future const first = new Date(significantSpan[0]).getTime(); const last = new Date(significantSpan[1]).getTime(); - return findAll(db, 'commits').then((res) => { + return findAllCommits(db, relations).then((res) => { res.docs = res.docs .filter((c) => new Date(c.date) >= first && new Date(c.date) <= last) .sort((a, b) => { @@ -77,8 +113,9 @@ export default class Commits { }); } - static getCommitDataForSha(db, sha) { - return findAll(db, 'commits').then((res) => { + static getCommitDataForSha(db, relations, sha) { + //TODO use findCommit here + return findAllCommits(db, relations).then((res) => { const result = res.docs.filter((c) => c.sha === sha)[0]; return result; }); @@ -88,7 +125,7 @@ export default class Commits { const first = new Date(significantSpan[0]).getTime(); const last = new Date(significantSpan[1]).getTime(); - return findAll(db, 'commits').then(async (res) => { + return findAllCommits(db, relations).then(async (res) => { const commits = res.docs .filter((c) => new Date(c.date) >= first && new Date(c.date) <= last) .sort((a, b) => { @@ -126,7 +163,7 @@ export default class Commits { const first = new Date(significantSpan[0]).getTime(); const last = new Date(significantSpan[1]).getTime(); - return findAll(db, 'commits').then(async (res) => { + return findAllCommits(db, relations).then(async (res) => { const commits = res.docs .filter((c) => new Date(c.date) >= first && new Date(c.date) <= last) .sort((a, b) => { @@ -229,7 +266,7 @@ export default class Commits { } static getOwnershipDataForCommit(db, relations, sha) { - return findCommit(db, sha).then(async (res) => { + return findCommit(db, relations, sha).then(async (res) => { const commit = res.docs[0]; const files = (await findAll(db, 'files')).docs; @@ -262,7 +299,7 @@ export default class Commits { } static getOwnershipDataForCommits(db, relations) { - return findAll(db, 'commits').then(async (res) => { + return findAllCommits(db, relations).then(async (res) => { const commits = res.docs; const files = (await findAll(db, 'files')).docs; @@ -294,7 +331,7 @@ export default class Commits { }); } - static getCommitDataOwnershipRiver(db, commitSpan, significantSpan, granularity, interval, excludeMergeCommits) { + static getCommitDataOwnershipRiver(db, relations, commitSpan, significantSpan, granularity, interval, excludeMergeCommits) { const statsByAuthor = {}; const totals = { @@ -366,7 +403,7 @@ export default class Commits { return data; } - return findAll(db, 'commits').then((res) => { + return findAllCommits(db, relations).then((res) => { let commits = res.docs .filter((c) => new Date(c.date) >= first && new Date(c.date) <= last) .sort((a, b) => { @@ -419,14 +456,14 @@ export default class Commits { }); } - static getRelatedCommitDataOwnershipRiver(db, issue) { + static getRelatedCommitDataOwnershipRiver(db, relations, issue) { if (issue !== null) { return findIssue(db, issue.iid).then(async (resIssue) => { const foundIssue = resIssue.docs[0]; foundIssue.commits = { count: 0, data: [] }; for (const mention of foundIssue.mentions) { if (mention.commit !== null) { - const commit = (await findCommit(db, mention.commit)).docs[0]; + const commit = (await findCommit(db, relations, mention.commit)).docs[0]; foundIssue.commits.count++; foundIssue.commits.data.push(commit); } @@ -438,7 +475,7 @@ export default class Commits { } } - static getCommitDateHistogram(db, granularity, dateField, since, until) { + static getCommitDateHistogram(db, relations, granularity, dateField, since, until) { function mapCommitToHistogram(histogram, commit, granularity) { const commitDate = new Date(commit.date); return histogram.map((commit) => { @@ -511,7 +548,7 @@ export default class Commits { } } - return findAll(db, 'commits').then((resCommits) => { + return findAllCommits(db, relations).then((resCommits) => { return findAll(db, 'issues').then((resIssues) => { return findAll(db, 'builds').then((resBuilds) => { const commitDateHistogram = []; diff --git a/ui/src/database/localDB/issues.js b/ui/src/database/localDB/issues.js index 8bfc33ba..a3ac0591 100644 --- a/ui/src/database/localDB/issues.js +++ b/ui/src/database/localDB/issues.js @@ -8,6 +8,23 @@ import _ from 'lodash'; PouchDB.plugin(PouchDBFind); PouchDB.plugin(PouchDBAdapterMemory); +//add stakeholder to commit +async function preprocessCommit(commit, database, commitStakeholderRelations) { + const relevantRelation = commitStakeholderRelations.filter((r) => r.to === commit._id)[0]; + if (!relevantRelation) { + console.log('Error in localDB: commit: no stakeholder found for commit ' + commit.sha); + return commit; + } + + const author = (await findID(database, relevantRelation.from)).docs[0]; + if (!author) { + console.log('Error in localDB: commit: no stakeholder found with ID ' + relevantRelation.from); + return commit; + } + + return _.assign(commit, { signature: author.gitSignature }); +} + // find all of given collection (example _id field for e.g. issues looks like 'issues/{issue_id}') function findAll(database, collection) { return database.find({ @@ -15,16 +32,30 @@ function findAll(database, collection) { }); } +async function findAllCommits(database, relations) { + let commits = await database.find({ + selector: { _id: { $regex: new RegExp('^commits/.*') } }, + }); + const relevantRelations = (await findCommitStakeholderConnections(relations)).docs; + commits.docs = await Promise.all(commits.docs.map((c) => preprocessCommit(c, database, relevantRelations))); + return commits; +} + function findIssue(database, iid) { return database.find({ selector: { _id: { $regex: new RegExp('^issues/.*') }, iid: { $eq: iid } }, }); } -function findCommit(database, sha) { - return database.find({ +async function findCommit(database, relations, sha) { + let commit = await database.find({ selector: { _id: { $regex: new RegExp('^commits/.*') }, sha: { $eq: sha } }, }); + if (commit.docs && commit.docs[0]) { + const commitStakeholderConnections = (await findCommitStakeholderConnections(relations)).docs; + commit.docs[0] = await preprocessCommit(commit.docs[0], database, commitStakeholderConnections); + } + return commit; } function findBuild(database, sha) { @@ -57,8 +88,14 @@ function findSpecificFileConnections(relations, commitID, fileId) { }); } +function findCommitStakeholderConnections(relations) { + return relations.find({ + selector: { _id: { $regex: new RegExp('^commits-stakeholders/.*') } }, + }); +} + export default class Issues { - static getIssueData(db, issueSpan, significantSpan) { + static getIssueData(db, relations, issueSpan, significantSpan) { // return all issues, filtering according to parameters can be added in the future return findAll(db, 'issues').then((res) => { res.docs = res.docs @@ -70,7 +107,7 @@ export default class Issues { }); } - static getCommitsForIssue(db, issueId) { + static getCommitsForIssue(db, relations, issueId) { let iid; if (typeof issueId === 'string') { iid = parseInt(issueId); @@ -80,7 +117,7 @@ export default class Issues { return findIssue(db, iid).then(async (resIssue) => { const issue = resIssue.docs[0]; - const allCommits = (await findAll(db, 'commits')).docs; + const allCommits = (await findAllCommits(db, relations)).docs; const result = []; if (issue.mentions === null || issue.mentions === undefined) { return result; @@ -172,7 +209,7 @@ export default class Issues { new Date(mentionedCommit.createdAt) >= new Date(since) && new Date(mentionedCommit.createdAt) <= new Date(until) ) { - const commit = (await findCommit(db, mentionedCommit.commit)).docs[0]; + const commit = (await findCommit(db, relations, mentionedCommit.commit)).docs[0]; const builds = (await findBuild(db, commit.sha)).docs; commit.files = { data: [] }; const fileConnections = (await findFileConnections(relations, commit.sha)).docs; @@ -214,7 +251,7 @@ export default class Issues { if (issue.mentions !== undefined) { for (const commitMention of issue.mentions) { if (commitMention.commit !== null) { - const resCommit = await findCommit(db, commitMention.commit); + const resCommit = await findCommit(db, relations, commitMention.commit); if (resCommit.docs.length > 0) { const commit = resCommit.docs[0]; const resFile = await findFile(db, file);