diff --git a/package.json b/package.json index d2a92eba..eb4ff120 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "mongoose": "^4.1.2", "morgan": "^1.7.0", "multiparty": "^4.1.1", + "octokat": "^0.4.18", "passport": "~0.3.0", "passport-facebook": "^2.0.0", "passport-google-oauth": "^1.0.0", diff --git a/server/api/commit/commit.controller.js b/server/api/commit/commit.controller.js index 3c8f7afe..2ae7cb12 100644 --- a/server/api/commit/commit.controller.js +++ b/server/api/commit/commit.controller.js @@ -58,7 +58,7 @@ exports.destroy = function(req, res) { // Show a list of a projects Commits // Get a single commit exports.showProjectCommits = function(req, res) { - Commit.findById(req.params.projectId, function (err, commits) { + Commit.find({project: req.params.projectId}).populate('project').exec(function (err, commits) { if(err) { return handleError(res, err); } if(!commits) { return res.send(404); } return res.json(commits); @@ -68,16 +68,9 @@ exports.showProjectCommits = function(req, res) { // Get a list of a user Commits exports.showUserCommits = function(req, res) { var prevDays = new Date(); - if (req.params.timeperiod){ - prevDays.setDate(prevDays.getDate()-Number(req.params.timeperiod)); - } - else{ - prevDays.setDate(prevDays.getDate()-14); - } Commit.find() .where('author.login').equals(String(req.params.githubProfile)) - .where('date').gt(prevDays) .exec(function(err, commits){ if(err) { return handleError(res, err); } if(!commits) { return res.json([]); } diff --git a/server/api/commit/commit.model.js b/server/api/commit/commit.model.js index 563f0670..98526b2d 100644 --- a/server/api/commit/commit.model.js +++ b/server/api/commit/commit.model.js @@ -2,6 +2,7 @@ var mongoose = require('mongoose'), Schema = mongoose.Schema; +var Project = require('../project/project.model'); var CommitSchema = new Schema({ url: String, @@ -10,14 +11,18 @@ var CommitSchema = new Schema({ unique: true, index: true, }, - userId: { type: String, index: true}, author: { login: {type: String, lowercase: true}, id: {type: Number} }, - branch: String, + commentCount: Number, message: String, date: Date, + project: { + type : Schema.Types.ObjectId, + ref: 'Project', + index: true + }, },{ timestamps: true}); module.exports = mongoose.model('Commit', CommitSchema); diff --git a/server/api/project/project.model.js b/server/api/project/project.model.js index 45b44a93..865f41ad 100644 --- a/server/api/project/project.model.js +++ b/server/api/project/project.model.js @@ -2,6 +2,7 @@ var mongoose = require('mongoose'), Schema = mongoose.Schema; +var Commit = require('../commit/commit.model'); var ProjectSchema = new Schema({ name: { @@ -42,7 +43,21 @@ var ProjectSchema = new Schema({ default: false, index: true }, - tech: [String] + tech: [String], + commits: [{ + type : Schema.Types.ObjectId, + ref: 'Commit', + index: true + }], },{ timestamps: true}); +/* + @returns: + */ +ProjectSchema + .virtual('fullRepoPath') + .get(function(){ + return (this.githubUsername + "/" + this.githubProjectName) ; + }); + module.exports = mongoose.model('Project', ProjectSchema); diff --git a/server/api/user/user.model.js b/server/api/user/user.model.js index 1e8bec6f..ae26114c 100644 --- a/server/api/user/user.model.js +++ b/server/api/user/user.model.js @@ -38,6 +38,7 @@ var UserSchema = new Schema({ index: true }], // project id + bio: String, password: { diff --git a/server/config/environment/development.js b/server/config/environment/development.js index 525aeeac..875b9d60 100644 --- a/server/config/environment/development.js +++ b/server/config/environment/development.js @@ -17,5 +17,8 @@ module.exports = { // For testing verification // attendanceVerificationRatio: 1, - seedDB: false + seedDB: false, + // Github token for worker, local.env.js kept returning undefined but it worked once I added it here... + GITHUB_WORKER_TOKEN: 'YOUR_KEY', + }; diff --git a/server/config/local.env.sample.js b/server/config/local.env.sample.js index 7e502417..da88c903 100644 --- a/server/config/local.env.sample.js +++ b/server/config/local.env.sample.js @@ -17,6 +17,8 @@ module.exports = { SERVER_ADDRESS: 'http://localhost:9000', SENDGRID_API_KEY: 'YOUR_KEY', + GITHUB_WORKER_TOKEN: 'YOUR_KEY', + // Control debug level for modules using visionmedia/debug DEBUG: '' diff --git a/server/config/seed.js b/server/config/seed.js index 6ed30533..0500fbac 100644 --- a/server/config/seed.js +++ b/server/config/seed.js @@ -99,6 +99,7 @@ var seed = function() { if (!module.parent) { if (args.length == 0) { seed().then(function(){ + db.disconnect() }); } diff --git a/server/workers/github.js b/server/workers/github.js new file mode 100644 index 00000000..049ec6e0 --- /dev/null +++ b/server/workers/github.js @@ -0,0 +1,191 @@ +'use strict'; + +if (!module.parent) { + // Set default node environment to development + process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + + var mongoose = require('mongoose'); + var config = require('../config/environment'); + var args = process.argv.slice(2); + + // Connect to database + var db = mongoose.connect(config.mongo.uri, config.mongo.options); +} +var Commit = require('../api/commit/commit.model'); +var Project = require('../api/project/project.model'); + +var Octokat = require('octokat'); + +var Promise = require("bluebird"); + +mongoose.Promise = require('bluebird'); + +/* + The github token currently needs to be defined in development.js env. + Since, this is a worker that is currently run individually, it does not have app.js loaded beforehand. + Meaning, magic env stuff isn't automagically setup. + @TODO: bring back the magic + */ +var gtoken = config.GITHUB_WORKER_TOKEN; +if (gtoken == "YOUR_KEY") { + console.error('ERROR: PLEASE SET YOUR GITHUB_WORKER_TOKEN IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); + throw new Error('ERROR: PLEASE SET YOUR GITHUB_WORKER_TOKEN IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); +} + +// setup & initilize our github API library +var octo = new Octokat({ + token: gtoken +}); + +/* + @TODO: add support for filtering commits by date/author/etc.. + ex: only get commits newer than date x, or only get commits older than date y + */ +function fetchCommitsFromProject(owner, repository, config) { + return fetchAll(octo.repos(owner, repository).commits.fetch, config); +} + +function fetchAll(fn, args) { + let acc = []; + if (!args) { + args = {per_page: 100}; + } + let p = new Promise((resolve, reject) => { + fn(args).then((val) => { + acc = acc.concat(val); + if (val.nextPage) { + fetchAll(val.nextPage).then((val2) => { + acc = acc.concat(val2); + resolve(acc); + }, reject); + } else { + resolve(acc); + } + }, reject); + }); + return p; +} + +var saveCommit = function (commitData, project) { + var newCommit = new Commit(commitData); + /* + establish the ownership relationship; i.e. what project does this commit belong to? + @TODO: setup the inverse relationship(?); i.e. what commits does the project own? + */ + newCommit.project = project; + /* + manually extract/configure the data to match our Commit schema. + */ + newCommit.message = commitData.commit.message; + newCommit.date = commitData.commit.author.date; + newCommit.commentCount = commitData.commit.comment_count; + // Save the new commit model to the db. This is an async operation, so it will be turned into a promise. + return newCommit.save(); +}; + +/* + @promiseObject: JS object containing 2 fields. + commits: promise array of fetched commits from github + project: instance of project model that we are fetching the commits for. + */ +var saveCommits = saveCommitsFn; +function saveCommitsFn(promiseObject) { + console.dir(promiseObject); + console.log("entering..."); + // extract the data from the promiseObject + var commits = promiseObject.commits; + var project = promiseObject.project; + // array of Commit.save() operations; to be turned into an array of Promises. + var commitPromises = []; + commits.forEach(function (commit) { + commitPromises.push(saveCommit(commit, project)); + }); + // return an array of promises, where each promise is a promise to save the commit model to the DB. + return Promise.all(commitPromises); +} + +exports.getCommitsForProjectSinceDate = function (project, date) { + date = new Date(Date.parse(date)) || new Date(); + var config = { + since: date.toISOString() + }; + return processProject(project, config); +}; + +exports.getCommitsForProjectUntilDate = function (project, date) { + date = new Date(Date.parse(date)) || new Date(); + var config = { + ungil: date.toISOString() + }; + return processProject(project, config); +}; + +exports.getCommitsForProject = function (project) { + return processProject(project); +}; + +/** + * + * @param project: instance of Project model + * @param config: + */ +var fetchCommits = function (project, config) { + // extract the github username & github repository name to fetch the commits from. + var owner = project.githubUsername; + var repository = project.githubProjectName; + + // Config is an optional arguement, so if it isn't pased init it ourselves + config = config || {}; + + // always set the results per page limit to the maximum, 100, to reduce github API calls + config["per_page"] = 100; + + /* + Needed to pass through project model; to saveCommits; but javascript. + So instead I just tacked it onto an object. + Promise.props is essentially the same as using Promise.all([]), + Except it returns an object with arbitrary fields that can be a promise or arbitrary data. + with an added variable, @project. + Probably a better way to do this, but I currently can't think of any because javascript. + */ + + return Promise.props({ + commits: fetchCommitsFromProject(owner, repository, config), + project: project, + }); +}; + +/* + For now just a hard coded example/P.O.C using the github repo rcos/observatory3 + Planning to refactor this "soon(tm)" + */ + +function processProject(project, config) { + /* + flow: + 1. fetch commits from github for @project + 2. save fetched commits to db + 3. disconnect from DB + */ + return fetchCommits(project, config).then(saveCommits); +} + +function processAllProjects() { + var start = Promise.resolve(); + return Promise.map(Project.find({}).exec(), function (proj) { + start = start.then(function () { + return fetchCommits(proj); + }); + return start; + }).map(saveCommitsFn).then(function (results) { + console.dir(results); + console.dir(results.length); + db.disconnect(); + }); +} + +if (!module.parent) { + if (args.length == 0) { + processAllProjects(); + } +} diff --git a/server/workers/github.py b/server/workers/github.py deleted file mode 100644 index f22ead0f..00000000 --- a/server/workers/github.py +++ /dev/null @@ -1,250 +0,0 @@ -import requests -import os -from datetime import datetime -import dateutil.parser - -from pymongo import MongoClient -from bson.objectid import ObjectId - -MONGO_URL = 'mongodb://localhost/' - -HOST = 'https://api.github.com' -PAYLOAD = {'client_id': os.environ['GITHUBCLIENTID'], - 'client_secret': os.environ['GITHUBCLIENTSECRET'], - 'per_page': 100} -headers = {'content-type': 'application/json'} - -client = MongoClient(MONGO_URL) -db = client['observatory3-dev'] - -def parseCommit(commitData): - '''Parses a commit from the Github URL''' - commit = {} - - commit['url'] = commitData['url'] - commit['sha'] = commitData['sha'] - commit['message'] = commitData['commit']['message'] - commit['author'] = {} - if commitData['author']: - commit['author']['login'] = commitData['author']['login'] - commit['author']['id'] = commitData['author']['id'] - commit['date'] = dateutil.parser.parse(commitData['commit']['committer']['date']) - user = db.users.find_one({'github.login': commit['author']['login']}) - - if user: - userId = user['_id'] - commit['userId'] = str(ObjectId(user['_id'])) - - return commit - -def getCommits(userName, repositoryName, since=None): - if since: - # Add the last checked date to the parameters if it is available. - PAYLOAD['since'] = since - else: - PAYLOAD['since'] = None - # form the initial API URL - path = HOST + '/repos/%s/%s/commits'%(userName, repositoryName) - Commits = db.commits - commits = [] - - #While there are still pages of new commits keep getting commits - while True: - r = requests.get(path, params=PAYLOAD, headers=headers) - commitsData = r.json() - - for com in commitsData: - commit = parseCommit(com) - - if not len(list(Commits.find({'sha': commit['sha']}))): - Commits.insert(commit, {'upsert':True}) - commits.append(commit) - try: - links = r.headers['link'] - links = links.split(',') - for link in links: - link = link.split(';') - if 'next' in link[1]: - newPath = link[0][1:len(link[0])-1] - if newPath == path: - break - else: - path = newPath - except: - break - - print "Found %d new commit(s) for project %s %s since %s"%( - len(commits), userName, repositoryName, str(since)) - return commits - -def getUserEvents(user): - Commits = db.commits - path = HOST + '/users/%s/events/public'%(user['github']['login']) - events = [] - r = requests.get(path, params=PAYLOAD, headers=headers) - eventData = r.json() - - for event in eventData: - print event - try: - if event['type'] == 'PushEvent': - # User pushed code - for com in event['payload']['commits']: - commit = {} - - dbCommit = db.commits.find_one({'sha': com['sha']}) - - if not dbCommit: - # Ensure commit isn't already in our database before making a new one - r = requests.get(com['url'], params=PAYLOAD, headers=headers) - data = r.json() - if 'message' in data: - # Message means that there was an error finding the commit - pass - else: - githubCommit = parseCommit(data) - - Commits.insert(githubCommit,{'upsert':True}) - elif event['type'] == 'IssueCommentEvent': - # User commented on an issue - newEvent = {} - newEvent['type'] = 'IssueCommentEvent' - newEvent['action'] = event['payload']['action'] - newEvent['message'] = event['payload']['comment']['body'] - newEvent['url'] = event['payload']['comment']['html_url'] - newEvent['date'] = dateutil.parser.parse(event['payload']['comment']['created_at']) - - events.append(newEvent) - elif event['type'] == 'PullRequestEvent': - # Events that are pull requests - newEvent = {} - newEvent['type'] = 'PullRequestEvent' - newEvent['action'] = event['payload']['action'] - newEvent['message'] = event['payload']['pull_request']['title'] - newEvent['url'] = event['payload']['pull_request']['_links']['html']['href'] - # Changes behavior based on whether the pull request was opened or closed - if newEvent['action'] == 'closed': - newEvent['date'] = dateutil.parser.parse(event['payload']['pull_request']['closed_at']) - elif newEvent['action'] == 'opened': - newEvent['date'] = dateutil.parser.parse(event['payload']['pull_request']['created_at']) - - events.append(newEvent) - elif event['type'] == 'IssuesEvent': - # IssuesEvent processing - newEvent = {} - newEvent['type'] = 'PullRequestEvent' - newEvent['action'] = event['payload']['action'] - newEvent['message'] = event['payload']['issue']['body'] - newEvent['url'] = event['payload']['issue']['html_url'] - # Changes behavior based on whether the issue was opened or closed - if newEvent['action'] == 'closed': - newEvent['date'] = dateutil.parser.parse(event['payload']['issue']['closed_at']) - elif newEvent['action'] == 'opened': - newEvent['date'] = dateutil.parser.parse(event['payload']['issue']['created_at']) - - events.append(newEvent) - elif event['type'] == 'CreateEvent': - # CreateEvent ignored - pass - elif event['type'] == 'WatchEvent': - # WatchEvent ignored - pass - elif event['type'] == 'ForkEvent': - # ForkEvent ignored - pass - elif event['type'] == 'CommitCommentEvent': - # CommitCommentEvent ignored - pass - else: - print event['type'] - - except: - pass - for event in events: - db.users.update({'_id': user['_id']}, {'$addToSet':{'github.events': event}}, multi=False) - -def createUser(name, username): - user = { - 'name': name, - 'active': True, - 'github': { - 'events': [], - 'login': username, - } - } - Users = db.users - user_id = db.users.insert(user) - print user_id - return user_id - -def getProjectCollaborators(owner, projectName): - path = HOST + '/repos/%s/%s/commits'%(owner, projectName) - - users = {} - count = 0 - while True: - r = requests.get(path, params=PAYLOAD, headers=headers) - commitsData = r.json() - - for comData in commitsData: - count += 1 - if comData['author']: - - login = comData['author']['login'] - name = comData['commit']['author']['name'] - users[login] = name - # name = comData['commit']['author'] - # login = '' - # if comData and 'committer' in comData.keys() and 'login' in comData['committer'].keys(): - # login = comData['committer']['login'] - # print login, name - # users.append((name, login)) - try: - links = r.headers['link'] - links = links.split(',') - for link in links: - link = link.split(';') - if 'next' in link[1]: - newPath = link[0][1:len(link[0])-1] - if newPath == path: - break - else: - path = newPath - except: - break - users = [(users[key], key) for key in users.keys()] - print list(users) - print len(list(users)) - for name, username in users: - print name + ',' + username - print count - -def getUserGravatar(user): #TODO: gravatar is not imported from github - path = HOST + '/users/%s'%(user['github']['login']) - events = [] - r = requests.get(path, params=PAYLOAD, headers=headers) - userData = r.json() - avatar_url = userData['avatar_url'] - db.users.update({'_id': user['_id']}, {'$set': {'avatar': avatar_url}}) - -def updateUser(user): - getUserEvents(user) - getUserGravatar(user) - -if __name__ == '__main__': - users = db.users.find({'github.login': {'$exists': True}}) - for user in users: - print "Fetching info for user: " + str(user['name']) - updateUser(user) - - for project in db.projects.find({}): - if project['repositoryType'] == 'github': - userName = project['githubUsername'] - projectName = project['githubProjectName'] - - if 'lastChecked' in project: - since = project['lastChecked'] - getCommits(userName, projectName, since) - else: - getCommits(userName, projectName) - db.projects.update({'_id': project['_id']},{'$set':{'lastChecked': datetime.now()}})