From 13f9dd993697869738878cb08683829d5152116f Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 18 Oct 2016 17:55:05 -0400 Subject: [PATCH 01/27] Started work on pulling commit data for project repositories --- server/workers/github.js | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 server/workers/github.js diff --git a/server/workers/github.js b/server/workers/github.js new file mode 100644 index 00000000..1a7c9510 --- /dev/null +++ b/server/workers/github.js @@ -0,0 +1,46 @@ +var octo = require('octonode'); +var token = "459827cd6419a8cc6974258c9c6864e64bab1605"; +var octoclient = octo.client(token); + +// Concurrency Variables +var maxConcurrentThreads = 10; +var runningThreads = 0; +var queuedThreads = []; + +function runThread(func){ + // If we've exceeded the max number of threads, delay the call + if (queuedThreads.length > 0 || runningThreads >= maxConcurrentThreads){ + queuedThreads.push(func); + }else{ + func(); + } +} + + +module.exports.getCommitStrings = function(githubRepo, callback){ + runThread(function(){ + /* + githubRepo needs to be in the format / + example: rcos/Observatory3 + */ + var ghrepo = octoclient.repo(githubRepo); + var commits = ghrepo.commits(function(err, data, headers) { + //@TODO: add logic to create a commit model & save it to DB + }); + // @TODO: remove/cleanup; just using this as a reference for now... + // var pushEvents = ghuser.events(['PushEvent'], function(ignore, pushEvents){ + // if (!pushEvents){ + // return callback([]); + // } + // var strings = []; + // for (var i = 0; i < pushEvents.length;i++){ + // var payload = pushEvents[i].payload; + // var commits = payload.commits; + // for (var u = 0; u < commits.length; u++){ + // strings.push(commits[u].message); + // } + // } + // callback(strings); + // }); + // }); +}; From 8d87fc732af677dd1e6d846fe9ff7de92b0a176d Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 18 Oct 2016 17:56:54 -0400 Subject: [PATCH 02/27] refactored function name to make more sense --- server/workers/github.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/workers/github.js b/server/workers/github.js index 1a7c9510..6cb9c9d2 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -17,7 +17,7 @@ function runThread(func){ } -module.exports.getCommitStrings = function(githubRepo, callback){ +module.exports.getCommitsForRepository = function(githubRepo, callback){ runThread(function(){ /* githubRepo needs to be in the format / From 4203d62206591ff1df57c475ae84176a8ad54be8 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 18 Oct 2016 18:02:17 -0400 Subject: [PATCH 03/27] Removed unused github.py worker --- server/workers/github.py | 250 --------------------------------------- 1 file changed, 250 deletions(-) delete mode 100644 server/workers/github.py diff --git a/server/workers/github.py b/server/workers/github.py deleted file mode 100644 index f22ead0f..00000000 --- a/server/workers/github.py +++ /dev/null @@ -1,250 +0,0 @@ -import requests -import os -from datetime import datetime -import dateutil.parser - -from pymongo import MongoClient -from bson.objectid import ObjectId - -MONGO_URL = 'mongodb://localhost/' - -HOST = 'https://api.github.com' -PAYLOAD = {'client_id': os.environ['GITHUBCLIENTID'], - 'client_secret': os.environ['GITHUBCLIENTSECRET'], - 'per_page': 100} -headers = {'content-type': 'application/json'} - -client = MongoClient(MONGO_URL) -db = client['observatory3-dev'] - -def parseCommit(commitData): - '''Parses a commit from the Github URL''' - commit = {} - - commit['url'] = commitData['url'] - commit['sha'] = commitData['sha'] - commit['message'] = commitData['commit']['message'] - commit['author'] = {} - if commitData['author']: - commit['author']['login'] = commitData['author']['login'] - commit['author']['id'] = commitData['author']['id'] - commit['date'] = dateutil.parser.parse(commitData['commit']['committer']['date']) - user = db.users.find_one({'github.login': commit['author']['login']}) - - if user: - userId = user['_id'] - commit['userId'] = str(ObjectId(user['_id'])) - - return commit - -def getCommits(userName, repositoryName, since=None): - if since: - # Add the last checked date to the parameters if it is available. - PAYLOAD['since'] = since - else: - PAYLOAD['since'] = None - # form the initial API URL - path = HOST + '/repos/%s/%s/commits'%(userName, repositoryName) - Commits = db.commits - commits = [] - - #While there are still pages of new commits keep getting commits - while True: - r = requests.get(path, params=PAYLOAD, headers=headers) - commitsData = r.json() - - for com in commitsData: - commit = parseCommit(com) - - if not len(list(Commits.find({'sha': commit['sha']}))): - Commits.insert(commit, {'upsert':True}) - commits.append(commit) - try: - links = r.headers['link'] - links = links.split(',') - for link in links: - link = link.split(';') - if 'next' in link[1]: - newPath = link[0][1:len(link[0])-1] - if newPath == path: - break - else: - path = newPath - except: - break - - print "Found %d new commit(s) for project %s %s since %s"%( - len(commits), userName, repositoryName, str(since)) - return commits - -def getUserEvents(user): - Commits = db.commits - path = HOST + '/users/%s/events/public'%(user['github']['login']) - events = [] - r = requests.get(path, params=PAYLOAD, headers=headers) - eventData = r.json() - - for event in eventData: - print event - try: - if event['type'] == 'PushEvent': - # User pushed code - for com in event['payload']['commits']: - commit = {} - - dbCommit = db.commits.find_one({'sha': com['sha']}) - - if not dbCommit: - # Ensure commit isn't already in our database before making a new one - r = requests.get(com['url'], params=PAYLOAD, headers=headers) - data = r.json() - if 'message' in data: - # Message means that there was an error finding the commit - pass - else: - githubCommit = parseCommit(data) - - Commits.insert(githubCommit,{'upsert':True}) - elif event['type'] == 'IssueCommentEvent': - # User commented on an issue - newEvent = {} - newEvent['type'] = 'IssueCommentEvent' - newEvent['action'] = event['payload']['action'] - newEvent['message'] = event['payload']['comment']['body'] - newEvent['url'] = event['payload']['comment']['html_url'] - newEvent['date'] = dateutil.parser.parse(event['payload']['comment']['created_at']) - - events.append(newEvent) - elif event['type'] == 'PullRequestEvent': - # Events that are pull requests - newEvent = {} - newEvent['type'] = 'PullRequestEvent' - newEvent['action'] = event['payload']['action'] - newEvent['message'] = event['payload']['pull_request']['title'] - newEvent['url'] = event['payload']['pull_request']['_links']['html']['href'] - # Changes behavior based on whether the pull request was opened or closed - if newEvent['action'] == 'closed': - newEvent['date'] = dateutil.parser.parse(event['payload']['pull_request']['closed_at']) - elif newEvent['action'] == 'opened': - newEvent['date'] = dateutil.parser.parse(event['payload']['pull_request']['created_at']) - - events.append(newEvent) - elif event['type'] == 'IssuesEvent': - # IssuesEvent processing - newEvent = {} - newEvent['type'] = 'PullRequestEvent' - newEvent['action'] = event['payload']['action'] - newEvent['message'] = event['payload']['issue']['body'] - newEvent['url'] = event['payload']['issue']['html_url'] - # Changes behavior based on whether the issue was opened or closed - if newEvent['action'] == 'closed': - newEvent['date'] = dateutil.parser.parse(event['payload']['issue']['closed_at']) - elif newEvent['action'] == 'opened': - newEvent['date'] = dateutil.parser.parse(event['payload']['issue']['created_at']) - - events.append(newEvent) - elif event['type'] == 'CreateEvent': - # CreateEvent ignored - pass - elif event['type'] == 'WatchEvent': - # WatchEvent ignored - pass - elif event['type'] == 'ForkEvent': - # ForkEvent ignored - pass - elif event['type'] == 'CommitCommentEvent': - # CommitCommentEvent ignored - pass - else: - print event['type'] - - except: - pass - for event in events: - db.users.update({'_id': user['_id']}, {'$addToSet':{'github.events': event}}, multi=False) - -def createUser(name, username): - user = { - 'name': name, - 'active': True, - 'github': { - 'events': [], - 'login': username, - } - } - Users = db.users - user_id = db.users.insert(user) - print user_id - return user_id - -def getProjectCollaborators(owner, projectName): - path = HOST + '/repos/%s/%s/commits'%(owner, projectName) - - users = {} - count = 0 - while True: - r = requests.get(path, params=PAYLOAD, headers=headers) - commitsData = r.json() - - for comData in commitsData: - count += 1 - if comData['author']: - - login = comData['author']['login'] - name = comData['commit']['author']['name'] - users[login] = name - # name = comData['commit']['author'] - # login = '' - # if comData and 'committer' in comData.keys() and 'login' in comData['committer'].keys(): - # login = comData['committer']['login'] - # print login, name - # users.append((name, login)) - try: - links = r.headers['link'] - links = links.split(',') - for link in links: - link = link.split(';') - if 'next' in link[1]: - newPath = link[0][1:len(link[0])-1] - if newPath == path: - break - else: - path = newPath - except: - break - users = [(users[key], key) for key in users.keys()] - print list(users) - print len(list(users)) - for name, username in users: - print name + ',' + username - print count - -def getUserGravatar(user): #TODO: gravatar is not imported from github - path = HOST + '/users/%s'%(user['github']['login']) - events = [] - r = requests.get(path, params=PAYLOAD, headers=headers) - userData = r.json() - avatar_url = userData['avatar_url'] - db.users.update({'_id': user['_id']}, {'$set': {'avatar': avatar_url}}) - -def updateUser(user): - getUserEvents(user) - getUserGravatar(user) - -if __name__ == '__main__': - users = db.users.find({'github.login': {'$exists': True}}) - for user in users: - print "Fetching info for user: " + str(user['name']) - updateUser(user) - - for project in db.projects.find({}): - if project['repositoryType'] == 'github': - userName = project['githubUsername'] - projectName = project['githubProjectName'] - - if 'lastChecked' in project: - since = project['lastChecked'] - getCommits(userName, projectName, since) - else: - getCommits(userName, projectName) - db.projects.update({'_id': project['_id']},{'$set':{'lastChecked': datetime.now()}}) From cf053060c6ee3358a23cc22bf683ed1691dd9a16 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 18 Oct 2016 18:09:46 -0400 Subject: [PATCH 04/27] removed hardcoded github token for security purposes --- server/workers/github.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/workers/github.js b/server/workers/github.js index 6cb9c9d2..16a47317 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -1,5 +1,8 @@ var octo = require('octonode'); -var token = "459827cd6419a8cc6974258c9c6864e64bab1605"; +/* + @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE +*/ +var token = "INSERT GITHUB API TOKEN HERE"; var octoclient = octo.client(token); // Concurrency Variables @@ -26,6 +29,7 @@ module.exports.getCommitsForRepository = function(githubRepo, callback){ var ghrepo = octoclient.repo(githubRepo); var commits = ghrepo.commits(function(err, data, headers) { //@TODO: add logic to create a commit model & save it to DB + }); // @TODO: remove/cleanup; just using this as a reference for now... // var pushEvents = ghuser.events(['PushEvent'], function(ignore, pushEvents){ From 391b739c468190e1f509bbb6e645f5e9ccb5f04a Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 18 Oct 2016 21:01:56 -0400 Subject: [PATCH 05/27] P.O.C/test example progress...still heavily debugging... --- server/config/seed.js | 13 +++++++++++-- server/workers/github.js | 20 ++++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/server/config/seed.js b/server/config/seed.js index 1c649692..189fe78a 100644 --- a/server/config/seed.js +++ b/server/config/seed.js @@ -22,7 +22,9 @@ var User = require('../api/user/user.model'); var Project = require('../api/project/project.model'); var Post = require('../api/post/post.model'); var Smallgroup = require('../api/smallgroup/smallgroup.model'); -var ClassYear = require('../api/classyear/classyear.model') +var ClassYear = require('../api/classyear/classyear.model'); +var Commit = require('../api/commit/commit.model'); +var GithubWorker = require('../workers/github'); // Seed the database with the original sample data @@ -32,7 +34,11 @@ var seed = function() { var projects = require('./seed/projects.json'); var posts = require('./seed/posts.json'); var smallgroups = require('./seed/smallgroups.json'); - +var d = GithubWorker.getCommitsForRepository(projects[0].githubUsername, projects[0].githubProjectName, function(data) { + console.error('1',data) + return data; +}); +console.error('d',d); var user = User.remove({}).exec() .then(function() { return User.create(users); @@ -48,7 +54,10 @@ var seed = function() { .then(function() { console.log('finished populating projects') }); + var commit = Commit.remove({}).exec() + .then(function(){ + }) var post = Post.remove({}).exec() .then(function(){ return Post.create(posts) diff --git a/server/workers/github.js b/server/workers/github.js index 16a47317..4983c8f9 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -2,7 +2,7 @@ var octo = require('octonode'); /* @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE */ -var token = "INSERT GITHUB API TOKEN HERE"; +var token = "26a687eb6476756cedf8951316c9542f3011378c"; var octoclient = octo.client(token); // Concurrency Variables @@ -20,17 +20,25 @@ function runThread(func){ } -module.exports.getCommitsForRepository = function(githubRepo, callback){ +module.exports.getCommitsForRepository = function(repoOwner, repoName, callback){ runThread(function(){ /* - githubRepo needs to be in the format / - example: rcos/Observatory3 + ex: github.com// + repoOwner: github username for owner of github repo + repoName: github repo name + + example: + repoOwner: rcos + repoName: Observatory3 */ + var githubRepo = repoOwner + "/" + repoName; var ghrepo = octoclient.repo(githubRepo); var commits = ghrepo.commits(function(err, data, headers) { //@TODO: add logic to create a commit model & save it to DB - + callback(data); }); + }); +}; // @TODO: remove/cleanup; just using this as a reference for now... // var pushEvents = ghuser.events(['PushEvent'], function(ignore, pushEvents){ // if (!pushEvents){ @@ -47,4 +55,4 @@ module.exports.getCommitsForRepository = function(githubRepo, callback){ // callback(strings); // }); // }); -}; +// }; From 3844aef2011990f97e4d46ad7aef228ace0285fb Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 18 Oct 2016 21:14:41 -0400 Subject: [PATCH 06/27] more debugging...just commiting to try to keep track of this mess... --- server/api/commit/commit.controller.js | 2 ++ server/config/seed.js | 37 ++++++++++++++++++-------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/server/api/commit/commit.controller.js b/server/api/commit/commit.controller.js index 3c8f7afe..f3577431 100644 --- a/server/api/commit/commit.controller.js +++ b/server/api/commit/commit.controller.js @@ -22,7 +22,9 @@ exports.show = function(req, res) { // Creates a new commit in the DB. exports.create = function(req, res) { + console.error('111CREATING??'); Commit.create(req.body, function(err, commit) { + console.error('CREATING??'); if(err) { return handleError(res, err); } return res.json(201, commit); }); diff --git a/server/config/seed.js b/server/config/seed.js index 189fe78a..e3096641 100644 --- a/server/config/seed.js +++ b/server/config/seed.js @@ -34,11 +34,9 @@ var seed = function() { var projects = require('./seed/projects.json'); var posts = require('./seed/posts.json'); var smallgroups = require('./seed/smallgroups.json'); -var d = GithubWorker.getCommitsForRepository(projects[0].githubUsername, projects[0].githubProjectName, function(data) { - console.error('1',data) - return data; -}); -console.error('d',d); +// var d = +// console.error('d',d); +var promises = []; var user = User.remove({}).exec() .then(function() { return User.create(users); @@ -46,7 +44,7 @@ console.error('d',d); .then(function(){ console.log('finished populating users'); }); - + promises.push(user); var project = Project.remove({}).exec() .then(function(){ return Project.create(projects) @@ -54,10 +52,25 @@ console.error('d',d); .then(function() { console.log('finished populating projects') }); - var commit = Commit.remove({}).exec() + promises.push(project); + Commit.remove({}).exec() .then(function(){ - - }) + // var allCommitPromises = []; + for (var i = 0;i Date: Tue, 25 Oct 2016 11:23:26 -0400 Subject: [PATCH 07/27] Revert "more debugging...just commiting to try to keep track of this mess..." This reverts commit 3844aef2011990f97e4d46ad7aef228ace0285fb. --- server/api/commit/commit.controller.js | 2 -- server/config/seed.js | 37 ++++++++------------------ 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/server/api/commit/commit.controller.js b/server/api/commit/commit.controller.js index f3577431..3c8f7afe 100644 --- a/server/api/commit/commit.controller.js +++ b/server/api/commit/commit.controller.js @@ -22,9 +22,7 @@ exports.show = function(req, res) { // Creates a new commit in the DB. exports.create = function(req, res) { - console.error('111CREATING??'); Commit.create(req.body, function(err, commit) { - console.error('CREATING??'); if(err) { return handleError(res, err); } return res.json(201, commit); }); diff --git a/server/config/seed.js b/server/config/seed.js index e3096641..189fe78a 100644 --- a/server/config/seed.js +++ b/server/config/seed.js @@ -34,9 +34,11 @@ var seed = function() { var projects = require('./seed/projects.json'); var posts = require('./seed/posts.json'); var smallgroups = require('./seed/smallgroups.json'); -// var d = -// console.error('d',d); -var promises = []; +var d = GithubWorker.getCommitsForRepository(projects[0].githubUsername, projects[0].githubProjectName, function(data) { + console.error('1',data) + return data; +}); +console.error('d',d); var user = User.remove({}).exec() .then(function() { return User.create(users); @@ -44,7 +46,7 @@ var promises = []; .then(function(){ console.log('finished populating users'); }); - promises.push(user); + var project = Project.remove({}).exec() .then(function(){ return Project.create(projects) @@ -52,25 +54,10 @@ var promises = []; .then(function() { console.log('finished populating projects') }); - promises.push(project); - Commit.remove({}).exec() + var commit = Commit.remove({}).exec() .then(function(){ - // var allCommitPromises = []; - for (var i = 0;i Date: Tue, 25 Oct 2016 11:31:30 -0400 Subject: [PATCH 08/27] removed api key from github worker (again...) --- server/workers/github.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/workers/github.js b/server/workers/github.js index 4983c8f9..113d834f 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -1,8 +1,8 @@ var octo = require('octonode'); -/* - @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE +/* + @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE */ -var token = "26a687eb6476756cedf8951316c9542f3011378c"; +var token = "INSERT GITHUB API TOKEN HERE"; var octoclient = octo.client(token); // Concurrency Variables @@ -22,12 +22,12 @@ function runThread(func){ module.exports.getCommitsForRepository = function(repoOwner, repoName, callback){ runThread(function(){ - /* + /* ex: github.com// - repoOwner: github username for owner of github repo + repoOwner: github username for owner of github repo repoName: github repo name - example: + example: repoOwner: rcos repoName: Observatory3 */ From c68570e20d2e5ec776984290df512c765af835f4 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 25 Oct 2016 11:40:14 -0400 Subject: [PATCH 09/27] removed runThread from github.js worker; don't think it's currently needed (?)\n & Added some docs/comments --- server/workers/github.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server/workers/github.js b/server/workers/github.js index 113d834f..685ab9c9 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -21,7 +21,6 @@ function runThread(func){ module.exports.getCommitsForRepository = function(repoOwner, repoName, callback){ - runThread(function(){ /* ex: github.com// repoOwner: github username for owner of github repo @@ -35,9 +34,14 @@ module.exports.getCommitsForRepository = function(repoOwner, repoName, callback) var ghrepo = octoclient.repo(githubRepo); var commits = ghrepo.commits(function(err, data, headers) { //@TODO: add logic to create a commit model & save it to DB - callback(data); + return data; }); - }); + /* + not sure if callback is even needed... doesn't make sense with current function naming; + maybe would be better to refactor function name to specify that it applies callback to list of commits from repo? + or maybe this is just a javascript convention, and having callbacks everywhere is normal? + */ + return callback(commits); }; // @TODO: remove/cleanup; just using this as a reference for now... // var pushEvents = ghuser.events(['PushEvent'], function(ignore, pushEvents){ From 459974ede2a877d0df69f2579400839f1ab0dd1c Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 25 Oct 2016 16:41:22 -0400 Subject: [PATCH 10/27] Added GITHUB_WORKER_KEY env variable.\n Removed hard coded github key/token --- server/config/local.env.sample.js | 2 ++ server/workers/github.js | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/config/local.env.sample.js b/server/config/local.env.sample.js index 7e502417..da88c903 100644 --- a/server/config/local.env.sample.js +++ b/server/config/local.env.sample.js @@ -17,6 +17,8 @@ module.exports = { SERVER_ADDRESS: 'http://localhost:9000', SENDGRID_API_KEY: 'YOUR_KEY', + GITHUB_WORKER_TOKEN: 'YOUR_KEY', + // Control debug level for modules using visionmedia/debug DEBUG: '' diff --git a/server/workers/github.js b/server/workers/github.js index 685ab9c9..055aa01a 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -2,7 +2,8 @@ var octo = require('octonode'); /* @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE */ -var token = "INSERT GITHUB API TOKEN HERE"; + +var token = process.env.GITHUB_WORKER_TOKEN; var octoclient = octo.client(token); // Concurrency Variables From a3d0b6b6dd3b73c413457707c6e82d6b04dddcc3 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 25 Oct 2016 18:01:02 -0400 Subject: [PATCH 11/27] Github worker now populates commit model with commits! Still a W.I.P. --- server/api/project/project.model.js | 9 ++ server/config/seed.js | 1 + server/workers/github.js | 133 +++++++++++++++++----------- 3 files changed, 90 insertions(+), 53 deletions(-) diff --git a/server/api/project/project.model.js b/server/api/project/project.model.js index f3e34dc7..2bbf7730 100644 --- a/server/api/project/project.model.js +++ b/server/api/project/project.model.js @@ -44,4 +44,13 @@ var ProjectSchema = new Schema({ tech: [String] },{ timestamps: true}); +/* + @returns: + */ +ProjectSchema + .virtual('fullRepoPath') + .get(function(){ + return (this.githubUsername + "/" + this.githubProjectName) ; + }); + module.exports = mongoose.model('Project', ProjectSchema); diff --git a/server/config/seed.js b/server/config/seed.js index c849b6a3..beebb2b0 100644 --- a/server/config/seed.js +++ b/server/config/seed.js @@ -90,6 +90,7 @@ var seed = function() { if (!module.parent) { if (args.length == 0) { seed().then(function(){ + db.disconnect() }); } diff --git a/server/workers/github.js b/server/workers/github.js index 055aa01a..5f71892e 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -1,3 +1,19 @@ +'use strict'; + +if (!module.parent) { + // Set default node environment to development + process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + + var mongoose = require('mongoose'); + var config = require('../config/environment'); + var args = process.argv.slice(2); + + // Connect to database + var db = mongoose.connect(config.mongo.uri, config.mongo.options); +} +var Commit = require('../api/commit/commit.model'); +var Project = require('../api/project/project.model'); + var octo = require('octonode'); /* @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE @@ -6,58 +22,69 @@ var octo = require('octonode'); var token = process.env.GITHUB_WORKER_TOKEN; var octoclient = octo.client(token); -// Concurrency Variables -var maxConcurrentThreads = 10; -var runningThreads = 0; -var queuedThreads = []; - -function runThread(func){ - // If we've exceeded the max number of threads, delay the call - if (queuedThreads.length > 0 || runningThreads >= maxConcurrentThreads){ - queuedThreads.push(func); - }else{ - func(); - } -} - - -module.exports.getCommitsForRepository = function(repoOwner, repoName, callback){ - /* - ex: github.com// - repoOwner: github username for owner of github repo - repoName: github repo name - - example: - repoOwner: rcos - repoName: Observatory3 - */ - var githubRepo = repoOwner + "/" + repoName; - var ghrepo = octoclient.repo(githubRepo); - var commits = ghrepo.commits(function(err, data, headers) { - //@TODO: add logic to create a commit model & save it to DB - return data; +// var projects +Project.find().exec().then(function(projects){ + projects.forEach(function(project) { + console.error("PROJECT: ", project.fullRepoPath); + var ghrepo = octoclient.repo(project.fullRepoPath); + ghrepo.commits( + function(err, commitDataArray, headers) { + commitDataArray.forEach(function(commitData) { + //console.log("creating ...."); + return Commit.create(commitData); }); - /* - not sure if callback is even needed... doesn't make sense with current function naming; - maybe would be better to refactor function name to specify that it applies callback to list of commits from repo? - or maybe this is just a javascript convention, and having callbacks everywhere is normal? - */ - return callback(commits); -}; - // @TODO: remove/cleanup; just using this as a reference for now... - // var pushEvents = ghuser.events(['PushEvent'], function(ignore, pushEvents){ - // if (!pushEvents){ - // return callback([]); - // } - // var strings = []; - // for (var i = 0; i < pushEvents.length;i++){ - // var payload = pushEvents[i].payload; - // var commits = payload.commits; - // for (var u = 0; u < commits.length; u++){ - // strings.push(commits[u].message); - // } - // } - // callback(strings); - // }); - // }); + }); + }); +}); +// +// projects.forEach(function(project) { +// console.error("PROJECT: ", project.fullRepoPath); +// var ghrepo = octoclient.repo(project.fullRepoPath); +// ghrepo.commits( +// function(err, commitDataArray, headers) { +// commitDataArray.forEach(function(commitData) { +// console.log(commitData); +// //Commit.create(commitData); +// }); +// }); +// }); +// function getCommitsForRepository(repoOwner, repoName, callback){ +// /* +// ex: github.com// +// repoOwner: github username for owner of github repo +// repoName: github repo name +// +// example: +// repoOwner: rcos +// repoName: Observatory3 +// */ +// var githubRepo = repoOwner + "/" + repoName; +// var ghrepo = octoclient.repo(githubRepo); +// var commits = ghrepo.commits(function(err, data, headers) { +// //@TODO: add logic to create a commit model & save it to DB +// return data; +// }); +// /* +// not sure if callback is even needed... doesn't make sense with current function naming; +// maybe would be better to refactor function name to specify that it applies callback to list of commits from repo? +// or maybe this is just a javascript convention, and having callbacks everywhere is normal? +// */ +// return callback(commits); // }; +// // @TODO: remove/cleanup; just using this as a reference for now... +// // var pushEvents = ghuser.events(['PushEvent'], function(ignore, pushEvents){ +// // if (!pushEvents){ +// // return callback([]); +// // } +// // var strings = []; +// // for (var i = 0; i < pushEvents.length;i++){ +// // var payload = pushEvents[i].payload; +// // var commits = payload.commits; +// // for (var u = 0; u < commits.length; u++){ +// // strings.push(commits[u].message); +// // } +// // } +// // callback(strings); +// // }); +// // }); +// // }; From 07b1ce24d1a4b7b60fada4258d61ea9c63e331d4 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 1 Nov 2016 17:38:00 -0400 Subject: [PATCH 12/27] pulling commits from github now works! Currently fixing up commit api, model & controller --- server/api/commit/commit.controller.js | 7 --- server/api/commit/commit.model.js | 1 + server/workers/github.js | 62 +++++++++++++++++++++----- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/server/api/commit/commit.controller.js b/server/api/commit/commit.controller.js index 3c8f7afe..fa3801ad 100644 --- a/server/api/commit/commit.controller.js +++ b/server/api/commit/commit.controller.js @@ -68,16 +68,9 @@ exports.showProjectCommits = function(req, res) { // Get a list of a user Commits exports.showUserCommits = function(req, res) { var prevDays = new Date(); - if (req.params.timeperiod){ - prevDays.setDate(prevDays.getDate()-Number(req.params.timeperiod)); - } - else{ - prevDays.setDate(prevDays.getDate()-14); - } Commit.find() .where('author.login').equals(String(req.params.githubProfile)) - .where('date').gt(prevDays) .exec(function(err, commits){ if(err) { return handleError(res, err); } if(!commits) { return res.json([]); } diff --git a/server/api/commit/commit.model.js b/server/api/commit/commit.model.js index 563f0670..4f71879a 100644 --- a/server/api/commit/commit.model.js +++ b/server/api/commit/commit.model.js @@ -18,6 +18,7 @@ var CommitSchema = new Schema({ branch: String, message: String, date: Date, + projectId: { type: String, index: true}, },{ timestamps: true}); module.exports = mongoose.model('Commit', CommitSchema); diff --git a/server/workers/github.js b/server/workers/github.js index 5f71892e..ba3a9b7c 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -20,22 +20,64 @@ var octo = require('octonode'); */ var token = process.env.GITHUB_WORKER_TOKEN; +console.log(process.env); +throw new AssertionError('wtasd'); var octoclient = octo.client(token); -// var projects -Project.find().exec().then(function(projects){ - projects.forEach(function(project) { +// var updateProjectCommits = function() { +// // var projects +// Project.find().exec().then(function (projects) { +// projects.forEach(function (project) { +// console.error("PROJECT: ", project.fullRepoPath); +// var ghrepo = octoclient.repo(project.fullRepoPath); +// ghrepo.commits( +// function (err, commitDataArray, headers) { +// commitDataArray.forEach(function (commitData) { +// //console.log("creating ...."); +// return Commit.create(commitData); +// }); +// }); +// }); +// }); +// } + +Project.find().exec().then(function (projects) { + var responses = []; + projects.forEach(function (project) { console.error("PROJECT: ", project.fullRepoPath); var ghrepo = octoclient.repo(project.fullRepoPath); - ghrepo.commits( - function(err, commitDataArray, headers) { - commitDataArray.forEach(function(commitData) { - //console.log("creating ...."); - return Commit.create(commitData); - }); - }); + var r = ghrepo.commits( + function (err, commitDataArray, headers) { + console.log(err,commitDataArray); + commitDataArray.forEach( + function (commitData) { + //console.log("creating ...."); + commitData["projectId"] = project.id; + console.log("projectId:", commitData.projectId); + console.log(commitData); + console.log(Object.getOwnPropertyNames(commitData)); + // process.exit(); + throw new Error('abort abort abort ABORT'); + return Commit.create(commitData); + }); + // console.error(err, headers); + } + ); + responses.push(r); }); + // console.error("responses", responses); +}).then(function() { + db.disconnect(); + console.log('finished populating commits'); }); +// if (!module.parent) { +// if (args.length == 0) { +// seed().then(function(){ +// +// db.disconnect() +// }); +// } +// } // // projects.forEach(function(project) { // console.error("PROJECT: ", project.fullRepoPath); From c312e90af82d74ba3c58f11ba5c50d5d1b3411b6 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 1 Nov 2016 17:49:45 -0400 Subject: [PATCH 13/27] fixed github token env variable, now imports correctly --- server/workers/github.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/workers/github.js b/server/workers/github.js index ba3a9b7c..0dc6d842 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -19,9 +19,7 @@ var octo = require('octonode'); @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE */ -var token = process.env.GITHUB_WORKER_TOKEN; -console.log(process.env); -throw new AssertionError('wtasd'); +var token = config.GITHUB_WORKER_TOKEN; var octoclient = octo.client(token); // var updateProjectCommits = function() { @@ -56,7 +54,6 @@ Project.find().exec().then(function (projects) { console.log("projectId:", commitData.projectId); console.log(commitData); console.log(Object.getOwnPropertyNames(commitData)); - // process.exit(); throw new Error('abort abort abort ABORT'); return Commit.create(commitData); }); From 2e092c121c720d962b51d931beaa656e800c8573 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 1 Nov 2016 21:08:15 -0400 Subject: [PATCH 14/27] finished commit-project DB relationship link; cleaned up github worker code --- server/api/commit/commit.model.js | 10 +- server/api/project/project.model.js | 8 +- server/api/user/user.model.js | 1 + server/config/environment/development.js | 3 +- server/workers/github.js | 139 ++++++----------------- 5 files changed, 50 insertions(+), 111 deletions(-) diff --git a/server/api/commit/commit.model.js b/server/api/commit/commit.model.js index 4f71879a..98526b2d 100644 --- a/server/api/commit/commit.model.js +++ b/server/api/commit/commit.model.js @@ -2,6 +2,7 @@ var mongoose = require('mongoose'), Schema = mongoose.Schema; +var Project = require('../project/project.model'); var CommitSchema = new Schema({ url: String, @@ -10,15 +11,18 @@ var CommitSchema = new Schema({ unique: true, index: true, }, - userId: { type: String, index: true}, author: { login: {type: String, lowercase: true}, id: {type: Number} }, - branch: String, + commentCount: Number, message: String, date: Date, - projectId: { type: String, index: true}, + project: { + type : Schema.Types.ObjectId, + ref: 'Project', + index: true + }, },{ timestamps: true}); module.exports = mongoose.model('Commit', CommitSchema); diff --git a/server/api/project/project.model.js b/server/api/project/project.model.js index 2bbf7730..cecd485c 100644 --- a/server/api/project/project.model.js +++ b/server/api/project/project.model.js @@ -2,6 +2,7 @@ var mongoose = require('mongoose'), Schema = mongoose.Schema; +var Commit = require('../commit/commit.model'); var ProjectSchema = new Schema({ name: { @@ -41,7 +42,12 @@ var ProjectSchema = new Schema({ default: false, index: true }, - tech: [String] + tech: [String], + commits: [{ + type : Schema.Types.ObjectId, + ref: 'Commit', + index: true + }], },{ timestamps: true}); /* diff --git a/server/api/user/user.model.js b/server/api/user/user.model.js index 1e8bec6f..ae26114c 100644 --- a/server/api/user/user.model.js +++ b/server/api/user/user.model.js @@ -38,6 +38,7 @@ var UserSchema = new Schema({ index: true }], // project id + bio: String, password: { diff --git a/server/config/environment/development.js b/server/config/environment/development.js index 525aeeac..695bd1af 100644 --- a/server/config/environment/development.js +++ b/server/config/environment/development.js @@ -17,5 +17,6 @@ module.exports = { // For testing verification // attendanceVerificationRatio: 1, - seedDB: false + seedDB: false, + }; diff --git a/server/workers/github.js b/server/workers/github.js index 0dc6d842..b5877b42 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -16,114 +16,41 @@ var Project = require('../api/project/project.model'); var octo = require('octonode'); /* - @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE -*/ + @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE + */ var token = config.GITHUB_WORKER_TOKEN; var octoclient = octo.client(token); -// var updateProjectCommits = function() { -// // var projects -// Project.find().exec().then(function (projects) { -// projects.forEach(function (project) { -// console.error("PROJECT: ", project.fullRepoPath); -// var ghrepo = octoclient.repo(project.fullRepoPath); -// ghrepo.commits( -// function (err, commitDataArray, headers) { -// commitDataArray.forEach(function (commitData) { -// //console.log("creating ...."); -// return Commit.create(commitData); -// }); -// }); -// }); -// }); -// } +Project.find().exec().then( + function (projects) { + projects.forEach( + function (project) { + console.error("PROJECT: ", project.fullRepoPath); + var ghrepo = octoclient.repo(project.fullRepoPath); + ghrepo.commits( + function (err, commitDataArray, headers) { + commitDataArray.forEach( + function (commitData) { + console.log(JSON.stringify(commitData, null, 4)); + var newCommit = new Commit(commitData); -Project.find().exec().then(function (projects) { - var responses = []; - projects.forEach(function (project) { - console.error("PROJECT: ", project.fullRepoPath); - var ghrepo = octoclient.repo(project.fullRepoPath); - var r = ghrepo.commits( - function (err, commitDataArray, headers) { - console.log(err,commitDataArray); - commitDataArray.forEach( - function (commitData) { - //console.log("creating ...."); - commitData["projectId"] = project.id; - console.log("projectId:", commitData.projectId); - console.log(commitData); - console.log(Object.getOwnPropertyNames(commitData)); - throw new Error('abort abort abort ABORT'); - return Commit.create(commitData); - }); - // console.error(err, headers); - } - ); - responses.push(r); - }); - // console.error("responses", responses); -}).then(function() { - db.disconnect(); - console.log('finished populating commits'); -}); -// if (!module.parent) { -// if (args.length == 0) { -// seed().then(function(){ -// -// db.disconnect() -// }); -// } -// } -// -// projects.forEach(function(project) { -// console.error("PROJECT: ", project.fullRepoPath); -// var ghrepo = octoclient.repo(project.fullRepoPath); -// ghrepo.commits( -// function(err, commitDataArray, headers) { -// commitDataArray.forEach(function(commitData) { -// console.log(commitData); -// //Commit.create(commitData); -// }); -// }); -// }); -// function getCommitsForRepository(repoOwner, repoName, callback){ -// /* -// ex: github.com// -// repoOwner: github username for owner of github repo -// repoName: github repo name -// -// example: -// repoOwner: rcos -// repoName: Observatory3 -// */ -// var githubRepo = repoOwner + "/" + repoName; -// var ghrepo = octoclient.repo(githubRepo); -// var commits = ghrepo.commits(function(err, data, headers) { -// //@TODO: add logic to create a commit model & save it to DB -// return data; -// }); -// /* -// not sure if callback is even needed... doesn't make sense with current function naming; -// maybe would be better to refactor function name to specify that it applies callback to list of commits from repo? -// or maybe this is just a javascript convention, and having callbacks everywhere is normal? -// */ -// return callback(commits); -// }; -// // @TODO: remove/cleanup; just using this as a reference for now... -// // var pushEvents = ghuser.events(['PushEvent'], function(ignore, pushEvents){ -// // if (!pushEvents){ -// // return callback([]); -// // } -// // var strings = []; -// // for (var i = 0; i < pushEvents.length;i++){ -// // var payload = pushEvents[i].payload; -// // var commits = payload.commits; -// // for (var u = 0; u < commits.length; u++){ -// // strings.push(commits[u].message); -// // } -// // } -// // callback(strings); -// // }); -// // }); -// // }; + // super hacky, definitely need to abstract this before github's api changes and this goes boom!! + newCommit.project = project; + newCommit.message = commitData.commit.message; + newCommit.date = commitData.commit.author.date; + newCommit.commentCount = commitData.commit.comment_count; + return newCommit.save(); + } + ); + } + ); + } + ); + } +).then( + function () { + db.disconnect(); + console.log('finished populating commits'); + } +); From 8d7c02633afb032da2ebfd923563b3cc18f636ec Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 1 Nov 2016 21:08:59 -0400 Subject: [PATCH 15/27] more env variable issues >.< --- server/config/environment/development.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/config/environment/development.js b/server/config/environment/development.js index 695bd1af..875b9d60 100644 --- a/server/config/environment/development.js +++ b/server/config/environment/development.js @@ -18,5 +18,7 @@ module.exports = { // attendanceVerificationRatio: 1, seedDB: false, + // Github token for worker, local.env.js kept returning undefined but it worked once I added it here... + GITHUB_WORKER_TOKEN: 'YOUR_KEY', }; From c366a9c00e3f27cb095d6511884521437e180a45 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 1 Nov 2016 21:41:01 -0400 Subject: [PATCH 16/27] removed un-needed debug statement --- server/workers/github.js | 1 - 1 file changed, 1 deletion(-) diff --git a/server/workers/github.js b/server/workers/github.js index b5877b42..b5e3aa9e 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -32,7 +32,6 @@ Project.find().exec().then( function (err, commitDataArray, headers) { commitDataArray.forEach( function (commitData) { - console.log(JSON.stringify(commitData, null, 4)); var newCommit = new Commit(commitData); // super hacky, definitely need to abstract this before github's api changes and this goes boom!! From d11b05e442f42dcb3ab8f7822046af508168a1c0 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 1 Nov 2016 22:30:02 -0400 Subject: [PATCH 17/27] Commits working for the most part. Just need to figure out async issue with DB disconnecting before saving... --- server/workers/github.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/workers/github.js b/server/workers/github.js index b5e3aa9e..4afd79f7 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -49,7 +49,10 @@ Project.find().exec().then( } ).then( function () { - db.disconnect(); + // @TODO FIX ASYNC ISSUES SO DISCONNECT HAPPENS AFTER ALL SAVES HAVE COMPLETED. + // Spent like 4 hours trying to debug this, but javascript is just too magical and asynchronous for me =( + // It works perfectly with db.disconnect commented out, since it will always disconnect before the saves occur... + // db.disconnect(); console.log('finished populating commits'); } ); From e9649b967c926c2e87f29559ba9cdc45adae9d93 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 15 Nov 2016 16:44:51 -0500 Subject: [PATCH 18/27] removed un-needed console.log --- server/workers/github.js | 1 - 1 file changed, 1 deletion(-) diff --git a/server/workers/github.js b/server/workers/github.js index 4afd79f7..84a4c930 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -26,7 +26,6 @@ Project.find().exec().then( function (projects) { projects.forEach( function (project) { - console.error("PROJECT: ", project.fullRepoPath); var ghrepo = octoclient.repo(project.fullRepoPath); ghrepo.commits( function (err, commitDataArray, headers) { From 4df1f887b8f318d333df16d65d9dac7605437bea Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Tue, 15 Nov 2016 20:06:32 -0500 Subject: [PATCH 19/27] started on switching to a different github api wrapper and using promise chains --- package.json | 1 + server/workers/github.js | 140 +++++++++++++++++++++++++++++---------- 2 files changed, 106 insertions(+), 35 deletions(-) diff --git a/package.json b/package.json index 70a51d7d..47a4a488 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "mongoose": "^4.1.2", "morgan": "^1.7.0", "multiparty": "^4.1.1", + "octokat": "^0.4.18", "passport": "~0.3.0", "passport-facebook": "^2.0.0", "passport-google-oauth": "^1.0.0", diff --git a/server/workers/github.js b/server/workers/github.js index 84a4c930..c2b818e7 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -14,44 +14,114 @@ if (!module.parent) { var Commit = require('../api/commit/commit.model'); var Project = require('../api/project/project.model'); -var octo = require('octonode'); +var Octokat = require('octokat'); /* @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE */ +mongoose.Promise = require('bluebird'); -var token = config.GITHUB_WORKER_TOKEN; -var octoclient = octo.client(token); - -Project.find().exec().then( - function (projects) { - projects.forEach( - function (project) { - var ghrepo = octoclient.repo(project.fullRepoPath); - ghrepo.commits( - function (err, commitDataArray, headers) { - commitDataArray.forEach( - function (commitData) { - var newCommit = new Commit(commitData); - - // super hacky, definitely need to abstract this before github's api changes and this goes boom!! - newCommit.project = project; - newCommit.message = commitData.commit.message; - newCommit.date = commitData.commit.author.date; - newCommit.commentCount = commitData.commit.comment_count; - return newCommit.save(); - } - ); - } - ); - } - ); +var gtoken = config.GITHUB_WORKER_TOKEN; +var octo = new Octokat({ + token: gtoken +}); +var cb = function (err, val) { + if(err) { + console.log(err); + throw new Error(); } -).then( - function () { - // @TODO FIX ASYNC ISSUES SO DISCONNECT HAPPENS AFTER ALL SAVES HAVE COMPLETED. - // Spent like 4 hours trying to debug this, but javascript is just too magical and asynchronous for me =( - // It works perfectly with db.disconnect commented out, since it will always disconnect before the saves occur... - // db.disconnect(); - console.log('finished populating commits'); + console.log(val); + console.log(val.length); + return val.length; +}; + + + +function fetchCommitsFromProject (owner, project) { + console.log(owner, project,'000'); + return fetchAll(octo.repos(owner,project).commits.fetch, []) +} + +function fetchAll (fetch, results) { + console.log(fetch, results, '456'); + + return fetchUntil(fetch, results, () => false) +} + +function fetchUntil (fetch, results, done) { + // console.log(fetch, results, done, '123'); + return new Promise((resolve, reject) => { + fetch() + .then(result => { + results = results.concat(result); + + if (result.nextPage && ! done(result)) { + return resolve(fetchUntil(result.nextPage, results, done)) + } + + resolve(results) +}) +.catch(reject) +}) +} +var saveCommit = function (commitData) { + console.log('saviing....'); + var newCommit = new Commit(commitData); + + // super hacky, definitely need to abstract this before github's api changes and this goes boom!! + newCommit.project = project; + newCommit.message = commitData.commit.message; + newCommit.date = commitData.commit.author.date; + newCommit.commentCount = commitData.commit.comment_count; + return newCommit.save(); +}; +var saveCommits = function(commits) { + console.log("comm len", commits.length, commits[0].length); + var commitPromises = []; + commits[0].forEach(function(commit) { + console.log('pre-save...'); + commitPromises.push(saveCommit(commit)); + }); + console.log("cpro len", commitPromises.length); + return Promise.all(commitPromises); +}; +var fetchCommits = function (owner, project) { + return Promise.all([fetchCommitsFromProject(owner, project)]); +}; +if (!module.parent) { + if (args.length == 0) { + + fetchCommits('rcos', 'Observatory3').then(saveCommits).then(function (val) { + console.log(val); + console.log(val.length); + console.log(Object.keys(val)); + console.log(Object.getOwnPropertyNames(val)); + console.log("gg?"); + }); } -); +} +// Project.find().exec().then( +// function (projects) { +// var promises = []; +// projects.forEach( +// function (project) { +// console.log('test start...'); +// var owner = project.githubUsername; +// var proj = project.githubProjectName; +// console.log('...'); +// promises.push(fetchCommit(owner,proj));//octo.repos(owner,proj).commits.fetch() +// // .then(); //commits.fetchAll(cb); +// // console.log(test); +// // throw new Error(); +// } +// ); +// return Promise.all(promises); +// } +// ).then( +// function () { +// // @TODO FIX ASYNC ISSUES SO DISCONNECT HAPPENS AFTER ALL SAVES HAVE COMPLETED. +// // Spent like 4 hours trying to debug this, but javascript is just too magical and asynchronous for me =( +// // It works perfectly with db.disconnect commented out, since it will always disconnect before the saves occur... +// // db.disconnect(); +// console.log('finished populating commits'); +// } +// ); From 28dd8c5e0fbf5181c3c4664497cf89c2ba9b6f21 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Fri, 18 Nov 2016 18:18:21 -0500 Subject: [PATCH 20/27] Completely overhauled github commit retrival & storage; much more streamline. Currently only a P.O.C. Will abstract soon. --- server/workers/github.js | 156 ++++++++++++++++++++++----------------- 1 file changed, 89 insertions(+), 67 deletions(-) diff --git a/server/workers/github.js b/server/workers/github.js index c2b818e7..d618d07c 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -15,113 +15,135 @@ var Commit = require('../api/commit/commit.model'); var Project = require('../api/project/project.model'); var Octokat = require('octokat'); + +var Promise = require("bluebird"); + +mongoose.Promise = require('bluebird'); /* - @TODO: IMPORT TOKEN FROM AN UNTRACKED/PRIVATE CONFIG FILE + The github token currently needs to be defined in development.js env. + Since, this is a worker that is currently run individually, it does not have app.js loaded beforehand. + Meaning, magic env stuff isn't automagically setup. + @TODO: bring back the magic */ -mongoose.Promise = require('bluebird'); - var gtoken = config.GITHUB_WORKER_TOKEN; +if(gtoken == "YOUR_KEY") { + console.error('ERROR: PLEASE SET YOUR GITHUB AUTH IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); + throw new Error('ERROR: PLEASE SET YOUR GITHUB AUTH IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); +} +// setup & initilize our github API library var octo = new Octokat({ token: gtoken }); -var cb = function (err, val) { - if(err) { - console.log(err); - throw new Error(); - } - console.log(val); - console.log(val.length); - return val.length; -}; - - -function fetchCommitsFromProject (owner, project) { - console.log(owner, project,'000'); - return fetchAll(octo.repos(owner,project).commits.fetch, []) +/* + @TODO: add support for filtering commits by date/author/etc.. + ex: only get commits newer than date x, or only get commits older than date y + */ +function fetchCommitsFromProject (owner, repository) { + return fetchAll(octo.repos(owner,repository).commits.fetch, []) } function fetchAll (fetch, results) { - console.log(fetch, results, '456'); - return fetchUntil(fetch, results, () => false) } function fetchUntil (fetch, results, done) { - // console.log(fetch, results, done, '123'); return new Promise((resolve, reject) => { fetch() .then(result => { results = results.concat(result); if (result.nextPage && ! done(result)) { - return resolve(fetchUntil(result.nextPage, results, done)) + return resolve(fetchUntil(result.nextPage, results, done)); } - resolve(results) + resolve(results); }) .catch(reject) }) } -var saveCommit = function (commitData) { - console.log('saviing....'); +var saveCommit = function (commitData, project) { var newCommit = new Commit(commitData); - - // super hacky, definitely need to abstract this before github's api changes and this goes boom!! + /* + establish the ownership relationship; i.e. what project does this commit belong to? + @TODO: setup the inverse relationship(?); i.e. what commits does the project own? + */ newCommit.project = project; + /* + manually extract/configure the data to match our Commit schema. + */ newCommit.message = commitData.commit.message; newCommit.date = commitData.commit.author.date; newCommit.commentCount = commitData.commit.comment_count; + // Save the new commit model to the db. This is an async operation, so it will be turned into a promise. return newCommit.save(); }; -var saveCommits = function(commits) { - console.log("comm len", commits.length, commits[0].length); +/* + @promiseObject: JS object containing 2 fields. + commits: promise array of fetched commits from github + project: instance of project model that we are fetching the commits for. + */ +var saveCommits = function (promiseObject) { + // extract the data from the promiseObject + var commits = promiseObject.commits; + var project = promiseObject.project; + // array of Commit.save() operations; to be turned into an array of Promises. var commitPromises = []; - commits[0].forEach(function(commit) { - console.log('pre-save...'); - commitPromises.push(saveCommit(commit)); + commits.forEach(function(commit) { + commitPromises.push(saveCommit(commit, project)); }); - console.log("cpro len", commitPromises.length); + // return an array of promises, where each promise is a promise to save the commit model to the DB. return Promise.all(commitPromises); }; -var fetchCommits = function (owner, project) { - return Promise.all([fetchCommitsFromProject(owner, project)]); + +var fetchCommits = function (project) { + // extract the github username & github repository name to fetch the commits from. + var owner = project.githubUsername; + var repository = project.githubProjectName; + /* + Needed to pass through project model; to saveCommits; but javascript. + So instead I just tacked it onto an object. + Promise.props is essentially the same as using Promise.all([]), + Except it returns an object with arbitrary fields that can be a promise or arbitrary data. + with an added variable, @project. + Probably a better way to do this, but I currently can't think of any because javascript. + */ + + return Promise.props({ + commits: fetchCommitsFromProject(owner, repository), + project: project, + }); }; + +/* + For now just a hard coded example/P.O.C using the github repo rcos/observatory3 + Planning to refactor this "soon(tm)" + */ if (!module.parent) { if (args.length == 0) { - - fetchCommits('rcos', 'Observatory3').then(saveCommits).then(function (val) { - console.log(val); - console.log(val.length); - console.log(Object.keys(val)); - console.log(Object.getOwnPropertyNames(val)); - console.log("gg?"); + // + Project.findOne({githubUsername: 'rcos', githubProjectName: 'observatory3'}, function(err, project) { + //@TODO: more robust error handling & feedback + if(err) { + console.error('Error finding project!'); + throw new Error('Error finding project!', err); + } + /* + .then may not be needed? May just be able to do it all in this function? + Probably un-needed, but I don't even anymore. Javascript is just too mystical... + */ + }).then(function(project) { + /* + flow: + 1. fetch commits from github for @project + 2. save fetched commits to db + 3. disconnect from DB + */ + fetchCommits(project).then(saveCommits).then(function () { + db.disconnect(); + console.log("Done fetching & saving commits!\n Check the commits collection in your database!"); + }); }); + } } -// Project.find().exec().then( -// function (projects) { -// var promises = []; -// projects.forEach( -// function (project) { -// console.log('test start...'); -// var owner = project.githubUsername; -// var proj = project.githubProjectName; -// console.log('...'); -// promises.push(fetchCommit(owner,proj));//octo.repos(owner,proj).commits.fetch() -// // .then(); //commits.fetchAll(cb); -// // console.log(test); -// // throw new Error(); -// } -// ); -// return Promise.all(promises); -// } -// ).then( -// function () { -// // @TODO FIX ASYNC ISSUES SO DISCONNECT HAPPENS AFTER ALL SAVES HAVE COMPLETED. -// // Spent like 4 hours trying to debug this, but javascript is just too magical and asynchronous for me =( -// // It works perfectly with db.disconnect commented out, since it will always disconnect before the saves occur... -// // db.disconnect(); -// console.log('finished populating commits'); -// } -// ); From 9764bd871d6c3b9e35191fb54eb1a2c137e1d071 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Fri, 2 Dec 2016 14:48:43 -0500 Subject: [PATCH 21/27] minor formatting cleanups --- server/workers/github.js | 62 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/server/workers/github.js b/server/workers/github.js index d618d07c..b3a4ff9b 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -20,10 +20,10 @@ var Promise = require("bluebird"); mongoose.Promise = require('bluebird'); /* - The github token currently needs to be defined in development.js env. - Since, this is a worker that is currently run individually, it does not have app.js loaded beforehand. - Meaning, magic env stuff isn't automagically setup. - @TODO: bring back the magic + The github token currently needs to be defined in development.js env. + Since, this is a worker that is currently run individually, it does not have app.js loaded beforehand. + Meaning, magic env stuff isn't automagically setup. + @TODO: bring back the magic */ var gtoken = config.GITHUB_WORKER_TOKEN; if(gtoken == "YOUR_KEY") { @@ -36,8 +36,8 @@ var octo = new Octokat({ }); /* - @TODO: add support for filtering commits by date/author/etc.. - ex: only get commits newer than date x, or only get commits older than date y + @TODO: add support for filtering commits by date/author/etc.. + ex: only get commits newer than date x, or only get commits older than date y */ function fetchCommitsFromProject (owner, repository) { return fetchAll(octo.repos(owner,repository).commits.fetch, []) @@ -50,27 +50,27 @@ function fetchAll (fetch, results) { function fetchUntil (fetch, results, done) { return new Promise((resolve, reject) => { fetch() - .then(result => { - results = results.concat(result); + .then(result => { + results = results.concat(result); - if (result.nextPage && ! done(result)) { - return resolve(fetchUntil(result.nextPage, results, done)); - } + if (result.nextPage && ! done(result)) { + return resolve(fetchUntil(result.nextPage, results, done)); + } - resolve(results); -}) -.catch(reject) -}) + resolve(results); + }) + .catch(reject) + }) } var saveCommit = function (commitData, project) { var newCommit = new Commit(commitData); /* establish the ownership relationship; i.e. what project does this commit belong to? @TODO: setup the inverse relationship(?); i.e. what commits does the project own? - */ + */ newCommit.project = project; /* - manually extract/configure the data to match our Commit schema. + manually extract/configure the data to match our Commit schema. */ newCommit.message = commitData.commit.message; newCommit.date = commitData.commit.author.date; @@ -79,9 +79,9 @@ var saveCommit = function (commitData, project) { return newCommit.save(); }; /* - @promiseObject: JS object containing 2 fields. - commits: promise array of fetched commits from github - project: instance of project model that we are fetching the commits for. + @promiseObject: JS object containing 2 fields. + commits: promise array of fetched commits from github + project: instance of project model that we are fetching the commits for. */ var saveCommits = function (promiseObject) { // extract the data from the promiseObject @@ -101,12 +101,12 @@ var fetchCommits = function (project) { var owner = project.githubUsername; var repository = project.githubProjectName; /* - Needed to pass through project model; to saveCommits; but javascript. - So instead I just tacked it onto an object. - Promise.props is essentially the same as using Promise.all([]), - Except it returns an object with arbitrary fields that can be a promise or arbitrary data. - with an added variable, @project. - Probably a better way to do this, but I currently can't think of any because javascript. + Needed to pass through project model; to saveCommits; but javascript. + So instead I just tacked it onto an object. + Promise.props is essentially the same as using Promise.all([]), + Except it returns an object with arbitrary fields that can be a promise or arbitrary data. + with an added variable, @project. + Probably a better way to do this, but I currently can't think of any because javascript. */ return Promise.props({ @@ -131,13 +131,13 @@ if (!module.parent) { /* .then may not be needed? May just be able to do it all in this function? Probably un-needed, but I don't even anymore. Javascript is just too mystical... - */ + */ }).then(function(project) { /* - flow: - 1. fetch commits from github for @project - 2. save fetched commits to db - 3. disconnect from DB + flow: + 1. fetch commits from github for @project + 2. save fetched commits to db + 3. disconnect from DB */ fetchCommits(project).then(saveCommits).then(function () { db.disconnect(); From a88fef9a3f963cfc20ea2c8d3f8bc7b0d80fe054 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Fri, 2 Dec 2016 18:03:30 -0500 Subject: [PATCH 22/27] cleanup & refactor. WIP: adding configuration options for fetching commits (since/until/per_page/etc..) --- server/workers/github.js | 94 ++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 24 deletions(-) diff --git a/server/workers/github.js b/server/workers/github.js index b3a4ff9b..a51bdce1 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -39,29 +39,49 @@ var octo = new Octokat({ @TODO: add support for filtering commits by date/author/etc.. ex: only get commits newer than date x, or only get commits older than date y */ -function fetchCommitsFromProject (owner, repository) { - return fetchAll(octo.repos(owner,repository).commits.fetch, []) +function fetchCommitsFromProject (owner, repository, config) { + return fetchAll(octo.repos(owner, repository).commits.fetch, config); } - -function fetchAll (fetch, results) { - return fetchUntil(fetch, results, () => false) +function fetchAll(fn, args) { + let acc = []; + if (!args) { + args = {per_page: 100}; + } + let p = new Promise((resolve, reject) => { + fn(args).then((val) => { + acc = acc.concat(val); + if (val.nextPage) { + fetchAll(val.nextPage).then((val2) => { + acc = acc.concat(val2); + resolve(acc); + }, reject); + } else { + resolve(acc); + } + }, reject); + }); + return p; } + // var fn = octo.repos(owner,repository).commits.fetch; + // + // let acc = []; // Accumulated results + // let p = new Promise((resolve, reject) => { + // fn(config).then((val) => { + // acc = acc.concat(val); + // if (val.nextPage) { + // return fetchAll(val.nextPage).then((val2) => { + // acc = acc.concat(val2); + // resolve(acc); + // }, reject); + // } else { + // resolve(acc); + // } + // }, reject); + // }); + // console.log("Fetched commits for project:::", repository); + // return p; +// } -function fetchUntil (fetch, results, done) { - return new Promise((resolve, reject) => { - fetch() - .then(result => { - results = results.concat(result); - - if (result.nextPage && ! done(result)) { - return resolve(fetchUntil(result.nextPage, results, done)); - } - - resolve(results); - }) - .catch(reject) - }) -} var saveCommit = function (commitData, project) { var newCommit = new Commit(commitData); /* @@ -96,10 +116,22 @@ var saveCommits = function (promiseObject) { return Promise.all(commitPromises); }; -var fetchCommits = function (project) { +/** + * + * @param project: instance of Project model + * @param config: + */ +var fetchCommits = function (project, config) { // extract the github username & github repository name to fetch the commits from. var owner = project.githubUsername; var repository = project.githubProjectName; + + // Config is an optional arguement, so if it isn't pased init it ourselves + config = config || {}; + + // always set the results per page limit to the maximum, 100, to reduce github API calls + config["per_page"] = 100; + /* Needed to pass through project model; to saveCommits; but javascript. So instead I just tacked it onto an object. @@ -110,7 +142,7 @@ var fetchCommits = function (project) { */ return Promise.props({ - commits: fetchCommitsFromProject(owner, repository), + commits: fetchCommitsFromProject(owner, repository, config), project: project, }); }; @@ -121,6 +153,18 @@ var fetchCommits = function (project) { */ if (!module.parent) { if (args.length == 0) { + + // Project.find({}).exec(function(err, projects) { + // console.log(projects); + // projects.forEach(function(project) { + // fetchCommits(project).then(saveCommits).then(function() { + // console.log("Processed commits for project::: ", project.fullRepoPath()); + // }); + // }) + // }).then(function () { + // db.disconnect(); + // console.log("Done fetching & saving commits!\n Check the commits collection in your database!"); + // }); // Project.findOne({githubUsername: 'rcos', githubProjectName: 'observatory3'}, function(err, project) { //@TODO: more robust error handling & feedback @@ -139,11 +183,13 @@ if (!module.parent) { 2. save fetched commits to db 3. disconnect from DB */ - fetchCommits(project).then(saveCommits).then(function () { + fetchCommits(project).then(saveCommits).then(function() { + console.log("Processed commits for project::: ", project.githubProjectName); + }).then(function () { db.disconnect(); console.log("Done fetching & saving commits!\n Check the commits collection in your database!"); }); - }); + }); } } From 0365fec7be2069b87c182f72c2a411da15efcdaf Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Fri, 9 Dec 2016 16:44:53 -0500 Subject: [PATCH 23/27] refactored out some un-needed & commented code --- server/workers/github.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/server/workers/github.js b/server/workers/github.js index a51bdce1..bc28b68e 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -154,18 +154,6 @@ var fetchCommits = function (project, config) { if (!module.parent) { if (args.length == 0) { - // Project.find({}).exec(function(err, projects) { - // console.log(projects); - // projects.forEach(function(project) { - // fetchCommits(project).then(saveCommits).then(function() { - // console.log("Processed commits for project::: ", project.fullRepoPath()); - // }); - // }) - // }).then(function () { - // db.disconnect(); - // console.log("Done fetching & saving commits!\n Check the commits collection in your database!"); - // }); - // Project.findOne({githubUsername: 'rcos', githubProjectName: 'observatory3'}, function(err, project) { //@TODO: more robust error handling & feedback if(err) { From 3faa6294408f62be478758b18f5f7184dd6b86c4 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Wed, 14 Dec 2016 06:56:45 -0500 Subject: [PATCH 24/27] fixed find project query --- server/api/commit/commit.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/commit/commit.controller.js b/server/api/commit/commit.controller.js index fa3801ad..90f8f302 100644 --- a/server/api/commit/commit.controller.js +++ b/server/api/commit/commit.controller.js @@ -58,7 +58,7 @@ exports.destroy = function(req, res) { // Show a list of a projects Commits // Get a single commit exports.showProjectCommits = function(req, res) { - Commit.findById(req.params.projectId, function (err, commits) { + Commit.find.where('project').equals(req.params.projectId)).exec(function (err, commits) { if(err) { return handleError(res, err); } if(!commits) { return res.send(404); } return res.json(commits); From aacba9a805097c06406d9fafabd6ad3c1ea5875b Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Wed, 14 Dec 2016 07:14:48 -0500 Subject: [PATCH 25/27] fixed showProjectCommits query again --- server/api/commit/commit.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/commit/commit.controller.js b/server/api/commit/commit.controller.js index 90f8f302..2ae7cb12 100644 --- a/server/api/commit/commit.controller.js +++ b/server/api/commit/commit.controller.js @@ -58,7 +58,7 @@ exports.destroy = function(req, res) { // Show a list of a projects Commits // Get a single commit exports.showProjectCommits = function(req, res) { - Commit.find.where('project').equals(req.params.projectId)).exec(function (err, commits) { + Commit.find({project: req.params.projectId}).populate('project').exec(function (err, commits) { if(err) { return handleError(res, err); } if(!commits) { return res.send(404); } return res.json(commits); From fd3346fe7a8fad0f4c5ddc4d7d3300bdb761d8dc Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Wed, 14 Dec 2016 10:08:17 -0500 Subject: [PATCH 26/27] implemented worker function to get commits for all projects; also added functions to get commits for a project with date filtering --- server/workers/github.js | 108 +++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/server/workers/github.js b/server/workers/github.js index bc28b68e..108b1c5d 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -19,6 +19,7 @@ var Octokat = require('octokat'); var Promise = require("bluebird"); mongoose.Promise = require('bluebird'); + /* The github token currently needs to be defined in development.js env. Since, this is a worker that is currently run individually, it does not have app.js loaded beforehand. @@ -27,8 +28,8 @@ mongoose.Promise = require('bluebird'); */ var gtoken = config.GITHUB_WORKER_TOKEN; if(gtoken == "YOUR_KEY") { - console.error('ERROR: PLEASE SET YOUR GITHUB AUTH IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); - throw new Error('ERROR: PLEASE SET YOUR GITHUB AUTH IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); + console.error('ERROR: PLEASE SET YOUR GITHUB_WORKER_TOKEN IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); + throw new Error('ERROR: PLEASE SET YOUR GITHUB_WORKER_TOKEN IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); } // setup & initilize our github API library var octo = new Octokat({ @@ -62,26 +63,6 @@ function fetchAll(fn, args) { }); return p; } - // var fn = octo.repos(owner,repository).commits.fetch; - // - // let acc = []; // Accumulated results - // let p = new Promise((resolve, reject) => { - // fn(config).then((val) => { - // acc = acc.concat(val); - // if (val.nextPage) { - // return fetchAll(val.nextPage).then((val2) => { - // acc = acc.concat(val2); - // resolve(acc); - // }, reject); - // } else { - // resolve(acc); - // } - // }, reject); - // }); - // console.log("Fetched commits for project:::", repository); - // return p; -// } - var saveCommit = function (commitData, project) { var newCommit = new Commit(commitData); /* @@ -103,7 +84,10 @@ var saveCommit = function (commitData, project) { commits: promise array of fetched commits from github project: instance of project model that we are fetching the commits for. */ -var saveCommits = function (promiseObject) { +var saveCommits = saveCommitsFn; + function saveCommitsFn (promiseObject) { + console.dir(promiseObject); + console.log("entering..."); // extract the data from the promiseObject var commits = promiseObject.commits; var project = promiseObject.project; @@ -116,6 +100,25 @@ var saveCommits = function (promiseObject) { return Promise.all(commitPromises); }; +exports.getCommitsForProjectSinceDate = function (project, date) { + date = new Date(Date.parse(date)) || new Date(); + var config = { + since: date.toISOString() + }; + return processProject(project, config); +}; + +exports.getCommitsForProjectUntilDate = function (project, date) { + date = new Date(Date.parse(date)) || new Date(); + var config = { + ungil: date.toISOString() + }; + return processProject(project, config); +}; + +exports.getCommitsForProject = function(project) { + return processProject(project); +}; /** * * @param project: instance of Project model @@ -151,33 +154,40 @@ var fetchCommits = function (project, config) { For now just a hard coded example/P.O.C using the github repo rcos/observatory3 Planning to refactor this "soon(tm)" */ -if (!module.parent) { - if (args.length == 0) { - - Project.findOne({githubUsername: 'rcos', githubProjectName: 'observatory3'}, function(err, project) { - //@TODO: more robust error handling & feedback - if(err) { - console.error('Error finding project!'); - throw new Error('Error finding project!', err); - } - /* - .then may not be needed? May just be able to do it all in this function? - Probably un-needed, but I don't even anymore. Javascript is just too mystical... - */ - }).then(function(project) { - /* - flow: - 1. fetch commits from github for @project - 2. save fetched commits to db - 3. disconnect from DB - */ - fetchCommits(project).then(saveCommits).then(function() { - console.log("Processed commits for project::: ", project.githubProjectName); - }).then(function () { - db.disconnect(); - console.log("Done fetching & saving commits!\n Check the commits collection in your database!"); - }); +function processProject(project, config) { + /* + flow: + 1. fetch commits from github for @project + 2. save fetched commits to db + 3. disconnect from DB + */ + return fetchCommits(project, config).then(saveCommits); +} +function reflect(promise){ + console.log('reflecttest'); + return promise.then(function(v){ return {v:v, status: "resolved" }}, + function(e){ return {e:e, status: "rejected" }}); +} +function processAllProjects() { + var start = Promise.resolve(); + return Promise.map(Project.find({}).exec(), function (proj) { + start = start.then(function () { + return fetchCommits(proj); }); + return start; + }).map(saveCommitsFn) + .then(function(results) { + console.dir(results); + console.dir(results.length); + db.disconnect(); + }); +} + +if (!module.parent) { + if (args.length == 0) { + console.log("??????????"); + processAllProjects(); + console.log("??????????"); } } From 7732b4bc484d65df1a919821ce74f524a9973c09 Mon Sep 17 00:00:00 2001 From: Michael Zemsky Date: Wed, 14 Dec 2016 10:11:08 -0500 Subject: [PATCH 27/27] cleaned up code & removed unused code --- server/workers/github.js | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/server/workers/github.js b/server/workers/github.js index 108b1c5d..049ec6e0 100644 --- a/server/workers/github.js +++ b/server/workers/github.js @@ -27,10 +27,11 @@ mongoose.Promise = require('bluebird'); @TODO: bring back the magic */ var gtoken = config.GITHUB_WORKER_TOKEN; -if(gtoken == "YOUR_KEY") { +if (gtoken == "YOUR_KEY") { console.error('ERROR: PLEASE SET YOUR GITHUB_WORKER_TOKEN IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); throw new Error('ERROR: PLEASE SET YOUR GITHUB_WORKER_TOKEN IN DEVELOPMENT.JS TOKEN BEFORE PROCEEDING'); } + // setup & initilize our github API library var octo = new Octokat({ token: gtoken @@ -40,9 +41,10 @@ var octo = new Octokat({ @TODO: add support for filtering commits by date/author/etc.. ex: only get commits newer than date x, or only get commits older than date y */ -function fetchCommitsFromProject (owner, repository, config) { +function fetchCommitsFromProject(owner, repository, config) { return fetchAll(octo.repos(owner, repository).commits.fetch, config); } + function fetchAll(fn, args) { let acc = []; if (!args) { @@ -63,6 +65,7 @@ function fetchAll(fn, args) { }); return p; } + var saveCommit = function (commitData, project) { var newCommit = new Commit(commitData); /* @@ -79,26 +82,27 @@ var saveCommit = function (commitData, project) { // Save the new commit model to the db. This is an async operation, so it will be turned into a promise. return newCommit.save(); }; + /* @promiseObject: JS object containing 2 fields. commits: promise array of fetched commits from github project: instance of project model that we are fetching the commits for. */ var saveCommits = saveCommitsFn; - function saveCommitsFn (promiseObject) { - console.dir(promiseObject); - console.log("entering..."); +function saveCommitsFn(promiseObject) { + console.dir(promiseObject); + console.log("entering..."); // extract the data from the promiseObject var commits = promiseObject.commits; var project = promiseObject.project; // array of Commit.save() operations; to be turned into an array of Promises. var commitPromises = []; - commits.forEach(function(commit) { + commits.forEach(function (commit) { commitPromises.push(saveCommit(commit, project)); }); // return an array of promises, where each promise is a promise to save the commit model to the DB. return Promise.all(commitPromises); -}; +} exports.getCommitsForProjectSinceDate = function (project, date) { date = new Date(Date.parse(date)) || new Date(); @@ -116,9 +120,10 @@ exports.getCommitsForProjectUntilDate = function (project, date) { return processProject(project, config); }; -exports.getCommitsForProject = function(project) { +exports.getCommitsForProject = function (project) { return processProject(project); }; + /** * * @param project: instance of Project model @@ -164,11 +169,7 @@ function processProject(project, config) { */ return fetchCommits(project, config).then(saveCommits); } -function reflect(promise){ - console.log('reflecttest'); - return promise.then(function(v){ return {v:v, status: "resolved" }}, - function(e){ return {e:e, status: "rejected" }}); -} + function processAllProjects() { var start = Promise.resolve(); return Promise.map(Project.find({}).exec(), function (proj) { @@ -176,8 +177,7 @@ function processAllProjects() { return fetchCommits(proj); }); return start; - }).map(saveCommitsFn) - .then(function(results) { + }).map(saveCommitsFn).then(function (results) { console.dir(results); console.dir(results.length); db.disconnect(); @@ -186,8 +186,6 @@ function processAllProjects() { if (!module.parent) { if (args.length == 0) { - console.log("??????????"); processAllProjects(); - console.log("??????????"); } }