-
Notifications
You must be signed in to change notification settings - Fork 0
/
loader.js
127 lines (119 loc) · 3.78 KB
/
loader.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
/* jslint node: true, nomen: true*/
'use strict'
var MongoClient = require('mongodb').MongoClient
var config = require('./config')
var spotify = require('./spotify')
var status = require('./loadStatus')
var extractor = require('./extractor')
function insertPlaylists(playlists) {
return MongoClient.connect(config.db_url)
.then(function(db) {
if (playlists.length > 0) {
console.log('inserting', playlists.length)
return db.collection('playlists').insertMany(playlists)
.then(function(result) {
var loaded = result.insertedCount
console.log('inserted: ' + loaded)
return loaded
})
} else {
console.log('nothing to load')
return 0
}
})
}
function aggregateNames() {
var aggregateCollName = 'playlist-names'
console.log('aggregating names')
return MongoClient.connect(config.db_url)
.then(function(db) {
return db.collection(aggregateCollName).remove().then(function() {
console.log(aggregateCollName, ' removed')
return db
})
})
.then(function(db) {
return db.collection('playlists').aggregate([{
$group: {
_id: '$id',
id: {
$first: '$id'
},
owner: {
$first: '$owner.id'
},
names: {
$addToSet: '$name'
}
}
}, {
$out: aggregateCollName
}]).toArray().then(function() {
console.log('aggregated')
})
})
}
function discoverWeeklyFilter(playlists) {
return playlists.filter(function(playlist) {
return playlist.owner.id === 'spotifydiscover'
})
}
function loadAllUsersPlaylists(rootPlaylists) {
var allPlaylists = rootPlaylists
var cursor
function iterateCursor() {
return cursor.hasNext().then(function(hasNext) {
if (hasNext) {
return cursor.next().then(function(user) {
return extractor.loadUserPlaylists(
spotify.userApi(user),
user.id,
user.displayName,
discoverWeeklyFilter)
}).then(function(playlists) {
allPlaylists = allPlaylists.concat(playlists)
return iterateCursor()
})
}
})
}
return MongoClient.connect(config.db_url)
.then(function(db) {
cursor = db.collection('users').find()
return iterateCursor().then(function() {
return allPlaylists
})
})
}
function load() {
status.start()
extractor.loadUserPlaylists(spotify.rootApi, 'spotify', 'spotify')
.then(loadAllUsersPlaylists)
.then(insertPlaylists)
.then(aggregateNames)
.then(status.displayTime)
.then(function() {
console.log('finished')
}, status.reportError)
.then(status.store)
}
module.exports = {}
module.exports.routes = function(router) {
router.get('/load', function(req, res) {
if (req.query.secret !== config.client_secret) {
res.status(403)
res.send('Unauthorized')
} else {
load()
res.redirect('/loadstatus')
}
})
router.get('/loadstatus', function(req, res) {
status.load().then(function(status) {
if (status.err) {
res.status(500)
}
res.json(status)
})
})
}