Temporal patch

Author: seia-soto

index.js

@@ -14,12 +14,16 @@ const router = new Router()
 
 router.all('/', async ctx => ctx.redirect('https://ohys.seia.io'))
 
-utils.database.autofill()
-utils.routing.autofill(router, functions)
-utils.ohys.automate()
+const initFn = async () => {
+  await utils.database.autofill()
+  await utils.routing.autofill(router, functions)
+  utils.ohys.automate()
 
-app
-  .use(cors(config.app.cors))
-  .use(router.routes())
-  .use(router.allowedMethods())
-  .listen(config.app.port, () => log(`${pkg.name}@v${pkg.version} is listening at port ${config.app.port}.`))
+  app
+    .use(cors(config.app.cors))
+    .use(router.routes())
+    .use(router.allowedMethods())
+    .listen(config.app.port, () => log(`${pkg.name}@v${pkg.version} is listening at port ${config.app.port}.`))
+}
+
+initFn()

utils/crypto/hash/index.js

@@ -0,0 +1 @@
+module.exports.md5 = require('./md5')

utils/crypto/hash/md5.js

@@ -0,0 +1,3 @@
+const crypto = require('crypto')
+
+module.exports = data => crypto.createHash('md5').update(data).digest('hex')

utils/crypto/index.js

@@ -0,0 +1 @@
+module.exports.hash = require('./hash')

utils/database/schemas/animes.js

@@ -1,6 +1,8 @@
 module.exports = async lib => {
   await lib.schema.createTable('animes', table => {
-    table.increments() // NOTE: id int unsigned not null auto_increment primary key
+    table.increments().primary()
+
+    table.string('hash', 32).notNullable()
 
     table.integer('episode').notNullable()
     table.string('series', 256).notNullable()

utils/index.js

@@ -1,3 +1,4 @@
+module.exports.crypto = require('./crypto')
 module.exports.database = require('./database')
 module.exports.ohys = require('./ohys')
 module.exports.routing = require('./routing')

utils/ohys/automate.js

@@ -1,29 +1,13 @@
 const config = require('../../config')
 const log = require('../../log')
-const database = require('../database')
-const data = require('./data')
-const fetch = require('./fetch')
+const database = require('./database')
 
 module.exports = async () => {
-  if (!config.ohys.passUpdateAtStart) {
-    const latestFeed = await fetch.allList()
-    const existingFeed = await database.knex('animes').select('*')
-
-    if (latestFeed.length > existingFeed.length) {
-      await data.insert(latestFeed.slice(0, latestFeed.length - existingFeed.length))
-    }
-  }
+  await database.update(true)
 
   setInterval(async () => {
-    log(`updating database: ${Date.now()}`)
-
-    const latestFeedUpdate = await fetch.list()
-    const existingFeedUpdate = await database.knex('animes').select('*')
-
-    const newItems = await latestFeedUpdate.serialized.filter(item => {
-      return !existingFeedUpdate.find(eItem => eItem.original === item.name)
-    })
+    log('updating database at ' + Date.now())
 
-    await data.insert(newItems)
+    await database.update()
   }, config.ohys.refreshRate)
 }

utils/ohys/data/insert.js

@@ -1,4 +1,5 @@
 const log = require('../../../log')
+const crypto = require('../../crypto')
 const database = require('../../database')
 const patterns = require('../patterns')
 
@@ -16,6 +17,7 @@ module.exports = async items => {
     await database.knex('animes')
       .insert({
         id: null,
+        hash: await crypto.hash.md5(await JSON.stringify(items[i])),
         episode: Number(episode),
         series,
         link: items[i].url,

utils/ohys/database/index.js

@@ -0,0 +1 @@
+module.exports.update = require('./update')

utils/ohys/database/update.js

@@ -0,0 +1,32 @@
+const crypto = require('../../crypto')
+const database = require('../../database')
+const data = require('../data')
+const fetch = require('../fetch')
+// const scopes = require('../scopes')
+
+const update = async freshly => {
+  const existingSources = await database.knex('animes')
+    .select('*')
+
+  let latestFeed = await fetch.list()
+
+  if (freshly) {
+    latestFeed = {
+      serialized: await fetch.allList()
+    }
+  }
+
+  for (let k = 0; k < latestFeed.serialized.length; k++) {
+    const item = latestFeed.serialized[k]
+    const itemStringified = await JSON.stringify(item)
+    const itemHash = await crypto.hash.md5(itemStringified)
+
+    const existingItem = await existingSources.find(item => item.hash === itemHash)
+
+    if (!existingItem) {
+      await data.insert([item])
+    }
+  }
+}
+
+module.exports = update