diff --git a/LICENSE.md b/LICENSE.md
new file mode 100644
index 0000000..47a807e
--- /dev/null
+++ b/LICENSE.md
@@ -0,0 +1,20 @@
+# MIT License
+
+Copyright © 2018 Between Our Worlds
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..08920e3
--- /dev/null
+++ b/README.md
@@ -0,0 +1,30 @@
+# kitsu-downloader
+
+This tool downloads the anime and character data from [Kitsu](https://kitsu.io).
+Linked Data can then be generated from this data via the corresponding [rules](https://github.com/betweenourworlds/generation-rules).
+
+## Installation
+
+1. Clone this repo
+2. Navigate into the folder: `cd kitsu-downloader`
+3. Install Node dependencies: `npm i`
+4. Link the bin: `npm link`
+
+## Usage
+
+The tool takes the following arguments
+
+- `-V, --version`: output the version number
+- `-e, --entity [entity]`: Choose between "anime" and "character"
+- `-s, --start [start]`: Number of start page
+- `-t, --stop [stop]`: Number of stop page
+- `-o, --output [output]`: Path to output folder (default: "")
+- `-v, --verbose`: Make the application more talkative
+- `-h, --help`: output usage information
+
+When you execute `kitsu-downloader -e anime -s 0 -t 199 -o test`,
+the first 200 pages of anime data will be downloaded and the 200 files will be put in the folder `test`.
+
+## License
+
+2018 Between Our Worlds, [MIT License](https://github.com/betweenourworlds/kitsu-downloader/blob/master/LICENSE.md)
\ No newline at end of file
diff --git a/bin/cli.js b/bin/cli.js
new file mode 100755
index 0000000..a06a8fb
--- /dev/null
+++ b/bin/cli.js
@@ -0,0 +1,44 @@
+#! /usr/bin/env node
+
+const program = require('commander');
+const logger = require('../lib/logger');
+const AnimeDownloader = require('../lib/animeDownloader');
+const CharacterDownloader = require('../lib/characterDownloader');
+const Writer = require('../lib/writer');
+
+program
+ .version('0.0.1')
+ .option('-e, --entity [entity]', 'Choose between "anime" and "character"')
+ .option('-s, --start [start]', 'Number of start page')
+ .option('-t, --stop [stop]', 'Number of stop page')
+ .option('-o, --output [output]', 'Path to output folder', '')
+ .option('-v, --verbose', 'Make the application more talkative', '')
+ .parse(process.argv);
+
+if (program.verbose) {
+ //todo set level of logger
+}
+
+if (program.entity === undefined || program.start === undefined || program.start === undefined) {
+ logger.error('Not all required parameters are provided.');
+ process.exit(1);
+} else {
+ if (program.output === undefined) {
+ program.output = process.cwd();
+ }
+
+ const writer = new Writer(program.output);
+ let Downloader;
+
+ if (program.entity === 'anime') {
+ Downloader = AnimeDownloader;
+ } else if (program.entity === 'character') {
+ Downloader = CharacterDownloader;
+ } else {
+ logger.error(`-e expects the value "anime" or "character"`);
+ process.exit(1);
+ }
+
+ const downloader = new Downloader(writer);
+ downloader.downloadRecursive(parseInt(program.start), parseInt(program.stop));
+}
\ No newline at end of file
diff --git a/lib/animeDownloader.js b/lib/animeDownloader.js
new file mode 100644
index 0000000..a254716
--- /dev/null
+++ b/lib/animeDownloader.js
@@ -0,0 +1,332 @@
+const Kitsu = require('kitsu');
+const https = require('https');
+const Q = require('q');
+const request = require('sync-request');
+const validator = require('validator');
+const logger = require('./logger');
+
+class AnimeDownloader {
+
+ constructor(writer) {
+ this.writer = writer;
+ this.kitsu = new Kitsu();
+ }
+
+ downloadRecursive(currentPage, stopPage) {
+ logger.debug(`working on page ${currentPage}`);
+
+ this._download(20, currentPage * 20).then(res => {
+ let filenameAnime = `anime-${currentPage}.json`;
+ let filenameStreams = `streams-${currentPage}.json`;
+ let filenameEpisodes = `episodes-${currentPage}.json`;
+ let filenameSeasons = `seasons-${currentPage}.json`;
+
+ const data = [{
+ data: res.anime,
+ path: filenameAnime
+ }, {
+ data: res.streams,
+ path: filenameStreams
+ }, {
+ data: res.episodes,
+ path: filenameEpisodes
+ }, {
+ data: res.seasons,
+ path: filenameSeasons
+ }];
+
+ this.writer.write(data).then(() => {
+ if (currentPage !== stopPage) {
+ this.downloadRecursive(currentPage + 1, stopPage);
+ }
+ });
+ }).catch(e => {
+ logger.error(`error during ${currentPage}. Retrying...`);
+ this.downloadRecursive(currentPage, stopPage);
+ });
+ }
+
+ _download(limit, offset) {
+ let deferred = Q.defer();
+ let promises = [];
+ let streamsArray = [];
+ let episodesArray = [];
+
+ this.kitsu.get('anime', {
+ page: { limit, offset }
+ }).then(res => {
+ res.data.forEach(anime => {
+ anime.otherWebsites = [];
+ let animeDeferred = Q.defer();
+ promises.push(animeDeferred.promise);
+ this._deleteUnwantedAnimeData(anime);
+
+ let id = anime.id;
+
+ //console.log(`anime id = ${id}`);
+
+ //parse streaming links
+ let streamDeferred = Q.defer();
+ let streamingLinksURL = `https://kitsu.io/api/edge/anime/${id}/streaming-links?include=streamer`;
+
+ https.get(streamingLinksURL, streams => {
+ streams.setEncoding('utf8');
+ let rawData = '';
+ streams.on('data', (chunk) => { rawData += chunk; });
+ streams.on('end', () => {
+ try {
+ const parsedData = JSON.parse(rawData);
+ parsedData.data.forEach(s => {
+ let service = this._getStreamService(parsedData.included, s.relationships.streamer.data.id);
+
+ if (service && this._isValidServiceURL(s.attributes.url)) {
+ streamsArray.push({
+ service,
+ url: s.attributes.url,
+ slug: anime.attributes.slug
+ });
+ }
+ anime.otherWebsites.push(s.attributes.url);
+ });
+
+ streamDeferred.resolve();
+ } catch (e) {
+ //console.error(e.message);
+ deferred.reject(e);
+ }
+ });
+ });
+
+ //parse episodes
+ let episodePromise = this._recursiveEpisodes(`https://kitsu.io/api/edge/anime/${id}/episodes`, anime);
+
+ episodePromise.then(a => {
+ //console.log(a.length);
+ episodesArray = episodesArray.concat(a);
+ });
+
+ //parse mappings to other websites
+ let mappingDeferred = Q.defer();
+ let mappingsURL = `https://kitsu.io/api/edge/anime/${id}/mappings`;
+
+ https.get(mappingsURL, mappings => {
+ mappings.setEncoding('utf8');
+ let rawData = '';
+ mappings.on('data', (chunk) => { rawData += chunk; });
+ mappings.on('end', () => {
+ try {
+ const parsedData = JSON.parse(rawData);
+ parsedData.data.forEach(m => {
+ let website = this._convertMappingToWebsiteURL(m);
+
+ if (website) {
+ anime.otherWebsites.push(website);
+ }
+ });
+
+ mappingDeferred.resolve();
+ } catch (e) {
+ //console.error(e.message);
+ deferred.reject(e);
+ }
+ });
+ });
+
+ //parse characters to other websites
+ let characterDeferred = Q.defer();
+ let characterURL = `https://kitsu.io/api/edge/anime/${id}?include=characters.character`;
+
+ https.get(characterURL, res => {
+ res.setEncoding('utf8');
+ let rawData = '';
+
+ res.on('data', (chunk) => { rawData += chunk; });
+ res.on('end', () => {
+ try {
+ const parsedData = JSON.parse(rawData);
+
+ if (parsedData.included) {
+ anime.characters = [];
+
+ parsedData.included.forEach(item => {
+ if (item.type === 'characters') {
+ anime.characters.push(item.attributes.slug);
+ }
+ });
+ } else {
+ logger.warn(`no characters found for anime with id "${anime.id}"`);
+ }
+
+ characterDeferred.resolve();
+ } catch (e) {
+ //console.error(e.message);
+ logger.error(`error during processing of characters`);
+ deferred.reject(e);
+ }
+ });
+ });
+
+ //characterDeferred.resolve();
+
+ Q.all([mappingDeferred.promise, streamDeferred.promise, episodePromise, characterDeferred.promise]).then(() => {
+ animeDeferred.resolve();
+ });
+ });
+
+ Q.all(promises).then(() => {
+ deferred.resolve({
+ anime: res,
+ streams: streamsArray,
+ episodes: episodesArray,
+ seasons: this._getSeasonsFromEpisodes(episodesArray),
+ });
+ });
+ }).catch(e => {
+ logger.error(`retrieving data with offset ${offset} failed.`);
+ deferred.reject(e);
+ });
+
+ return deferred.promise;
+ }
+
+ _deleteUnwantedAnimeData(anime){
+ delete anime.relationships;
+ delete anime.attributes.ratingFrequencies;
+ delete anime.attributes.userCount;
+ delete anime.attributes.favoritesCount;
+ delete anime.attributes.popularityRank;
+ delete anime.attributes.ratingRank;
+ delete anime.attributes.averageRating;
+ delete anime.links;
+ }
+
+ _convertMappingToWebsiteURL(m){
+ switch (m.attributes.externalSite) {
+ case 'myanimelist/anime':
+ return `https://myanimelist.net/anime/${m.attributes.externalId}`;
+
+ case 'thetvdb/series':
+ return `http://thetvdb.com/?tab=series&id=${m.attributes.externalId}`;
+
+ case 'anidb':
+ return `https://anidb.net/perl-bin/animedb.pl?show=anime&aid=${m.attributes.externalId}`;
+
+ case 'animenewsnetwork':
+ return `https://www.animenewsnetwork.com/encyclopedia/anime.php?id=${m.attributes.externalId}`;
+
+ case 'anilist':
+ return `https://anilist.co/${m.attributes.externalId}`;
+
+ case 'trakt':
+ let text = request('GET', `https://trakt.tv/shows/${m.attributes.externalId}`, {followRedirects: false}).body.toString('utf-8');
+ text = text.replace(`You are being redirected.`, '');
+
+ if (validator.isURL(text)) {
+ return text;
+ } else {
+ logger.debug(`creating Trakt URL failed for ID ${m.attributes.externalId}.`);
+ return null;
+ }
+
+ default:
+ logger.warn(`creating website URL: ${m.attributes.externalSite} was not found. ID is ${m.attributes.externalId}.`);
+ return null;
+ }
+ }
+
+ _isValidServiceURL(url) {
+ return url.indexOf('http://') !== -1 || url.indexOf('https://') !== -1;
+ }
+
+ _getStreamService(data, id) {
+ let i = 0;
+
+ while (i < data.length && data[i].id !== id) {
+ i ++;
+ }
+
+ if (i < data.length) {
+ return data[i].attributes.siteName.toLowerCase();
+ } else {
+ return null;
+ }
+ }
+
+ _recursiveEpisodes(url, anime) {
+ let episodesArray = [];
+ const deferred = Q.defer();
+
+ https.get(url, episodes => {
+ episodes.setEncoding('utf8');
+ let rawData = '';
+ episodes.on('data', (chunk) => { rawData += chunk; });
+ episodes.on('end', () => {
+ let parsedData;
+
+ try {
+ parsedData = JSON.parse(rawData);
+
+ if (parsedData.data) {
+ parsedData.data.forEach(e => {
+ e.anime_id = anime.id;
+ e.anime_slug = anime.attributes.slug;
+
+ delete e.relationships;
+ delete e.links;
+ });
+
+ //console.log(url + ' before ' + episodesArray.length);
+ episodesArray = episodesArray.concat(parsedData.data);
+ //console.log(url + ' after ' + episodesArray.length);
+
+ if (parsedData.links.next) {
+ this._recursiveEpisodes(parsedData.links.next, anime).then(a => {
+ deferred.resolve(episodesArray.concat(a));
+ });
+ } else {
+ deferred.resolve(episodesArray);
+ }
+ } else {
+ logger.debug(`No episodes are available for ${anime.attributes.slug}`);
+ deferred.resolve(episodesArray);
+ }
+ } catch (e) {
+ logger.debug(anime.attributes.slug);
+ logger.error(e.message);
+ deferred.reject(e);
+ }
+ });
+ });
+
+ return deferred.promise;
+ }
+
+ _getSeasonsFromEpisodes(episodes) {
+ const seasons = [];
+ const hashes = [];
+
+ episodes.forEach(episode => {
+ if (episode.attributes.seasonNumber) {
+ const hash = episode.attributes.seasonNumber + ' - ' + episode.anime_id;
+
+ if (hashes.indexOf(hash) === -1) {
+ seasons.push(
+ {
+ seasonNumber: episode.attributes.seasonNumber,
+ anime: {
+ id: episode.anime_id,
+ slug: episode.anime_slug
+ }
+ }
+ );
+
+ hashes.push(hash);
+ }
+ }
+ });
+
+ return seasons;
+ }
+}
+
+module.exports = AnimeDownloader;
diff --git a/lib/characterDownloader.js b/lib/characterDownloader.js
new file mode 100644
index 0000000..f4bc8f0
--- /dev/null
+++ b/lib/characterDownloader.js
@@ -0,0 +1,71 @@
+/**
+ * author: Pieter Heyvaert (pheyvaer.heyvaert@ugent.be)
+ * Ghent University - imec - IDLab
+ */
+
+/**
+ * author: Pieter Heyvaert (pheyvaer.heyvaert@ugent.be)
+ * Ghent University - imec - IDLab
+ */
+
+const Q = require('q');
+const Kitsu = require('kitsu');
+const logger = require('./logger');
+
+class CharacterDownloader {
+
+ constructor(writer) {
+ this.writer = writer;
+ this.kitsu = new Kitsu();
+ }
+
+ downloadRecursive(currentPage, stopPage) {
+ logger.debug(`working on page ${currentPage}`);
+
+ this._download(20, currentPage * 20).then(res => {
+ //console.log(res);
+
+ let filenameCharacter = `character-${currentPage}.json`;
+
+ const data = [{
+ data: res,
+ path: filenameCharacter
+ }];
+
+ this.writer.write(data).then(() => {
+ if (currentPage !== stopPage) {
+ this.downloadRecursive(currentPage + 1, stopPage);
+ }
+ });
+ }).catch(e => {
+ logger.error(`error during ${currentPage}: ${e.message}. Retrying...`);
+ this.downloadRecursive(currentPage, stopPage);
+ });
+ }
+
+ _download(limit, offset) {
+ const deferred = Q.defer();
+
+ this.kitsu.get('characters', {
+ page: { limit, offset }
+ }).then(res => {
+ res.data.forEach(character => {
+ this._deleteUnwantedData(character);
+ });
+
+ deferred.resolve(res);
+ });
+
+ return deferred.promise;
+ }
+
+ _deleteUnwantedData(character) {
+ delete character.relationships;
+ delete character.attributes.createdAt;
+ delete character.attributes.updatedAt;
+ delete character.type;
+ delete character.links;
+ }
+}
+
+module.exports = CharacterDownloader;
\ No newline at end of file
diff --git a/lib/downloader.js b/lib/downloader.js
deleted file mode 100644
index 8b5104e..0000000
--- a/lib/downloader.js
+++ /dev/null
@@ -1,253 +0,0 @@
-const Kitsu = require('kitsu');
-const https = require('https');
-const Q = require('q');
-const request = require('sync-request');
-const validator = require('validator');
-
-const kitsu = new Kitsu();
-
-function downloadAnime(limit, offset) {
- let deferred = Q.defer();
- let promises = [];
- let streamsArray = [];
- let episodesArray = [];
-
- kitsu.get('anime', {
- page: { limit, offset }
- }).then(res => {
- res.data.forEach(anime => {
- anime.otherWebsites = [];
- let animeDeferred = Q.defer();
- promises.push(animeDeferred.promise);
- deleteUnwantedData(anime);
-
- let id = anime.id;
-
- //console.log(`anime id = ${id}`);
-
- //parse streaming links
- let streamDeferred = Q.defer();
- let streamingLinksURL = `https://kitsu.io/api/edge/anime/${id}/streaming-links?include=streamer`;
- https.get(streamingLinksURL, streams => {
- streams.setEncoding('utf8');
- let rawData = '';
- streams.on('data', (chunk) => { rawData += chunk; });
- streams.on('end', () => {
- try {
- const parsedData = JSON.parse(rawData);
- parsedData.data.forEach(s => {
- let service = getStreamService(parsedData.included, s.relationships.streamer.data.id);
-
- if (service && isValidServiceURL(s.attributes.url)) {
- streamsArray.push({
- service,
- url: s.attributes.url,
- slug: anime.attributes.slug
- });
- }
- anime.otherWebsites.push(s.attributes.url);
- });
-
- streamDeferred.resolve();
- } catch (e) {
- //console.error(e.message);
- deferred.reject(e);
- }
- });
- });
-
- //parse episodes
- let episodePromise = recursiveEpisodes(`https://kitsu.io/api/edge/anime/${id}/episodes`, anime);
-
- episodePromise.then(a => {
- //console.log(a.length);
- episodesArray = episodesArray.concat(a);
- });
-
- //parse mappings to other websites
- let mappingDeferred = Q.defer();
- let mappingsURL = `https://kitsu.io/api/edge/anime/${id}/mappings`;
- https.get(mappingsURL, mappings => {
- mappings.setEncoding('utf8');
- let rawData = '';
- mappings.on('data', (chunk) => { rawData += chunk; });
- mappings.on('end', () => {
- try {
- const parsedData = JSON.parse(rawData);
- parsedData.data.forEach(m => {
- let website = convertMappingToWebsiteURL(m);
-
- if (website) {
- anime.otherWebsites.push(website);
- }
- });
-
- mappingDeferred.resolve();
- } catch (e) {
- //console.error(e.message);
- deferred.reject(e);
- }
- });
- });
-
- Q.all([mappingDeferred.promise, streamDeferred.promise, episodePromise]).then(() => {
- animeDeferred.resolve();
- });
- });
-
- Q.all(promises).then(() => {
- deferred.resolve({
- anime: res,
- streams: streamsArray,
- episodes: episodesArray,
- seasons: getSeasonsFromEpisodes(episodesArray)
- });
- });
- }).catch(e => {
- console.error(`retrieving data with offset ${offset} failed.`);
- deferred.reject(e);
- });
-
- return deferred.promise;
-}
-
-function deleteUnwantedData(anime){
- delete anime.relationships;
- delete anime.attributes.ratingFrequencies;
- delete anime.attributes.userCount;
- delete anime.attributes.favoritesCount;
- delete anime.attributes.popularityRank;
- delete anime.attributes.ratingRank;
- delete anime.attributes.averageRating;
- delete anime.links;
-}
-
-function convertMappingToWebsiteURL(m){
- switch (m.attributes.externalSite) {
- case 'myanimelist/anime':
- return `https://myanimelist.net/anime/${m.attributes.externalId}`;
-
- case 'thetvdb/series':
- return `http://thetvdb.com/?tab=series&id=${m.attributes.externalId}`;
-
- case 'anidb':
- return `https://anidb.net/perl-bin/animedb.pl?show=anime&aid=${m.attributes.externalId}`;
-
- case 'animenewsnetwork':
- return `https://www.animenewsnetwork.com/encyclopedia/anime.php?id=${m.attributes.externalId}`;
-
- case 'anilist':
- return `https://anilist.co/${m.attributes.externalId}`;
-
- case 'trakt':
- let text = request('GET', `https://trakt.tv/shows/${m.attributes.externalId}`, {followRedirects: false}).body.toString('utf-8');
- text = text.replace(`You are being redirected.`, '');
-
- if (validator.isURL(text)) {
- return text;
- } else {
- console.error(`creating Trakt URL failed for ID ${m.attributes.externalId}.`);
- return null;
- }
-
- default:
- console.log(`creating website URL: ${m.attributes.externalSite} was not found. ID is ${m.attributes.externalId}.`);
- return null;
- }
-}
-
-function isValidServiceURL(url) {
- return url.indexOf('http://') !== -1 || url.indexOf('https://') !== -1;
-}
-
-function getStreamService(data, id) {
- let i = 0;
-
- while (i < data.length && data[i].id !== id) {
- i ++;
- }
-
- if (i < data.length) {
- return data[i].attributes.siteName.toLowerCase();
- } else {
- return null;
- }
-}
-
-function recursiveEpisodes(url, anime) {
- let episodesArray = [];
- const deferred = Q.defer();
-
- https.get(url, episodes => {
- episodes.setEncoding('utf8');
- let rawData = '';
- episodes.on('data', (chunk) => { rawData += chunk; });
- episodes.on('end', () => {
- let parsedData;
-
- try {
- parsedData = JSON.parse(rawData);
-
- if (parsedData.data) {
- parsedData.data.forEach(e => {
- e.anime_id = anime.id;
- e.anime_slug = anime.attributes.slug;
-
- delete e.relationships;
- delete e.links;
- });
-
- //console.log(url + ' before ' + episodesArray.length);
- episodesArray = episodesArray.concat(parsedData.data);
- //console.log(url + ' after ' + episodesArray.length);
-
- if (parsedData.links.next) {
- recursiveEpisodes(parsedData.links.next, anime).then(a => {
- deferred.resolve(episodesArray.concat(a));
- });
- } else {
- deferred.resolve(episodesArray);
- }
- } else {
- console.error(`No episodes are available for ${anime.attributes.slug}`);
- deferred.resolve(episodesArray);
- }
- } catch (e) {
- console.log(anime.attributes.slug);
- console.error(e.message);
- deferred.reject(e);
- }
- });
- });
-
- return deferred.promise;
-}
-
-function getSeasonsFromEpisodes(episodes) {
- const seasons = [];
- const hashes = [];
-
- episodes.forEach(episode => {
- if (episode.attributes.seasonNumber) {
- const hash = episode.attributes.seasonNumber + ' - ' + episode.anime_id;
-
- if (hashes.indexOf(hash) === -1) {
- seasons.push(
- {
- seasonNumber: episode.attributes.seasonNumber,
- anime: {
- id: episode.anime_id,
- slug: episode.anime_slug
- }
- }
- );
-
- hashes.push(hash);
- }
- }
- });
-
- return seasons;
-}
-
-module.exports = downloadAnime;
diff --git a/lib/logger.js b/lib/logger.js
new file mode 100644
index 0000000..d87d6b5
--- /dev/null
+++ b/lib/logger.js
@@ -0,0 +1,14 @@
+/**
+ * author: Pieter Heyvaert (pheyvaer.heyvaert@ugent.be)
+ * Ghent University - imec - IDLab
+ */
+
+const winston = require('winston');
+
+const logger = winston.createLogger({
+ transports: [
+ new winston.transports.Console()
+ ]
+});
+
+module.exports = logger;
\ No newline at end of file
diff --git a/lib/writer.js b/lib/writer.js
new file mode 100644
index 0000000..22d4b39
--- /dev/null
+++ b/lib/writer.js
@@ -0,0 +1,39 @@
+/**
+ * author: Pieter Heyvaert (pheyvaer.heyvaert@ugent.be)
+ * Ghent University - imec - IDLab
+ */
+
+const Q = require('q');
+const logger = require('./logger');
+const fs = require('fs');
+
+class Writer {
+
+ constructor(outputFolderPath) {
+ this.outputFolderPath = outputFolderPath;
+ }
+
+ write(results) {
+ const promises = [];
+
+ results.forEach(result => {
+ const deferred = Q.defer();
+ promises.push(deferred.promise);
+ const filePath = `${this.outputFolderPath}/${result.path}`;
+
+ fs.writeFile(filePath, JSON.stringify(result.data, null, 2), err => {
+ if (err) {
+ logger.error(`Something went wrong! We couldn't write the content to file '${filePath}'.`);
+ logger.error(err);
+ deferred.reject(err);
+ } else {
+ deferred.resolve();
+ }
+ });
+ });
+
+ return Q.all(promises);
+ }
+}
+
+module.exports = Writer;
\ No newline at end of file
diff --git a/package-lock.json b/package-lock.json
index 0ab34e5..bf5d93f 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,6 +9,14 @@
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
"integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
},
+ "async": {
+ "version": "2.6.1",
+ "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz",
+ "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==",
+ "requires": {
+ "lodash": "^4.17.10"
+ }
+ },
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -19,8 +27,8 @@
"resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.23.0.tgz",
"integrity": "sha1-CpSJ8UTecO+zzkMArM2zKeL8VDs=",
"requires": {
- "core-js": "2.4.1",
- "regenerator-runtime": "0.10.5"
+ "core-js": "^2.4.0",
+ "regenerator-runtime": "^0.10.0"
}
},
"camelcase": {
@@ -38,7 +46,57 @@
"resolved": "https://registry.npmjs.org/client-oauth2/-/client-oauth2-4.1.0.tgz",
"integrity": "sha1-t284FtIf9/5Lfm62nqujPiixdLY=",
"requires": {
- "popsicle": "9.1.0"
+ "popsicle": "^9.1.0"
+ }
+ },
+ "color": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz",
+ "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==",
+ "requires": {
+ "color-convert": "^1.9.1",
+ "color-string": "^1.5.2"
+ }
+ },
+ "color-convert": {
+ "version": "1.9.3",
+ "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+ "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+ "requires": {
+ "color-name": "1.1.3"
+ }
+ },
+ "color-name": {
+ "version": "1.1.3",
+ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+ "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
+ },
+ "color-string": {
+ "version": "1.5.3",
+ "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.3.tgz",
+ "integrity": "sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw==",
+ "requires": {
+ "color-name": "^1.0.0",
+ "simple-swizzle": "^0.2.2"
+ }
+ },
+ "colornames": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/colornames/-/colornames-1.1.1.tgz",
+ "integrity": "sha1-+IiQMGhcfE/54qVZ9Qd+t2qBb5Y="
+ },
+ "colors": {
+ "version": "1.3.2",
+ "resolved": "https://registry.npmjs.org/colors/-/colors-1.3.2.tgz",
+ "integrity": "sha512-rhP0JSBGYvpcNQj4s5AdShMeE5ahMop96cTeDl/v9qQQm2fYClE2QXZRi8wLzc+GmXSxdIqqbOIAhyObEXDbfQ=="
+ },
+ "colorspace": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.1.tgz",
+ "integrity": "sha512-pI3btWyiuz7Ken0BWh9Elzsmv2bM9AhA7psXib4anUXy/orfZ/E0MbQwhSOG/9L8hLlalqrU0UhOuqxW1YjmVw==",
+ "requires": {
+ "color": "3.0.x",
+ "text-hex": "1.0.x"
}
},
"combined-stream": {
@@ -46,7 +104,7 @@
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz",
"integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=",
"requires": {
- "delayed-stream": "1.0.0"
+ "delayed-stream": "~1.0.0"
}
},
"command-exists": {
@@ -54,14 +112,19 @@
"resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.2.tgz",
"integrity": "sha1-EoGcZPr5VEbsCuB/5sr7brNwiyI="
},
+ "commander": {
+ "version": "2.19.0",
+ "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz",
+ "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg=="
+ },
"concat-stream": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz",
"integrity": "sha1-CqxmL9Ur54lk1VMvaUeE5wEQrPc=",
"requires": {
- "inherits": "2.0.3",
- "readable-stream": "2.3.3",
- "typedarray": "0.0.6"
+ "inherits": "^2.0.3",
+ "readable-stream": "^2.2.2",
+ "typedarray": "^0.0.6"
}
},
"core-js": {
@@ -84,7 +147,7 @@
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
"integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=",
"requires": {
- "mimic-response": "1.0.0"
+ "mimic-response": "^1.0.0"
}
},
"delayed-stream": {
@@ -92,19 +155,52 @@
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
},
+ "diagnostics": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz",
+ "integrity": "sha512-8wn1PmdunLJ9Tqbx+Fx/ZEuHfJf4NKSN2ZBj7SJC/OWRWha843+WsTjqMe1B5E3p28jqBlp+mJ2fPVxPyNgYKQ==",
+ "requires": {
+ "colorspace": "1.1.x",
+ "enabled": "1.0.x",
+ "kuler": "1.0.x"
+ }
+ },
"duplexer3": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz",
"integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI="
},
+ "enabled": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz",
+ "integrity": "sha1-ll9lE9LC0cX0ZStkouM5ZGf8L5M=",
+ "requires": {
+ "env-variable": "0.0.x"
+ }
+ },
+ "env-variable": {
+ "version": "0.0.5",
+ "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz",
+ "integrity": "sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA=="
+ },
+ "fast-safe-stringify": {
+ "version": "2.0.6",
+ "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.6.tgz",
+ "integrity": "sha512-q8BZ89jjc+mz08rSxROs8VsrBBcn1SIw1kq9NjolL509tkABRk9io01RAjSaEv1Xb2uFLt8VtRiZbGp5H8iDtg=="
+ },
+ "fecha": {
+ "version": "2.3.3",
+ "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz",
+ "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg=="
+ },
"form-data": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.2.0.tgz",
"integrity": "sha1-ml47kpX5gLJiPPZPojixTOvKcHs=",
"requires": {
- "asynckit": "0.4.0",
- "combined-stream": "1.0.5",
- "mime-types": "2.1.15"
+ "asynckit": "^0.4.0",
+ "combined-stream": "^1.0.5",
+ "mime-types": "^2.1.12"
}
},
"get-port": {
@@ -122,20 +218,20 @@
"resolved": "https://registry.npmjs.org/got/-/got-7.1.0.tgz",
"integrity": "sha512-Y5WMo7xKKq1muPsxD+KmrR8DH5auG7fBdDVueZwETwV6VytKyU9OX/ddpq2/1hp1vIPvVb4T81dKQz3BivkNLw==",
"requires": {
- "decompress-response": "3.3.0",
- "duplexer3": "0.1.4",
- "get-stream": "3.0.0",
- "is-plain-obj": "1.1.0",
- "is-retry-allowed": "1.1.0",
- "is-stream": "1.1.0",
- "isurl": "1.0.0",
- "lowercase-keys": "1.0.0",
- "p-cancelable": "0.3.0",
- "p-timeout": "1.2.0",
- "safe-buffer": "5.1.1",
- "timed-out": "4.0.1",
- "url-parse-lax": "1.0.0",
- "url-to-options": "1.0.1"
+ "decompress-response": "^3.2.0",
+ "duplexer3": "^0.1.4",
+ "get-stream": "^3.0.0",
+ "is-plain-obj": "^1.1.0",
+ "is-retry-allowed": "^1.0.0",
+ "is-stream": "^1.0.0",
+ "isurl": "^1.0.0-alpha5",
+ "lowercase-keys": "^1.0.0",
+ "p-cancelable": "^0.3.0",
+ "p-timeout": "^1.1.1",
+ "safe-buffer": "^5.0.1",
+ "timed-out": "^4.0.0",
+ "url-parse-lax": "^1.0.0",
+ "url-to-options": "^1.0.1"
}
},
"has-symbol-support-x": {
@@ -148,7 +244,7 @@
"resolved": "https://registry.npmjs.org/has-to-string-tag-x/-/has-to-string-tag-x-1.3.0.tgz",
"integrity": "sha512-Fu9Nwv8/VNJMvKjkldzXHO+yeX+TCelwUQ9dGW2LrAfHfHi6zVqQt+Qjilf0qGHvpl6Fap6o8aDhWhMt5hY/1g==",
"requires": {
- "has-symbol-support-x": "1.3.0"
+ "has-symbol-support-x": "^1.3.0"
}
},
"http-basic": {
@@ -156,9 +252,9 @@
"resolved": "https://registry.npmjs.org/http-basic/-/http-basic-2.5.1.tgz",
"integrity": "sha1-jORHvbW2xXf4pj4/p4BW7Eu02/s=",
"requires": {
- "caseless": "0.11.0",
- "concat-stream": "1.6.0",
- "http-response-object": "1.1.0"
+ "caseless": "~0.11.0",
+ "concat-stream": "^1.4.6",
+ "http-response-object": "^1.0.0"
}
},
"http-response-object": {
@@ -171,6 +267,11 @@
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
},
+ "is-arrayish": {
+ "version": "0.3.2",
+ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
+ "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="
+ },
"is-object": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.1.tgz",
@@ -201,8 +302,8 @@
"resolved": "https://registry.npmjs.org/isurl/-/isurl-1.0.0.tgz",
"integrity": "sha512-1P/yWsxPlDtn7QeRD+ULKQPaIaN6yF368GZ2vDfv0AL0NwpStafjWCDDdn0k8wgFMWpVAqG7oJhxHnlud42i9w==",
"requires": {
- "has-to-string-tag-x": "1.3.0",
- "is-object": "1.0.1"
+ "has-to-string-tag-x": "^1.2.0",
+ "is-object": "^1.0.1"
}
},
"kitsu": {
@@ -210,12 +311,37 @@
"resolved": "https://registry.npmjs.org/kitsu/-/kitsu-1.2.1.tgz",
"integrity": "sha1-x7xJ0Y6JymMPNfK4T6+Hx4cRorI=",
"requires": {
- "babel-runtime": "6.23.0",
- "camelcase": "4.1.0",
- "client-oauth2": "4.1.0",
- "decamelize": "1.2.0",
- "got": "7.1.0",
- "pluralize": "5.0.0"
+ "babel-runtime": "~6.23.0",
+ "camelcase": "~4.1.0",
+ "client-oauth2": "~4.1.0",
+ "decamelize": "~1.2.0",
+ "got": "~7.1.0",
+ "pluralize": "~5.0.0"
+ }
+ },
+ "kuler": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz",
+ "integrity": "sha512-J9nVUucG1p/skKul6DU3PUZrhs0LPulNaeUOox0IyXDi8S4CztTHs1gQphhuZmzXG7VOQSf6NJfKuzteQLv9gQ==",
+ "requires": {
+ "colornames": "^1.1.1"
+ }
+ },
+ "lodash": {
+ "version": "4.17.11",
+ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
+ "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
+ },
+ "logform": {
+ "version": "1.10.0",
+ "resolved": "https://registry.npmjs.org/logform/-/logform-1.10.0.tgz",
+ "integrity": "sha512-em5ojIhU18fIMOw/333mD+ZLE2fis0EzXl1ZwHx4iQzmpQi6odNiY/t+ITNr33JZhT9/KEaH+UPIipr6a9EjWg==",
+ "requires": {
+ "colors": "^1.2.1",
+ "fast-safe-stringify": "^2.0.4",
+ "fecha": "^2.3.3",
+ "ms": "^2.1.1",
+ "triple-beam": "^1.2.0"
}
},
"lowercase-keys": {
@@ -233,7 +359,7 @@
"resolved": "https://registry.npmjs.org/make-error-cause/-/make-error-cause-1.2.2.tgz",
"integrity": "sha1-3wOI/NCzeBbf8KX7gQiTl3fcvJ0=",
"requires": {
- "make-error": "1.3.0"
+ "make-error": "^1.2.0"
}
},
"mime-db": {
@@ -246,7 +372,7 @@
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.15.tgz",
"integrity": "sha1-pOv1BkCUVpI3uM9wBGd20J/JKu0=",
"requires": {
- "mime-db": "1.27.0"
+ "mime-db": "~1.27.0"
}
},
"mimic-response": {
@@ -254,6 +380,16 @@
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.0.tgz",
"integrity": "sha1-3z02Uqc/3ta5sLJBRub9BSNTRY4="
},
+ "ms": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+ "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
+ },
+ "one-time": {
+ "version": "0.0.4",
+ "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz",
+ "integrity": "sha1-+M33eISCb+Tf+T46nMN7HkSAdC4="
+ },
"p-cancelable": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.3.0.tgz",
@@ -269,7 +405,7 @@
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-1.2.0.tgz",
"integrity": "sha1-mCD5lDTFgXhotPNICe5SkWYNW2w=",
"requires": {
- "p-finally": "1.0.0"
+ "p-finally": "^1.0.0"
}
},
"pluralize": {
@@ -282,10 +418,10 @@
"resolved": "https://registry.npmjs.org/popsicle/-/popsicle-9.1.0.tgz",
"integrity": "sha1-T5APONV6V07BcO2kBJbjZAgr/2Y=",
"requires": {
- "concat-stream": "1.6.0",
- "form-data": "2.2.0",
- "make-error-cause": "1.2.2",
- "tough-cookie": "2.3.2"
+ "concat-stream": "^1.4.7",
+ "form-data": "^2.0.0",
+ "make-error-cause": "^1.2.1",
+ "tough-cookie": "^2.0.0"
}
},
"prepend-http": {
@@ -303,7 +439,7 @@
"resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
"integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
"requires": {
- "asap": "2.0.6"
+ "asap": "~2.0.3"
}
},
"punycode": {
@@ -326,13 +462,13 @@
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz",
"integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==",
"requires": {
- "core-util-is": "1.0.2",
- "inherits": "2.0.3",
- "isarray": "1.0.0",
- "process-nextick-args": "1.0.7",
- "safe-buffer": "5.1.1",
- "string_decoder": "1.0.3",
- "util-deprecate": "1.0.2"
+ "core-util-is": "~1.0.0",
+ "inherits": "~2.0.3",
+ "isarray": "~1.0.0",
+ "process-nextick-args": "~1.0.6",
+ "safe-buffer": "~5.1.1",
+ "string_decoder": "~1.0.3",
+ "util-deprecate": "~1.0.1"
}
},
"regenerator-runtime": {
@@ -345,12 +481,25 @@
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
"integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
},
+ "simple-swizzle": {
+ "version": "0.2.2",
+ "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
+ "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=",
+ "requires": {
+ "is-arrayish": "^0.3.1"
+ }
+ },
+ "stack-trace": {
+ "version": "0.0.10",
+ "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
+ "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA="
+ },
"string_decoder": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
"integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
"requires": {
- "safe-buffer": "5.1.1"
+ "safe-buffer": "~5.1.0"
}
},
"sync-request": {
@@ -358,24 +507,29 @@
"resolved": "https://registry.npmjs.org/sync-request/-/sync-request-4.1.0.tgz",
"integrity": "sha512-iFbOBWYaznBNbheIKaMkj+3EabpEsXbuwcTVuYkRjoav+Om5L8VXXLIXms0cHxkouXMRCQaSfhfau9/HyIbM2Q==",
"requires": {
- "command-exists": "1.2.2",
- "concat-stream": "1.6.0",
- "get-port": "3.2.0",
- "http-response-object": "1.1.0",
- "then-request": "2.2.0"
+ "command-exists": "^1.2.2",
+ "concat-stream": "^1.6.0",
+ "get-port": "^3.1.0",
+ "http-response-object": "^1.1.0",
+ "then-request": "^2.2.0"
}
},
+ "text-hex": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz",
+ "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="
+ },
"then-request": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/then-request/-/then-request-2.2.0.tgz",
"integrity": "sha1-ZnizL6DKIY/laZgbvYhxtZQGDYE=",
"requires": {
- "caseless": "0.11.0",
- "concat-stream": "1.6.0",
- "http-basic": "2.5.1",
- "http-response-object": "1.1.0",
- "promise": "7.3.1",
- "qs": "6.5.1"
+ "caseless": "~0.11.0",
+ "concat-stream": "^1.4.7",
+ "http-basic": "^2.5.1",
+ "http-response-object": "^1.1.0",
+ "promise": "^7.1.1",
+ "qs": "^6.1.0"
}
},
"timed-out": {
@@ -388,9 +542,14 @@
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz",
"integrity": "sha1-8IH3bkyFcg5sN6X6ztc3FQ2EByo=",
"requires": {
- "punycode": "1.4.1"
+ "punycode": "^1.4.1"
}
},
+ "triple-beam": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz",
+ "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw=="
+ },
"typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
@@ -401,7 +560,7 @@
"resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz",
"integrity": "sha1-evjzA2Rem9eaJy56FKxovAYJ2nM=",
"requires": {
- "prepend-http": "1.0.4"
+ "prepend-http": "^1.0.1"
}
},
"url-to-options": {
@@ -418,6 +577,89 @@
"version": "9.4.0",
"resolved": "https://registry.npmjs.org/validator/-/validator-9.4.0.tgz",
"integrity": "sha512-ftkCYp/7HrGdybVCuwSje07POAd93ksZJpb5GVDBzm8SLKIm3QMJcZugb5dOJsONBoWhIXl0jtoGHTyou3DAgA=="
+ },
+ "winston": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/winston/-/winston-3.1.0.tgz",
+ "integrity": "sha512-FsQfEE+8YIEeuZEYhHDk5cILo1HOcWkGwvoidLrDgPog0r4bser1lEIOco2dN9zpDJ1M88hfDgZvxe5z4xNcwg==",
+ "requires": {
+ "async": "^2.6.0",
+ "diagnostics": "^1.1.1",
+ "is-stream": "^1.1.0",
+ "logform": "^1.9.1",
+ "one-time": "0.0.4",
+ "readable-stream": "^2.3.6",
+ "stack-trace": "0.0.x",
+ "triple-beam": "^1.3.0",
+ "winston-transport": "^4.2.0"
+ },
+ "dependencies": {
+ "process-nextick-args": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+ "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
+ },
+ "readable-stream": {
+ "version": "2.3.6",
+ "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+ "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+ "requires": {
+ "core-util-is": "~1.0.0",
+ "inherits": "~2.0.3",
+ "isarray": "~1.0.0",
+ "process-nextick-args": "~2.0.0",
+ "safe-buffer": "~5.1.1",
+ "string_decoder": "~1.1.1",
+ "util-deprecate": "~1.0.1"
+ }
+ },
+ "string_decoder": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+ "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+ "requires": {
+ "safe-buffer": "~5.1.0"
+ }
+ }
+ }
+ },
+ "winston-transport": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.2.0.tgz",
+ "integrity": "sha512-0R1bvFqxSlK/ZKTH86nymOuKv/cT1PQBMuDdA7k7f0S9fM44dNH6bXnuxwXPrN8lefJgtZq08BKdyZ0DZIy/rg==",
+ "requires": {
+ "readable-stream": "^2.3.6",
+ "triple-beam": "^1.2.0"
+ },
+ "dependencies": {
+ "process-nextick-args": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+ "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
+ },
+ "readable-stream": {
+ "version": "2.3.6",
+ "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+ "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+ "requires": {
+ "core-util-is": "~1.0.0",
+ "inherits": "~2.0.3",
+ "isarray": "~1.0.0",
+ "process-nextick-args": "~2.0.0",
+ "safe-buffer": "~5.1.1",
+ "string_decoder": "~1.1.1",
+ "util-deprecate": "~1.0.1"
+ }
+ },
+ "string_decoder": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+ "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+ "requires": {
+ "safe-buffer": "~5.1.0"
+ }
+ }
+ }
}
}
}
diff --git a/package.json b/package.json
index 13b1293..78ae2c6 100644
--- a/package.json
+++ b/package.json
@@ -1,17 +1,22 @@
{
"name": "kitsu-downloader",
"version": "0.0.1",
- "description": "",
+ "description": "Tool to download the anime and character data from Kitsu.io",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
+ "bin": {
+ "kitsu-downloader": "./bin/cli.js"
+ },
"author": "Pieter Heyvaert",
"license": "MIT",
"dependencies": {
+ "commander": "^2.19.0",
"kitsu": "^1.2.1",
"q": "^1.5.0",
"sync-request": "^4.1.0",
- "validator": "^9.4.0"
+ "validator": "^9.4.0",
+ "winston": "^3.1.0"
}
}
diff --git a/range.js b/range.js
deleted file mode 100644
index aeeff79..0000000
--- a/range.js
+++ /dev/null
@@ -1,67 +0,0 @@
-const fs = require('fs');
-const download = require('./lib/downloader.js');
-
-let start = parseInt(process.argv[2]);
-let stop = parseInt(process.argv[3]);
-let filePrefix = process.argv[4];
-let outputFolder = process.argv[5];
-
-if (start === undefined || stop === undefined || !filePrefix || !outputFolder) {
- console.error(`Incorrect usage. Use 'node index.js '.`);
- process.exit(1);
-}
-
-recursive(start);
-
-function recursive(current) {
- console.log(`working on ${current}`);
-
- download(20, current * 20).then(res => {
- let a = res.anime;
- delete a.links;
-
- let filenameAnime = `${filePrefix}-${current}.json`;
- let pathAnime = `${outputFolder}/${filenameAnime}`;
- let filenameStreams = `${filePrefix}-streams-${current}.json`;
- let pathStreams = `${outputFolder}/${filenameStreams}`;
- let filenameEpisodes = `${filePrefix}-episodes-${current}.json`;
- let pathEpisodes = `${outputFolder}/${filenameEpisodes}`;
- let filenameSeasons = `${filePrefix}-seasons-${current}.json`;
- let pathSeasons = `${outputFolder}/${filenameSeasons}`;
-
- fs.writeFile(pathAnime, JSON.stringify(a, null, 2), err => {
- if (err) {
- console.error(`Something went wrong! We couldn't write the content to file '${pathAnime}'.`);
- console.error(err);
- }
-
- fs.writeFile(pathStreams, JSON.stringify(res.streams, null, 2), err => {
- if (err) {
- console.error(`Something went wrong! We couldn't write the content to file '${pathStreams}'.`);
- console.error(err);
- }
-
- fs.writeFile(pathEpisodes, JSON.stringify(res.episodes, null, 2), err => {
- if (err) {
- console.error(`Something went wrong! We couldn't write the content to file '${pathEpisodes}'.`);
- console.error(err);
- }
-
- fs.writeFile(pathSeasons, JSON.stringify(res.seasons, null, 2), err => {
- if (err) {
- console.error(`Something went wrong! We couldn't write the content to file '${pathSeasons}'.`);
- console.error(err);
- }
-
- if (current !== stop) {
- recursive(current + 1);
- }
- });
- });
- });
- });
- }).catch(e => {
- console.error(`error during ${current}. Retrying...`);
- recursive(current);
- });
-}