Merge pull request #121 from DIYgod/master

[pull] master from diygod:master

lib/config.js

@@ -108,6 +108,7 @@ const calculateValue = () => {
             allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true',
             filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
             allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true',
+            mediaProxyKey: envs.MEDIA_PROXY_KEY,
         },
         suffix: envs.SUFFIX,
         titleLengthLimit: Number.parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
@@ -293,6 +294,10 @@ const calculateValue = () => {
         },
         telegram: {
             token: envs.TELEGRAM_TOKEN,
+            session: envs.TELEGRAM_SESSION,
+            apiId: envs.TELEGRAM_API_ID,
+            apiHash: envs.TELEGRAM_API_HASH,
+            maxConcurrentDownloads: envs.TELEGRAM_MAX_CONCURRENT_DOWNLOADS,
         },
         tophub: {
             cookie: envs.TOPHUB_COOKIE,

lib/v2/ciweimao/chapter.js

@@ -0,0 +1,61 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const limit = Number.parseInt(ctx.query.limit) || 10;
+
+    const baseUrl = 'https://wap.ciweimao.com';
+    const chapterUrl = 'https://mip.ciweimao.com';
+
+    const { data: response } = await got(`${baseUrl}/book/${id}`);
+    const $ = cheerio.load(response);
+
+    const firstChapterUrl = $('ul.catalogue-list li a').attr('href');
+    const firstChapterId = firstChapterUrl.substring(firstChapterUrl.lastIndexOf('/') + 1);
+
+    const { data: chapters } = await got(`${chapterUrl}/chapter/${id}/${firstChapterId}`);
+    const $c = cheerio.load(chapters);
+
+    const list = $c('ul.book-chapter li a')
+        .slice(-limit)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                chapterLocked: item.find('h3 i.icon-lock').length > 0,
+                title: item.find('h3').text(),
+                pubDate: timezone(parseDate(item.find('p').text().replace('发布于 ', '')), +8),
+                link: item.attr('href'),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (item.chapterLocked) {
+                    return item;
+                }
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+
+                const content = $('div.read-bd');
+                content.find('span, a').remove();
+                content.find('p').removeAttr('class');
+                item.description = content.html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `刺猬猫 ${$('.book-name').text()}`,
+        link: `${baseUrl}/book/${id}`,
+        description: $('.book-desc div p').text(),
+        image: $('meta[name=image]').attr('content'),
+        item: items,
+    };
+};

lib/v2/ciweimao/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/chapter/:id': ['keocheung'],
+};

lib/v2/ciweimao/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'ciweimao.com': {
+        _name: '刺猬猫',
+        wap: [
+            {
+                title: '章节',
+                docs: 'https://docs.rsshub.app/routes/reading#ci-wei-mao',
+                source: ['/book/:id'],
+                target: '/ciweimao/chapter/:id',
+            },
+        ],
+    },
+};

lib/v2/ciweimao/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/chapter/:id', require('./chapter'));
+};

lib/v2/hbooker/chapter.js

@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const limit = Number.parseInt(ctx.query.limit) || 10;
+
+    const baseUrl = 'https://www.hbooker.com';
+
+    const { data: response } = await got(`${baseUrl}/book/${id}?arr_reverse=1`);
+    const $ = cheerio.load(response);
+
+    const list = $('div.book-chapter-list ul li a')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+
+                const rawDate = $('div.read-hd p span').eq(2).text();
+                item.pubDate = timezone(parseDate(rawDate.replace('更新时间：', '')), +8);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `欢乐书客 ${$('div.book-title h1').text()}`,
+        link: `${baseUrl}/book/${id}`,
+        description: $('div.book-desc').text(),
+        image: $('div.book-cover img').attr('src'),
+        item: items,
+    };
+};

lib/v2/hbooker/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/chapter/:id': ['keocheung'],
+};

lib/v2/hbooker/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'hbooker.com': {
+        _name: '欢乐书客',
+        '.': [
+            {
+                title: '章节',
+                docs: 'https://docs.rsshub.app/routes/reading#huan-le-shu-ke',
+                source: '/book/:id',
+                target: '/hbooker/chapter/:id',
+            },
+        ],
+    },
+};

lib/v2/hbooker/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/chapter/:id', require('./chapter'));
+};

lib/v2/rsshub/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/m/:key/:url': ['TonyRL'],
     '/routes/:lang?': ['DIYgod'],
     '/transform/html/:url/:routeParams': ['ttttmr'],
     '/transform/json/:url/:routeParams': ['ttttmr'],

lib/v2/rsshub/media.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const config = require('@/config').value;
+const { getDomain } = require('tldts');
+const { refererMap } = require('./referer-map');
+
+module.exports = async (ctx) => {
+    if (!config.feature.mediaProxyKey) {
+        ctx.throw(403, 'Internal media proxy is disabled.');
+    }
+
+    const { key } = ctx.params;
+    if (key !== config.feature.mediaProxyKey) {
+        ctx.throw(401, 'Invalid media proxy key.');
+    }
+
+    const url = decodeURIComponent(ctx.params.url);
+    const requestUrl = new URL(url);
+    const { hostname, origin } = requestUrl;
+
+    const domain = getDomain(hostname);
+
+    let referer = refererMap.get(domain);
+    referer ||= origin;
+
+    const { headers } = await got.head(url, {
+        headers: {
+            referer,
+        },
+    });
+
+    const cacheControl = headers['cache-control'];
+    const contentType = headers['content-type'];
+    const contentLength = headers['content-length'];
+
+    if (!contentType.startsWith('image/') || headers.server === 'RSSHub') {
+        return ctx.redirect(url);
+    }
+
+    ctx.set({
+        'cache-control': cacheControl || `public, max-age=${config.cache.contentExpire}`,
+        'content-length': contentLength,
+        'content-type': contentType,
+        server: 'RSSHub',
+    });
+
+    ctx.body = await got.stream(url, {
+        headers: {
+            referer,
+        },
+    });
+};

lib/v2/rsshub/referer-map.js

@@ -0,0 +1,16 @@
+const refererMap = new Map([
+    ['fbcdn.net', 'https://www.facebook.com/'],
+    ['cdninstagram.com', 'https://www.instagram.com/'],
+    ['moyu.im', 'https://jandan.net/'],
+    ['lightnovel.us', 'https://www.lightnovel.us/'],
+    ['indienova.com', 'https://indienova.com/'],
+    ['pximg.net', 'https://www.pixiv.net/'],
+    ['me8gs.app', 'https://www.sehuatang.net/'],
+    ['rxn30.app', 'https://www.sehuatang.net/'],
+    ['sex.com', 'https://www.sex.com/'],
+    ['sinaimg.cn', 'https://weibo.com/'],
+]);
+
+module.exports = {
+    refererMap,
+};

lib/v2/rsshub/router.js

@@ -1,4 +1,5 @@
 module.exports = (router) => {
+    router.get('/m/:key/:url', require('./media'));
     router.get('/routes/:lang?', require('./routes'));
     router.get('/transform/html/:url/:routeParams', require('./transform/html'));
     router.get('/transform/json/:url/:routeParams', require('./transform/json'));

lib/v2/sfacg/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/novel/chapter/:id': ['keocheung'],
+};

lib/v2/sfacg/novel-chapter.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const limit = Number.parseInt(ctx.query.limit) || 20;
+
+    const baseUrl = 'https://book.sfacg.com';
+
+    const { data: response } = await got(`${baseUrl}/Novel/${id}/MainIndex/`);
+    const $ = cheerio.load(response);
+
+    const list = $('div.catalog-list ul li a')
+        .slice(-limit)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.attr('title'),
+                link: `${baseUrl}${item.attr('href')}`,
+            };
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+
+                item.description = $('div.article-content').html();
+
+                const rawDate = $('div.article-desc span').eq(1).text();
+                item.pubDate = timezone(parseDate(rawDate.replace('更新时间：', '')), +8);
+
+                return item;
+            })
+        )
+    );
+
+    const { data: intro } = await got(`${baseUrl}/Novel/${id}/`);
+    const $i = cheerio.load(intro);
+
+    ctx.state.data = {
+        title: `SF轻小说 ${$('h1.story-title').text()}`,
+        link: `${baseUrl}/Novel/${id}`,
+        description: $i('p.introduce').text(),
+        image: $i('div.summary-pic img').attr('src').replace('http://', 'https://'),
+        item: items,
+    };
+};

lib/v2/sfacg/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'sfacg.com': {
+        _name: 'SF 轻小说',
+        book: [
+            {
+                title: '章节',
+                docs: 'https://docs.rsshub.app/routes/reading#sf-qing-xiao-shuo',
+                source: ['/Novel/:id/*'],
+                target: '/sfacg/novel/chapter/:id',
+            },
+        ],
+    },
+};

lib/v2/sfacg/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/novel/chapter/:id', require('./novel-chapter'));
+};

lib/v2/telegram/maintainer.js

@@ -1,5 +1,6 @@
 module.exports = {
     '/blog': ['fengkx'],
+    '/channel/:username': ['synchrone'],
     '/channel/:username/:routeParams?': ['DIYgod', 'Rongronggg9'],
     '/stickerpack/:name': ['DIYgod'],
 };

lib/v2/telegram/router.js

@@ -1,5 +1,18 @@
+const config = require('@/config').value;
+const coreRouter = require('@/core-router');
+
 module.exports = function (router) {
-    router.get('/channel/:username/:routeParams?', require('./channel'));
+    if (config.telegram.session && config.feature.mediaProxyKey) {
+        // core_router does not cache, which is necessary for video streaming
+        coreRouter.get('/telegram/channel/:username/:key/:media(.+)', require('./tglib/channel').getMedia);
+    }
+
+    router.get('/channel/:username/:routeParams?', (ctx) => {
+        // tglib impl does not support routeParams yet
+        const useWeb = ctx.params.routeParams || !(config.telegram.session && config.feature.mediaProxyKey);
+        return useWeb ? require('./channel')(ctx) : require('./tglib/channel')(ctx);
+    });
+
     router.get('/stickerpack/:name', require('./stickerpack'));
     router.get('/blog', require('./blog'));
 };