-
Notifications
You must be signed in to change notification settings - Fork 1
/
puppeteerServer.js
61 lines (51 loc) · 1.93 KB
/
puppeteerServer.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
const express = require('express');
const puppeteer = require('puppeteer');
const randomUseragent = require('random-useragent');
const app = express();
const port = process.env.PORT || 8000;
app.get('/scrape', function (req, res) {
console.log("hello world")
let stockSymbol = 'AAPL';
let urls = [
`https://www.marketwatch.com/investing/stock/${stockSymbol}`,
`https://www.fool.com/quote/${stockSymbol}`,
`https://finance.yahoo.com/quote/${stockSymbol}`,
`https://www.thestreet.com/quote/${stockSymbol}.html`, // 404
`https://seekingalpha.com/symbol/${stockSymbol}`, // captcha
`https://www.bloomberg.com/quote/${stockSymbol}:US` // captcha
];
puppeteer.launch({ headless: false }).then(async browser => {
console.log("Launching Puppeteer");
const page = await browser.newPage();
// await page.setUserAgent(randomUseragent.getRandom());
await page.goto(urls[4]);
// await page.goto(urls[4], {waitUntil: 'networkidle2'});
// await page.waitForNavigation();
// await page.addScriptTag({url: 'https://code.jquery.com/jquery-3.4.1.min.js'});
const result = await page.evaluate(() => {
let titles = [];
let articles = document.querySelectorAll('.symbol_article');
articles.forEach(article => {
titles.push(article.firstChild.innerText);
});
// try {
// // let titles = [];
// // $('.symbol_article').each((index, val) => {
// // titles.push($(val).children().first().text())
// // // console.log("SYMBOL ARTICLE", $(val).children().first().text())
// // })
// // return titles;
// const $ = window.$;
// return $('.symbol_article')
// } catch(err) {
// console.log(err);
// }
return titles;
})
console.log(result);
await page.close();
await browser.close();
})
})
app.listen(port, () => console.log(`Listening on port ${port}`));
module.exports = app;