-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathpopulate_news_feeds.py
72 lines (57 loc) · 1.97 KB
/
populate_news_feeds.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import json
def generate_feed_data(keywords, suffix="", m=4):
"""
Generate a fake feed data
:param keywords: list of keywords to generate feeds for
:param suffix: suffix to add to the research terms to specify a general context (e.g., 'Olympics')
:param m: number of pages to search
:return: list of feeds
"""
feeds = []
for k in keywords:
k = k.replace(" ", "+")
for i in range(1, m):
rss_url = f"https://www.bing.com/news/search?format=RSS&q={k}+{suffix}&&first={i*11}"
feed = {
"url_site": "",
"category": "",
"leaning": "",
"name": f"Bing - {k}",
"feed_url": rss_url,
}
feeds.append(feed)
return feeds
def generate_feed(filename, topics, suffix=""):
"""
Generate topical rss feed using Bing search engine
:param filename: the file to save the feeds
:param topics: list of topics to generate feeds for
:param suffix: suffix to add to the research terms to specify a general context (e.g., 'Olympics')
"""
feeds = generate_feed_data(topics, suffix=suffix)
json.dump(feeds, open(filename, "w"), indent=4)
if __name__ == "__main__":
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument(
"-t",
"--topics",
help="list of topics to generate feeds for separated by commas",
)
parser.add_argument(
"-s",
"--suffix",
default="",
help="suffix to add to the research terms to specify a general context (e.g., 'Olympics')",
)
parser.add_argument(
"-o",
"--out_file",
default="config_files/rss_feeds.json",
help="JSON to save the generated feeds",
)
args = parser.parse_args()
topics = args.topics.replace("_", " ").split(",")
suffix = args.suffix
out_file = args.out_file
generate_feed(out_file, topics=topics, suffix=suffix)