From a20a88c4c94308015b4166dfb7593a5ad224f52c Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 20 Mar 2016 14:15:45 +0100 Subject: [PATCH 01/23] changed database to class style, for ensureing that the filename is correct and that the databas is corrected closed removed local docopts changed urls.py to rss.yaml --- setup.py | 2 +- termfeed/database.py | 126 ++++++++ termfeed/dbinit.py | 21 -- termfeed/dbop.py | 107 ------- termfeed/feed.py | 15 +- termfeed/rss.yaml | 66 ++++ termfeed/support/__init__.py | 0 termfeed/support/docopt.py | 581 ----------------------------------- 8 files changed, 201 insertions(+), 717 deletions(-) create mode 100644 termfeed/database.py delete mode 100644 termfeed/dbinit.py delete mode 100644 termfeed/dbop.py create mode 100644 termfeed/rss.yaml delete mode 100644 termfeed/support/__init__.py delete mode 100644 termfeed/support/docopt.py diff --git a/setup.py b/setup.py index 6fe0463..bcac8ad 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ license = "MIT", author_email='iamaziz.alto@gmail.com', version='0.0.11', - install_requires=['feedparser'], + install_requires=['feedparser', 'pyyaml', 'docopt'], packages=['termfeed', 'termfeed.support'], scripts=[], entry_points={ diff --git a/termfeed/database.py b/termfeed/database.py new file mode 100644 index 0000000..661510c --- /dev/null +++ b/termfeed/database.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- + +""" +database operations. + +dbop.py manipulate database add, update, delete +""" + +import shelve, yaml +from os import path + +class DataBase: + + def rebuild_library(self): + # import termfeed.dbinit + if not path.exists('rss.yaml'): + from termfeed.urls import rss + with open('rss.yaml', 'w') as f: + f.write(yaml.dump(rss, default_flow_style=False)) + + with open('rss.yaml', 'r') as f: + rss = yaml.load(f) + + with shelve.open(self.file) as d: + for topic in rss: + links = rss[topic] + d[topic] = [link for link in links] + + print('created ".termfeed" in {}'.format(path.dirname(self.file))) + + def __init__(self): + homedir = path.expanduser('~') + + self.file = path.join(homedir, '.termfeed') + + # instantiate db if it's not created yet + if not (path.exists(self.file + '.dir') + or path.exists(self.file + '.dat')): + self.rebuild_library() + + # connect to db + self.db = shelve.open(self.file, 'w') + + + def __del__(self): + self.db.close() + + @property + def topics(self): + return list(self.db.keys()) + + + def read(self, topic): + if topic in self.topics: + return self.db[topic] + else: + return None + + def browse_links(self, topic): + if topic in self.topics: + links = self.db[topic] + print('{} resources:'.format(topic)) + for link in links: + print('\t{}'.format(link)) + else: + print('no category named {}'.format(topic)) + print_topics(d) + + def __repr__(self): + out = 'available topics: \n\t' + '\n\t'.join(self.topics) + return(out) + + def print_topics(self, d = None): + print(self) + + def add_link(self, link, topic='General'): + if topic in self.topics: + if link not in d[topic]: + # to add a new url: copy, mutates, store back + temp = d[topic] + temp.append(link) + self.db[topic] = temp + print('Updated .. {}'.format(topic)) + else: + print('{} already exists in {}!!'.format(link, topic)) + else: + print('Created new category .. {}'.format(topic)) + self.db[topic] = [link] + + + def remove_link(self, link): + done = False + for topic in self.topics: + if link in self.db[topic]: + self.db[topic] = [l for l in self.db[topic] if l != link] + print('removed: {}\nfrom: {}'.format(link, topic)) + done = True + + if not done: + print('URL not found: {}'.format(link)) + + + def delete_topic(self, topic): + if topic == 'General': + print('Default topic "General" cannot be removed.') + exit() + try: + del self.db[topic] + print('Removed "{}" from your library.'.format(topic)) + except KeyError: + print('"{}" is not in your library!'.format(topic)) + exit() + + +# if __name__ == '__main__': + +# for l in read('News'): +# print(l) + +# remove_link('http://rt.com/rss/') + +# add_link('http://rt.com/rss/', 'News') + +# for l in read('News'): +# print(l) diff --git a/termfeed/dbinit.py b/termfeed/dbinit.py deleted file mode 100644 index c98fcb9..0000000 --- a/termfeed/dbinit.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python - -# This should be exectued once to initialize the db from urls.py - -import shelve -from os import path - -from termfeed.urls import rss - -homedir = path.expanduser('~') - -# initiate database datafile -d = shelve.open(path.join(homedir, '.termfeed')) - - -# dump urls.py into rss_shelf.db -for topic in rss: - links = rss[topic] - d[topic] = [link for link in links] - -d.close() diff --git a/termfeed/dbop.py b/termfeed/dbop.py deleted file mode 100644 index 6c40d4a..0000000 --- a/termfeed/dbop.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- - -""" -database operations. - -dbop.py manipulate database add, update, delete -""" - -import shelve -from os import path - -homedir = path.expanduser('~') - -def rebuild_library(): - import termfeed.dbinit - print('created ".termfeed.db" in {}'.format(homedir)) - -# instantiate db if it's not created yet -if not path.exists(homedir + '/.termfeed.db'): - rebuild_library() - - -# connect to db -d = shelve.open(path.join(homedir, '.termfeed'), 'w') - - -def topics(): - return d.keys() - - -def read(topic): - if topic in d.keys(): - return d[topic] - else: - return None - - -def browse_links(topic): - if topic in d.keys(): - links = d[topic] - print('{} resources:'.format(topic)) - for link in links: - print('\t{}'.format(link)) - else: - print('no category named {}'.format(topic)) - print_topics() - - -def print_topics(): - print('available topics: ') - for t in topics(): - print('\t{}'.format(t)) - - -def add_link(link, topic='General'): - - if topic in d.keys(): - if link not in d[topic]: - # to add a new url: copy, mutates, store back - temp = d[topic] - temp.append(link) - d[topic] = temp - print('Updated .. {}'.format(topic)) - else: - print('{} already exists in {}!!'.format(link, topic)) - else: - print('Created new category .. {}'.format(topic)) - d[topic] = [link] - - -def remove_link(link): - - done = False - for topic in topics(): - if link in d[topic]: - d[topic] = [l for l in d[topic] if l != link] - print('removed: {}\nfrom: {}'.format(link, topic)) - done = True - - if not done: - print('URL not found: {}'.format(link)) - - -def delete_topic(topic): - if topic == 'General': - print('Default topic "General" cannot be removed.') - exit() - try: - del d[topic] - print('Removed "{}" from your library.'.format(topic)) - except KeyError: - print('"{}" is not in your library!'.format(topic)) - exit() - - -# if __name__ == '__main__': - -# for l in read('News'): -# print(l) - -# remove_link('http://rt.com/rss/') - -# add_link('http://rt.com/rss/', 'News') - -# for l in read('News'): -# print(l) diff --git a/termfeed/feed.py b/termfeed/feed.py index b1f9d92..a796b72 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -40,7 +40,9 @@ except ImportError: from urllib.request import urlopen -import termfeed.dbop as dbop +import termfeed.dbop + +dbop = termfeed.database.DataBase() class bcolors: @@ -190,18 +192,17 @@ def recurse(zipped): def topic_choice(browse): if browse: - topics = dbop.topics() tags = {} - for i, tag in enumerate(topics): + for i, tag in enumerate(dbop.topics): tags[i] = tag print("{}) {}".format(i, tags[i])) try: m = '\nChoose the topic (number)? : ' try: # python 2 - uin = raw_input(m) + uin = raw_input(m) except NameError: # python 3 uin = input(m) uin = int(uin) @@ -223,8 +224,8 @@ def validate_feed(url): else: exit() -from .support.docopt import docopt - +# from .support.docopt import docopt +from docopt import docopt def main(): args = docopt( @@ -269,7 +270,7 @@ def main(): if category: dbop.browse_links(category) else: - dbop.print_topics() + print(dbop) if rebuild: dbop.rebuild_library() diff --git a/termfeed/rss.yaml b/termfeed/rss.yaml new file mode 100644 index 0000000..80abcb1 --- /dev/null +++ b/termfeed/rss.yaml @@ -0,0 +1,66 @@ +AI: +- http://cervisia.org/rss_feed.pl/machine_learning_news.rss +- http://feeds.feedburner.com/FeaturedBlogPosts-DataScienceCentral +- http://feeds.feedburner.com/ResourcesDiscussions-DataScienceCentral +- http://feeds.feedburner.com/ResearchDiscussions-DataScienceCentral +- http://feeds.feedburner.com/FeaturedBlogPosts-Bigdatanews +- http://cacm.acm.org/browse-by-subject/artificial-intelligence.rss +- http://www.deepstuff.org/feed/ +- http://feeds.feedburner.com/miriblog +- http://www.csail.mit.edu/csailnews/rss +CS: +- http://feeds.sciencedaily.com/sciencedaily/computers_math/computer_science +- http://phys.org/rss-feed/technology-news/computer-sciences/ +- http://newsoffice.mit.edu/rss/topic/computers +- http://www.nature.com/subjects/computer-science.rss +- http://www.nature.com/subjects/mathematics-and-computing.rss +- http://newsoffice.mit.edu/rss/topic/robotics +- http://cacm.acm.org/news.rss +GEEK: +- http://feeds.feedburner.com/hacker-news-feed-200?format=xml +- http://feeds.feedburner.com/TheHackersNews +- https://news.ycombinator.com/rss +- http://www.reddit.com/r/learnprogramming/ +- http://www.reddit.com/r/algorithms/ +- http://www.reddit.com/r/programming/ +- http://feeds.feedburner.com/SingularityBlog +General: +- http://feeds.feedburner.com/PythonCentral +- http://cacm.acm.org/browse-by-subject/artificial-intelligence.rss +- feed:https://news.ycombinator.com/rss +ML: +- http://hunch.net/?feed=rss2 +- http://research.microsoft.com/rss/downloads.xml +- http://mlg.eng.cam.ac.uk/?feed=rss2 +News: +- feed://www.aljazeera.com/xml/rss/all.xml +- feed://america.aljazeera.com/content/ajam/articles.rss +- http://rt.com/rss/ +- http://feeds.feedburner.com/japantimes +- http://www.japantoday.com/feed +- http://api.breakingnews.com/api/v1/item/?format=rss +Python: +- feed://changelog.com/tagged/python/feed/ +- http://feeds.feedburner.com/PythonCentral +- feed://blog.jupyter.org/rss/ +- http://planetpython.org/rss20.xml +- https://pypi.python.org/pypi?%3Aaction=packages_rss +Research: +- http://www.nsf.gov/rss/rss_www_funding_pgm_annc_inf.xml +- http://www.nsf.gov/statistics/rss/srs_rss.xml +- http://www.darpa.mil/rss +- http://feeds.feedburner.com/blogspot/gJZg +Science: +- https://www.sciencenews.org/feeds/headlines.rss +- http://feeds.reuters.com/reuters/scienceNews +- http://feeds.sciencedaily.com/sciencedaily/top_news +- http://phys.org/rss-feed/space-news/space-exploration/ +- http://phys.org/rss-feed/space-news/astronomy/ +- http://phys.org/rss-feed/physics-news/quantum-physics/ +- http://phys.org/rss-feed/physics-news/physics/ +Stack: +- http://stackoverflow.com/questions/tagged/python +- http://stackoverflow.com/questions/tagged/python+numpy +- http://stackoverflow.com/feeds +- http://stackoverflow.com/questions/tagged/python+django +- http://stackoverflow.com/questions/tagged/python+django+javascript diff --git a/termfeed/support/__init__.py b/termfeed/support/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/termfeed/support/docopt.py b/termfeed/support/docopt.py deleted file mode 100644 index 4d59f1f..0000000 --- a/termfeed/support/docopt.py +++ /dev/null @@ -1,581 +0,0 @@ -"""Pythonic command-line interface parser that will make you smile. - - * http://docopt.org - * Repository and issue-tracker: https://github.com/docopt/docopt - * Licensed under terms of MIT license (see LICENSE-MIT) - * Copyright (c) 2013 Vladimir Keleshev, vladimir@keleshev.com - -""" -import sys -import re - - -__all__ = ['docopt'] -__version__ = '0.6.1' - - -class DocoptLanguageError(Exception): - - """Error in construction of usage-message by developer.""" - - -class DocoptExit(SystemExit): - - """Exit in case user invoked program with incorrect arguments.""" - - usage = '' - - def __init__(self, message=''): - SystemExit.__init__(self, (message + '\n' + self.usage).strip()) - - -class Pattern(object): - - def __eq__(self, other): - return repr(self) == repr(other) - - def __hash__(self): - return hash(repr(self)) - - def fix(self): - self.fix_identities() - self.fix_repeating_arguments() - return self - - def fix_identities(self, uniq=None): - """Make pattern-tree tips point to same object if they are equal.""" - if not hasattr(self, 'children'): - return self - uniq = list(set(self.flat())) if uniq is None else uniq - for i, child in enumerate(self.children): - if not hasattr(child, 'children'): - assert child in uniq - self.children[i] = uniq[uniq.index(child)] - else: - child.fix_identities(uniq) - - def fix_repeating_arguments(self): - """Fix elements that should accumulate/increment values.""" - either = [list(child.children) for child in transform(self).children] - for case in either: - for e in [child for child in case if case.count(child) > 1]: - if type(e) is Argument or type(e) is Option and e.argcount: - if e.value is None: - e.value = [] - elif type(e.value) is not list: - e.value = e.value.split() - if type(e) is Command or type(e) is Option and e.argcount == 0: - e.value = 0 - return self - - -def transform(pattern): - """Expand pattern into an (almost) equivalent one, but with single Either. - - Example: ((-a | -b) (-c | -d)) => (-a -c | -a -d | -b -c | -b -d) - Quirks: [-a] => (-a), (-a...) => (-a -a) - - """ - result = [] - groups = [[pattern]] - while groups: - children = groups.pop(0) - parents = [Required, Optional, OptionsShortcut, Either, OneOrMore] - if any(t in map(type, children) for t in parents): - child = [c for c in children if type(c) in parents][0] - children.remove(child) - if type(child) is Either: - for c in child.children: - groups.append([c] + children) - elif type(child) is OneOrMore: - groups.append(child.children * 2 + children) - else: - groups.append(child.children + children) - else: - result.append(children) - return Either(*[Required(*e) for e in result]) - - -class LeafPattern(Pattern): - - """Leaf/terminal node of a pattern tree.""" - - def __init__(self, name, value=None): - self.name, self.value = name, value - - def __repr__(self): - return '%s(%r, %r)' % (self.__class__.__name__, self.name, self.value) - - def flat(self, *types): - return [self] if not types or type(self) in types else [] - - def match(self, left, collected=None): - collected = [] if collected is None else collected - pos, match = self.single_match(left) - if match is None: - return False, left, collected - left_ = left[:pos] + left[pos + 1:] - same_name = [a for a in collected if a.name == self.name] - if type(self.value) in (int, list): - if type(self.value) is int: - increment = 1 - else: - increment = ([match.value] if type(match.value) is str - else match.value) - if not same_name: - match.value = increment - return True, left_, collected + [match] - same_name[0].value += increment - return True, left_, collected - return True, left_, collected + [match] - - -class BranchPattern(Pattern): - - """Branch/inner node of a pattern tree.""" - - def __init__(self, *children): - self.children = list(children) - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, - ', '.join(repr(a) for a in self.children)) - - def flat(self, *types): - if type(self) in types: - return [self] - return sum([child.flat(*types) for child in self.children], []) - - -class Argument(LeafPattern): - - def single_match(self, left): - for n, pattern in enumerate(left): - if type(pattern) is Argument: - return n, Argument(self.name, pattern.value) - return None, None - - @classmethod - def parse(class_, source): - name = re.findall('(<\S*?>)', source)[0] - value = re.findall('\[default: (.*)\]', source, flags=re.I) - return class_(name, value[0] if value else None) - - -class Command(Argument): - - def __init__(self, name, value=False): - self.name, self.value = name, value - - def single_match(self, left): - for n, pattern in enumerate(left): - if type(pattern) is Argument: - if pattern.value == self.name: - return n, Command(self.name, True) - else: - break - return None, None - - -class Option(LeafPattern): - - def __init__(self, short=None, long=None, argcount=0, value=False): - assert argcount in (0, 1) - self.short, self.long, self.argcount = short, long, argcount - self.value = None if value is False and argcount else value - - @classmethod - def parse(class_, option_description): - short, long, argcount, value = None, None, 0, False - options, _, description = option_description.strip().partition(' ') - options = options.replace(',', ' ').replace('=', ' ') - for s in options.split(): - if s.startswith('--'): - long = s - elif s.startswith('-'): - short = s - else: - argcount = 1 - if argcount: - matched = re.findall('\[default: (.*)\]', description, flags=re.I) - value = matched[0] if matched else None - return class_(short, long, argcount, value) - - def single_match(self, left): - for n, pattern in enumerate(left): - if self.name == pattern.name: - return n, pattern - return None, None - - @property - def name(self): - return self.long or self.short - - def __repr__(self): - return 'Option(%r, %r, %r, %r)' % (self.short, self.long, - self.argcount, self.value) - - -class Required(BranchPattern): - - def match(self, left, collected=None): - collected = [] if collected is None else collected - l = left - c = collected - for pattern in self.children: - matched, l, c = pattern.match(l, c) - if not matched: - return False, left, collected - return True, l, c - - -class Optional(BranchPattern): - - def match(self, left, collected=None): - collected = [] if collected is None else collected - for pattern in self.children: - m, left, collected = pattern.match(left, collected) - return True, left, collected - - -class OptionsShortcut(Optional): - - """Marker/placeholder for [options] shortcut.""" - - -class OneOrMore(BranchPattern): - - def match(self, left, collected=None): - assert len(self.children) == 1 - collected = [] if collected is None else collected - l = left - c = collected - l_ = None - matched = True - times = 0 - while matched: - # could it be that something didn't match but changed l or c? - matched, l, c = self.children[0].match(l, c) - times += 1 if matched else 0 - if l_ == l: - break - l_ = l - if times >= 1: - return True, l, c - return False, left, collected - - -class Either(BranchPattern): - - def match(self, left, collected=None): - collected = [] if collected is None else collected - outcomes = [] - for pattern in self.children: - matched, _, _ = outcome = pattern.match(left, collected) - if matched: - outcomes.append(outcome) - if outcomes: - return min(outcomes, key=lambda outcome: len(outcome[1])) - return False, left, collected - - -class Tokens(list): - - def __init__(self, source, error=DocoptExit): - self += source.split() if hasattr(source, 'split') else source - self.error = error - - @staticmethod - def from_pattern(source): - source = re.sub(r'([\[\]\(\)\|]|\.\.\.)', r' \1 ', source) - source = [s for s in re.split('\s+|(\S*<.*?>)', source) if s] - return Tokens(source, error=DocoptLanguageError) - - def move(self): - return self.pop(0) if len(self) else None - - def current(self): - return self[0] if len(self) else None - - -def parse_long(tokens, options): - """long ::= '--' chars [ ( ' ' | '=' ) chars ] ;""" - long, eq, value = tokens.move().partition('=') - assert long.startswith('--') - value = None if eq == value == '' else value - similar = [o for o in options if o.long == long] - if tokens.error is DocoptExit and similar == []: # if no exact match - similar = [o for o in options if o.long and o.long.startswith(long)] - if len(similar) > 1: # might be simply specified ambiguously 2+ times? - raise tokens.error('%s is not a unique prefix: %s?' % - (long, ', '.join(o.long for o in similar))) - elif len(similar) < 1: - argcount = 1 if eq == '=' else 0 - o = Option(None, long, argcount) - options.append(o) - if tokens.error is DocoptExit: - o = Option(None, long, argcount, value if argcount else True) - else: - o = Option(similar[0].short, similar[0].long, - similar[0].argcount, similar[0].value) - if o.argcount == 0: - if value is not None: - raise tokens.error('%s must not have an argument' % o.long) - else: - if value is None: - if tokens.current() in [None, '--']: - raise tokens.error('%s requires argument' % o.long) - value = tokens.move() - if tokens.error is DocoptExit: - o.value = value if value is not None else True - return [o] - - -def parse_shorts(tokens, options): - """shorts ::= '-' ( chars )* [ [ ' ' ] chars ] ;""" - token = tokens.move() - assert token.startswith('-') and not token.startswith('--') - left = token.lstrip('-') - parsed = [] - while left != '': - short, left = '-' + left[0], left[1:] - similar = [o for o in options if o.short == short] - if len(similar) > 1: - raise tokens.error('%s is specified ambiguously %d times' % - (short, len(similar))) - elif len(similar) < 1: - o = Option(short, None, 0) - options.append(o) - if tokens.error is DocoptExit: - o = Option(short, None, 0, True) - else: # why copying is necessary here? - o = Option(short, similar[0].long, - similar[0].argcount, similar[0].value) - value = None - if o.argcount != 0: - if left == '': - if tokens.current() in [None, '--']: - raise tokens.error('%s requires argument' % short) - value = tokens.move() - else: - value = left - left = '' - if tokens.error is DocoptExit: - o.value = value if value is not None else True - parsed.append(o) - return parsed - - -def parse_pattern(source, options): - tokens = Tokens.from_pattern(source) - result = parse_expr(tokens, options) - if tokens.current() is not None: - raise tokens.error('unexpected ending: %r' % ' '.join(tokens)) - return Required(*result) - - -def parse_expr(tokens, options): - """expr ::= seq ( '|' seq )* ;""" - seq = parse_seq(tokens, options) - if tokens.current() != '|': - return seq - result = [Required(*seq)] if len(seq) > 1 else seq - while tokens.current() == '|': - tokens.move() - seq = parse_seq(tokens, options) - result += [Required(*seq)] if len(seq) > 1 else seq - return [Either(*result)] if len(result) > 1 else result - - -def parse_seq(tokens, options): - """seq ::= ( atom [ '...' ] )* ;""" - result = [] - while tokens.current() not in [None, ']', ')', '|']: - atom = parse_atom(tokens, options) - if tokens.current() == '...': - atom = [OneOrMore(*atom)] - tokens.move() - result += atom - return result - - -def parse_atom(tokens, options): - """atom ::= '(' expr ')' | '[' expr ']' | 'options' - | long | shorts | argument | command ; - """ - token = tokens.current() - result = [] - if token in '([': - tokens.move() - matching, pattern = {'(': [')', Required], '[': [']', Optional]}[token] - result = pattern(*parse_expr(tokens, options)) - if tokens.move() != matching: - raise tokens.error("unmatched '%s'" % token) - return [result] - elif token == 'options': - tokens.move() - return [OptionsShortcut()] - elif token.startswith('--') and token != '--': - return parse_long(tokens, options) - elif token.startswith('-') and token not in ('-', '--'): - return parse_shorts(tokens, options) - elif token.startswith('<') and token.endswith('>') or token.isupper(): - return [Argument(tokens.move())] - else: - return [Command(tokens.move())] - - -def parse_argv(tokens, options, options_first=False): - """Parse command-line argument vector. - - If options_first: - argv ::= [ long | shorts ]* [ argument ]* [ '--' [ argument ]* ] ; - else: - argv ::= [ long | shorts | argument ]* [ '--' [ argument ]* ] ; - - """ - parsed = [] - while tokens.current() is not None: - if tokens.current() == '--': - return parsed + [Argument(None, v) for v in tokens] - elif tokens.current().startswith('--'): - parsed += parse_long(tokens, options) - elif tokens.current().startswith('-') and tokens.current() != '-': - parsed += parse_shorts(tokens, options) - elif options_first: - return parsed + [Argument(None, v) for v in tokens] - else: - parsed.append(Argument(None, tokens.move())) - return parsed - - -def parse_defaults(doc): - defaults = [] - for s in parse_section('options:', doc): - # FIXME corner case "bla: options: --foo" - _, _, s = s.partition(':') # get rid of "options:" - split = re.split('\n[ \t]*(-\S+?)', '\n' + s)[1:] - split = [s1 + s2 for s1, s2 in zip(split[::2], split[1::2])] - options = [Option.parse(s) for s in split if s.startswith('-')] - defaults += options - return defaults - - -def parse_section(name, source): - pattern = re.compile('^([^\n]*' + name + '[^\n]*\n?(?:[ \t].*?(?:\n|$))*)', - re.IGNORECASE | re.MULTILINE) - return [s.strip() for s in pattern.findall(source)] - - -def formal_usage(section): - _, _, section = section.partition(':') # drop "usage:" - pu = section.split() - return '( ' + ' '.join(') | (' if s == pu[0] else s for s in pu[1:]) + ' )' - - -def extras(help, version, options, doc): - if help and any((o.name in ('-h', '--help')) and o.value for o in options): - print(doc.strip("\n")) - sys.exit() - if version and any(o.name == '--version' and o.value for o in options): - print(version) - sys.exit() - - -class Dict(dict): - def __repr__(self): - return '{%s}' % ',\n '.join('%r: %r' % i for i in sorted(self.items())) - - -def docopt(doc, argv=None, help=True, version=None, options_first=False): - """Parse `argv` based on command-line interface described in `doc`. - - `docopt` creates your command-line interface based on its - description that you pass as `doc`. Such description can contain - --options, , commands, which could be - [optional], (required), (mutually | exclusive) or repeated... - - Parameters - ---------- - doc : str - Description of your command-line interface. - argv : list of str, optional - Argument vector to be parsed. sys.argv[1:] is used if not - provided. - help : bool (default: True) - Set to False to disable automatic help on -h or --help - options. - version : any object - If passed, the object will be printed if --version is in - `argv`. - options_first : bool (default: False) - Set to True to require options precede positional arguments, - i.e. to forbid options and positional arguments intermix. - - Returns - ------- - args : dict - A dictionary, where keys are names of command-line elements - such as e.g. "--verbose" and "", and values are the - parsed values of those elements. - - Example - ------- - >>> from docopt import docopt - >>> doc = ''' - ... Usage: - ... my_program tcp [--timeout=] - ... my_program serial [--baud=] [--timeout=] - ... my_program (-h | --help | --version) - ... - ... Options: - ... -h, --help Show this screen and exit. - ... --baud= Baudrate [default: 9600] - ... ''' - >>> argv = ['tcp', '127.0.0.1', '80', '--timeout', '30'] - >>> docopt(doc, argv) - {'--baud': '9600', - '--help': False, - '--timeout': '30', - '--version': False, - '': '127.0.0.1', - '': '80', - 'serial': False, - 'tcp': True} - - See also - -------- - * For video introduction see http://docopt.org - * Full documentation is available in README.rst as well as online - at https://github.com/docopt/docopt#readme - - """ - argv = sys.argv[1:] if argv is None else argv - - usage_sections = parse_section('usage:', doc) - if len(usage_sections) == 0: - raise DocoptLanguageError('"usage:" (case-insensitive) not found.') - if len(usage_sections) > 1: - raise DocoptLanguageError('More than one "usage:" (case-insensitive).') - DocoptExit.usage = usage_sections[0] - - options = parse_defaults(doc) - pattern = parse_pattern(formal_usage(DocoptExit.usage), options) - # [default] syntax for argument is disabled - #for a in pattern.flat(Argument): - # same_name = [d for d in arguments if d.name == a.name] - # if same_name: - # a.value = same_name[0].value - argv = parse_argv(Tokens(argv), list(options), options_first) - pattern_options = set(pattern.flat(Option)) - for options_shortcut in pattern.flat(OptionsShortcut): - doc_options = parse_defaults(doc) - options_shortcut.children = list(set(doc_options) - pattern_options) - #if any_options: - # options_shortcut.children += [Option(o.short, o.long, o.argcount) - # for o in argv if type(o) is Option] - extras(help, version, argv, doc) - matched, left, collected = pattern.fix().match(argv) - if matched and left == []: # better error message if left? - return Dict((a.name, a.value) for a in (pattern.flat() + collected)) - raise DocoptExit() \ No newline at end of file From 3dc85308d79cc77e58f208615be27581a0da4c03 Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 20 Mar 2016 14:42:51 +0100 Subject: [PATCH 02/23] Bugfix: removed termfeed.support in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bcac8ad..26491f2 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ author_email='iamaziz.alto@gmail.com', version='0.0.11', install_requires=['feedparser', 'pyyaml', 'docopt'], - packages=['termfeed', 'termfeed.support'], + packages=['termfeed'], scripts=[], entry_points={ 'console_scripts': [ From 9a4e0055cb3f33f54d5ff69dae6372c025dc4c55 Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 20 Mar 2016 14:46:09 +0100 Subject: [PATCH 03/23] Bugfix: import was wrong --- termfeed/feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/termfeed/feed.py b/termfeed/feed.py index a796b72..894b0fe 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -40,7 +40,7 @@ except ImportError: from urllib.request import urlopen -import termfeed.dbop +import termfeed.database dbop = termfeed.database.DataBase() From 9e075e3394c764773cc55dccca9ce64288ca983c Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 20 Mar 2016 19:55:18 +0100 Subject: [PATCH 04/23] add dependencies: 'plumbum', 'arrow' --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 26491f2..e52f69e 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ license = "MIT", author_email='iamaziz.alto@gmail.com', version='0.0.11', - install_requires=['feedparser', 'pyyaml', 'docopt'], + install_requires=['feedparser', 'pyyaml', 'docopt', 'plumbum', 'arrow'], packages=['termfeed'], scripts=[], entry_points={ From 2846cfa4564e1779daf277e4f1da8d99fe99e79a Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 20 Mar 2016 19:56:06 +0100 Subject: [PATCH 05/23] changed database backend from shelve to json --- termfeed/database.py | 68 ++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/termfeed/database.py b/termfeed/database.py index 661510c..af34809 100644 --- a/termfeed/database.py +++ b/termfeed/database.py @@ -7,93 +7,86 @@ dbop.py manipulate database add, update, delete """ -import shelve, yaml +import yaml, json from os import path +from cached_property import cached_property class DataBase: + file = path.join(path.expanduser('~'), '.termfeed.json') + def rebuild_library(self): - # import termfeed.dbinit - if not path.exists('rss.yaml'): - from termfeed.urls import rss - with open('rss.yaml', 'w') as f: - f.write(yaml.dump(rss, default_flow_style=False)) - with open('rss.yaml', 'r') as f: + with open(path.join(path.dirname(__file__), 'rss.yaml'), 'r') as f: rss = yaml.load(f) - with shelve.open(self.file) as d: - for topic in rss: - links = rss[topic] - d[topic] = [link for link in links] - - print('created ".termfeed" in {}'.format(path.dirname(self.file))) + self.save_rss_on_fs(rss) + print('created ".termfeed" in {}'.format(path.dirname(self.file))) def __init__(self): - homedir = path.expanduser('~') - - self.file = path.join(homedir, '.termfeed') # instantiate db if it's not created yet - if not (path.exists(self.file + '.dir') - or path.exists(self.file + '.dat')): + if not (path.exists(self.file)): self.rebuild_library() - # connect to db - self.db = shelve.open(self.file, 'w') + with open(self.file, 'r') as f: + self.rss = json.load(f) - - def __del__(self): - self.db.close() + def save_rss_on_fs(self, rss): + with open(self.file+'.json', 'w') as f: + json.load(rss, f) @property def topics(self): - return list(self.db.keys()) - + return self.rss.keys() + # return list(self.db.keys()) def read(self, topic): if topic in self.topics: - return self.db[topic] + return self.rss[topic] else: return None def browse_links(self, topic): if topic in self.topics: - links = self.db[topic] + links = self.rss[topic] print('{} resources:'.format(topic)) for link in links: print('\t{}'.format(link)) else: print('no category named {}'.format(topic)) - print_topics(d) + print(self) - def __repr__(self): + def __str__(self): out = 'available topics: \n\t' + '\n\t'.join(self.topics) return(out) - def print_topics(self, d = None): + def print_topics(self): print(self) def add_link(self, link, topic='General'): if topic in self.topics: - if link not in d[topic]: + if link not in self.rss[topic]: # to add a new url: copy, mutates, store back - temp = d[topic] + temp = self.rss[topic] temp.append(link) - self.db[topic] = temp + self.rss[topic] = temp + self.save_rss_on_fs(rss) print('Updated .. {}'.format(topic)) else: print('{} already exists in {}!!'.format(link, topic)) else: print('Created new category .. {}'.format(topic)) - self.db[topic] = [link] + self.rss[topic] = [link] + self.save_rss_on_fs(rss) def remove_link(self, link): done = False for topic in self.topics: - if link in self.db[topic]: - self.db[topic] = [l for l in self.db[topic] if l != link] + if link in self.rss[topic]: + self.rss[topic] = [l for l in self.rss[topic] if l != link] + self.save_rss_on_fs(rss) print('removed: {}\nfrom: {}'.format(link, topic)) done = True @@ -106,7 +99,8 @@ def delete_topic(self, topic): print('Default topic "General" cannot be removed.') exit() try: - del self.db[topic] + del self.rss[topic] + self.save_rss_on_fs(rss) print('Removed "{}" from your library.'.format(topic)) except KeyError: print('"{}" is not in your library!'.format(topic)) From a02d09c5a086ff7e90428b3ed60a1c02718f5897 Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 20 Mar 2016 20:00:31 +0100 Subject: [PATCH 06/23] replaced colors with plumbum, show all feed at the same time, show time --- termfeed/feed.py | 145 +++++++++++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 60 deletions(-) diff --git a/termfeed/feed.py b/termfeed/feed.py index 894b0fe..7baf383 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -23,7 +23,7 @@ -d URL Delete from the database file. -t See the stored categories in your library, or list the URLs stored under in your library. -D TOPIC Remove entire cateogry (and its urls) from your library. - -R Rebuild the library from the url.py + -R Rebuild the library from the rss.yaml -h --help Show this screen. """ @@ -34,6 +34,9 @@ import webbrowser import feedparser import re +import arrow +import dateutil.parser +from plumbum import colors as c try: from urllib import urlopen @@ -43,19 +46,6 @@ import termfeed.database dbop = termfeed.database.DataBase() - - -class bcolors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - - def _connected(): """check internet connect""" host = 'http://google.com' @@ -66,27 +56,48 @@ def _connected(): except: return False - def open_page(url, title): - print(bcolors.WARNING + - '\topening ... {}\n'.format(title.encode('utf8')) + bcolors.ENDC) + with c.info: + print('\topening ... {}\n'.format(title.encode('utf8'))) # open page in browser webbrowser.open(url) - +from tabulate import tabulate def print_feed(zipped): + #keys() + #dict_keys(['id', 'links', 'summary', 'author', 'guidislink', + #'author_detail', 'link', 'summary_detail', 'published', 'content', + #'authors', 'published_parsed', 'title', 'title_detail']) + + def parse_time(post): + return c.info | arrow.get(dateutil.parser.parse(post.published)).humanize() - for num, post in zipped.items(): - print(bcolors.OKGREEN + '[{}] '.format(num) + bcolors.ENDC, end='') - print('{}'.format(post.title.encode('utf8'))) + r = re.compile(r'(\w+)\.git') + + def repo(post): + repos = r.findall(post.summary_detail.base) + if repos: + return repos[0] + else: + return '' + + table = [[ c.green | '[{}] '.format(num), + repo(post), + parse_time(post), + c.dark_gray | post.author_detail.name, post.title] + for num, post in reversed(list(zipped.items()))] + + print(tabulate(table, tablefmt="plain"))#, tablefmt="plain")) def print_desc(topic, txt): - try: - print(bcolors.WARNING + '\n\n{}:'.format(topic) + bcolors.ENDC) - except UnicodeEncodeError: - pass - print(bcolors.BOLD + '\n\t{}'.format(txt.encode('utf8')) + bcolors.ENDC) + with c.info: + try: + print('\n\n{}:'.format(topic)) + except UnicodeEncodeError: + pass + with c.bold: + print('\n\t{}'.format(txt)) def open_it(): @@ -116,8 +127,9 @@ def clean_txt(txt): def _continue(): try: - msg = """\n\nPress: Enter to continue, ... [NUM] for short description / open a page, ... or CTRL-C to exit: """ - print(bcolors.FAIL + msg + bcolors.ENDC, end='') + msg = """\nPress: Enter to continue, ... [NUM] for short description / open a page, ... or CTRL-C to exit: """ + with c.warn: + print(msg, end='') # kb is the pressed keyboard key try: kb = raw_input() @@ -144,49 +156,62 @@ def parse_feed(url): def fetch_feeds(urls): + feeds = [] for i, url in enumerate(urls): - d = parse_feed(url) + feeds += [parse_feed(url)] + + #if d is None: + # continue # to next url + + # feeds source + l = len(urls) - 1 + + for i, d in enumerate(feeds): + title = url if d.feed.title else d.feed.title + print(c.magenta | " {}/{} SOURCE>> {}".format(i, l, d.feed.title) ) + + # print out feeds - if d is None: - continue # to next url + zipped = [] + for d in feeds: + zipped += d.entries - # feeds source - l = len(urls) - 1 - print( - bcolors.HEADER + "\n {}/{} SOURCE>> {}\n".format(i, l, url) + bcolors.ENDC) + # https://wiki.python.org/moin/HowTo/Sorting#The_Old_Way_Using_Decorate-Sort-Undecorate + decorated = [(dateutil.parser.parse(post.published), i, post) for i, post in enumerate(zipped)] + decorated.sort(reverse=True) + zipped = [post for time, i, post in decorated] # undecorate - # print out feeds - zipped = dict(enumerate(d.entries)) + zipped = dict(enumerate(zipped)) - def recurse(zipped): + def recurse(zipped): - print_feed(zipped) + print_feed(zipped) - kb = _continue() # keystroke listener + kb = _continue() # keystroke listener - if kb: - user_selected = kb is not '' and kb in str(zipped.keys()) - if user_selected: - # to open page in browser - link = zipped[int(kb)].link - title = zipped[int(kb)].title - try: - desc = zipped[int(kb)].description - desc = clean_txt(desc) - print_desc(title, desc) - except AttributeError: - print('\n\tNo description available!!') + if kb: + user_selected = kb is not '' and kb in str(zipped.keys()) + if user_selected: + # to open page in browser + link = zipped[int(kb)].link + title = zipped[int(kb)].title + try: + desc = zipped[int(kb)].description + desc = clean_txt(desc) + print_desc(title, desc) + except AttributeError: + print('\n\tNo description available!!') - if open_it(): - open_page(link, title) - else: - print( - bcolors.BOLD + 'Invalid entry ... {} '.format(kb) + bcolors.ENDC) - # repeat with same feeds and listen to kb again - recurse(zipped) + if open_it(): + open_page(link, title) + else: + with c.bold: + print('Invalid entry ... {} '.format(kb)) + # repeat with same feeds and listen to kb again + recurse(zipped) - recurse(zipped) + recurse(zipped) def topic_choice(browse): From 026214270e1eaea8585355666dcca6d63ddec8e8 Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 20 Mar 2016 21:03:15 +0100 Subject: [PATCH 07/23] Bugfix: some self's where missing --- termfeed/database.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/termfeed/database.py b/termfeed/database.py index af34809..ad40b92 100644 --- a/termfeed/database.py +++ b/termfeed/database.py @@ -33,8 +33,8 @@ def __init__(self): self.rss = json.load(f) def save_rss_on_fs(self, rss): - with open(self.file+'.json', 'w') as f: - json.load(rss, f) + with open(self.file, 'w') as f: + json.dump(rss, f) @property def topics(self): @@ -71,14 +71,14 @@ def add_link(self, link, topic='General'): temp = self.rss[topic] temp.append(link) self.rss[topic] = temp - self.save_rss_on_fs(rss) + self.save_rss_on_fs(self.rss) print('Updated .. {}'.format(topic)) else: print('{} already exists in {}!!'.format(link, topic)) else: print('Created new category .. {}'.format(topic)) self.rss[topic] = [link] - self.save_rss_on_fs(rss) + self.save_rss_on_fs(self.rss) def remove_link(self, link): @@ -86,7 +86,7 @@ def remove_link(self, link): for topic in self.topics: if link in self.rss[topic]: self.rss[topic] = [l for l in self.rss[topic] if l != link] - self.save_rss_on_fs(rss) + self.save_rss_on_fs(self.rss) print('removed: {}\nfrom: {}'.format(link, topic)) done = True @@ -100,7 +100,7 @@ def delete_topic(self, topic): exit() try: del self.rss[topic] - self.save_rss_on_fs(rss) + self.save_rss_on_fs(self.rss) print('Removed "{}" from your library.'.format(topic)) except KeyError: print('"{}" is not in your library!'.format(topic)) From d071cc72929de51aa29c7a05ac5d78b6a97da643 Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 20 Mar 2016 21:04:39 +0100 Subject: [PATCH 08/23] append options: -b -> -b [] shows the deeds from add support for github feeds --- termfeed/feed.py | 71 +++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/termfeed/feed.py b/termfeed/feed.py index 7baf383..94fa3a0 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -6,7 +6,7 @@ Usage: feed feed - feed -b + feed -b [] feed -a [] feed -d feed -t [] @@ -70,21 +70,31 @@ def print_feed(zipped): #'authors', 'published_parsed', 'title', 'title_detail']) def parse_time(post): - return c.info | arrow.get(dateutil.parser.parse(post.published)).humanize() + try: + return c.info | arrow.get(dateutil.parser.parse(post.published)).humanize() + except: + return c.info | arrow.get(dateutil.parser.parse(post.updated)).humanize() + + def parse_author(post): + if post.author_detail.name: + return post.author_detail.name + else: + return post.author_detail.email.split('@')[0] - r = re.compile(r'(\w+)\.git') + r = re.compile(r'(\w+)(?:/commits/\w+\.atom|\.git)') def repo(post): - repos = r.findall(post.summary_detail.base) + # print(post.keys()) + repos = r.findall(post.content[0].base)#summary_detail.base) if repos: return repos[0] else: return '' - table = [[ c.green | '[{}] '.format(num), + table = [[ c.green | '[{}]'.format(num), repo(post), parse_time(post), - c.dark_gray | post.author_detail.name, post.title] + c.dark_gray | parse_author(post), post.title] for num, post in reversed(list(zipped.items()))] print(tabulate(table, tablefmt="plain"))#, tablefmt="plain")) @@ -178,7 +188,11 @@ def fetch_feeds(urls): zipped += d.entries # https://wiki.python.org/moin/HowTo/Sorting#The_Old_Way_Using_Decorate-Sort-Undecorate - decorated = [(dateutil.parser.parse(post.published), i, post) for i, post in enumerate(zipped)] + try: + decorated = [(dateutil.parser.parse(post.published), i, post) for i, post in enumerate(zipped)] + except: + decorated = [(dateutil.parser.parse(post.updated), i, post) for i, post in enumerate(zipped)] + decorated.sort(reverse=True) zipped = [post for time, i, post in decorated] # undecorate @@ -214,30 +228,31 @@ def recurse(zipped): recurse(zipped) -def topic_choice(browse): +def topic_choice(browse, topic): - if browse: + if not topic in dbop.topics: + if browse: - tags = {} + tags = {} - for i, tag in enumerate(dbop.topics): - tags[i] = tag - print("{}) {}".format(i, tags[i])) + for i, tag in enumerate(dbop.topics): + tags[i] = tag + print("{}) {}".format(i, tags[i])) - try: - m = '\nChoose the topic (number)? : ' - try: # python 2 - uin = raw_input(m) - except NameError: # python 3 - uin = input(m) - uin = int(uin) - topic = tags[uin] - except: # catch all exceptions - print('\nInvalid choice!') - topic = 'General' + try: + m = '\nChoose the topic (number)? : ' + try: # python 2 + uin = raw_input(m) + except NameError: # python 3 + uin = input(m) + uin = int(uin) + topic = tags[uin] + except: # catch all exceptions + print('\nInvalid choice!') + topic = 'General' - else: - topic = 'General' + else: + topic = 'General' urls = dbop.read(topic) return urls @@ -272,10 +287,10 @@ def main(): if external: urls = [validate_feed(external)] else: - urls = topic_choice(browse) + urls = topic_choice(browse, category) # if not listing feeds - if add_link or delete or category or tags or rebuild or remove: + if add_link or delete or tags or rebuild or remove: fetch = False # updating URLs library From 587d4cb2ec0f2b9f170916a7e44f8a5b505103ea Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 20 Mar 2016 21:20:25 +0100 Subject: [PATCH 09/23] add option to load library from file and print library to stdout --- termfeed/database.py | 12 ++++++++++-- termfeed/feed.py | 18 +++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/termfeed/database.py b/termfeed/database.py index ad40b92..29abbe5 100644 --- a/termfeed/database.py +++ b/termfeed/database.py @@ -15,14 +15,22 @@ class DataBase: file = path.join(path.expanduser('~'), '.termfeed.json') - def rebuild_library(self): + def rebuild_library(self, file): + if not path: + file = path.join(path.dirname(__file__), 'rss.yaml') + elif not path.exists(file): + raise FileNotFoundError(file) - with open(path.join(path.dirname(__file__), 'rss.yaml'), 'r') as f: + with open(file, 'r') as f: rss = yaml.load(f) self.save_rss_on_fs(rss) print('created ".termfeed" in {}'.format(path.dirname(self.file))) + @property + def as_yaml(self): + return yaml.dump(self.rss, default_flow_style=False) + def __init__(self): # instantiate db if it's not created yet diff --git a/termfeed/feed.py b/termfeed/feed.py index 94fa3a0..10244a8 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -11,7 +11,8 @@ feed -d feed -t [] feed -D - feed -R + feed -R [] + feed --print-library feed (-h | --help) feed --version @@ -94,8 +95,9 @@ def repo(post): table = [[ c.green | '[{}]'.format(num), repo(post), parse_time(post), - c.dark_gray | parse_author(post), post.title] - for num, post in reversed(list(zipped.items()))] + c.dark_gray | parse_author(post), + post.title, + ] for num, post in reversed(list(zipped.items()))] print(tabulate(table, tablefmt="plain"))#, tablefmt="plain")) @@ -280,6 +282,9 @@ def main(): remove = args['-D'] tags = args['-t'] rebuild = args['-R'] + file = args[''] + print_library = args['--print-library'] + fetch = True @@ -290,7 +295,7 @@ def main(): urls = topic_choice(browse, category) # if not listing feeds - if add_link or delete or tags or rebuild or remove: + if add_link or delete or tags or rebuild or remove or print_library: fetch = False # updating URLs library @@ -313,11 +318,14 @@ def main(): print(dbop) if rebuild: - dbop.rebuild_library() + dbop.rebuild_library(file) if fetch: fetch_feeds(urls) + if print_library: + print(dbop.as_yaml) + # start if __name__ == '__main__': From 64c09373543f509ae15734a454850aabcee01526 Mon Sep 17 00:00:00 2001 From: boeddeker Date: Sun, 20 Mar 2016 21:26:09 +0100 Subject: [PATCH 10/23] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4cc5935..754247e 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,9 @@ If 1) you are a terminal addict, and 2) you want to stay up to date with the out - browse latest feed from the single link `` provided. - e.g. `$ feed https://news.ycombinator.com/rss` -`$ feed -b` +`$ feed -b []` -- browse latest feeds by category of your library. +- browse latest feeds by of your library. If is missing select input appear. `$ feed -t` @@ -51,9 +51,9 @@ If 1) you are a terminal addict, and 2) you want to stay up to date with the out `$ feed -D ` - Remove entire category (with its URLs) from library. -`$ feed -R` +`$ feed -R []` -- rebuild the library from `urls.py` +- rebuild the library. Default `rss.yaml` ### Features (what you can do?) From 750185b483f9b691391b9ecfdabf7975a6fe1c60 Mon Sep 17 00:00:00 2001 From: Christoph Date: Wed, 23 Mar 2016 20:58:12 +0100 Subject: [PATCH 11/23] start with change docopt and click and change database backend struct --- termfeed/database.py | 98 ++++++++++++++++++++++++++++++-------------- termfeed/feed.py | 60 +++++++++++++++++++++++---- 2 files changed, 121 insertions(+), 37 deletions(-) diff --git a/termfeed/database.py b/termfeed/database.py index 29abbe5..4de0610 100644 --- a/termfeed/database.py +++ b/termfeed/database.py @@ -7,42 +7,84 @@ dbop.py manipulate database add, update, delete """ -import yaml, json +import yaml, json, re +from plumbum import local from os import path from cached_property import cached_property +from pathlib import Path + + class DataBase: - file = path.join(path.expanduser('~'), '.termfeed.json') + file = local.env.home / '.termfeed.json' - def rebuild_library(self, file): - if not path: - file = path.join(path.dirname(__file__), 'rss.yaml') - elif not path.exists(file): - raise FileNotFoundError(file) + debug_flag = False + def debug(self, *args, **kwargs): + if self.debug_flag: + print(*args, **kwargs) + + def rebuild_library(self, file = None): + # force type local.path + file = local.path(file) if file else local.path(__file__).dirname / 'rss.yaml' + if not file.exists(): + raise FileNotFoundError(file) with open(file, 'r') as f: - rss = yaml.load(f) + self.rss = yaml.load(f) - self.save_rss_on_fs(rss) - print('created ".termfeed" in {}'.format(path.dirname(self.file))) + print('created ".termfeed" in {}'.format(file.dirname)) @property def as_yaml(self): return yaml.dump(self.rss, default_flow_style=False) - def __init__(self): - - # instantiate db if it's not created yet - if not (path.exists(self.file)): - self.rebuild_library() - - with open(self.file, 'r') as f: - self.rss = json.load(f) - - def save_rss_on_fs(self, rss): - with open(self.file, 'w') as f: - json.dump(rss, f) + @property + def as_yaml_v2(self): + import collections + data = collections.defaultdict(dict) + for topic in self.rss: + for link in self.rss[topic]: + data[link].setdefault('label', []).append(topic) + + r = re.compile(r'(\w+)(?:/commits/\w+\.atom|\.git)') + + for link in data: + if 'git' in link: + data[link].setdefault('flag', []).append('git') + + # ToDo: catch exeption + title, = r.findall(link)#summary_detail.base) + data[link]['title'] = title + + return yaml.dump(dict(data)) + + #def __init__(self): + # print(self.rss == self.rss) + + def save_on_fs_if_changed(self): + if not self.__rss == self.rss: + print('Backup changed library in {}.'.format(self.file)) + with open(self.file, 'w') as f: + json.dump(self.__rss, f) + + __rss = None + + @cached_property + def rss(self): + # The following will only called once + self.debug('Load library') + if not self.__rss: + if not self.file.exists(): + file = local.path(file) if file else local.path(__file__).dirname / 'rss.yaml' + if not file.exists(): + raise FileNotFoundError(file) + with open(file, 'r') as f: + return yaml.load(f) + else: + with open(self.file, 'r') as f: + self.__rss = json.load(f) + return self.__rss.copy() # ensure copy, for comp in __del__ @property def topics(self): @@ -76,25 +118,22 @@ def add_link(self, link, topic='General'): if topic in self.topics: if link not in self.rss[topic]: # to add a new url: copy, mutates, store back - temp = self.rss[topic] - temp.append(link) - self.rss[topic] = temp - self.save_rss_on_fs(self.rss) + #temp = self.rss[topic] + #temp.append(link) + #self.rss[topic] = temp + self.rss.append(link) print('Updated .. {}'.format(topic)) else: print('{} already exists in {}!!'.format(link, topic)) else: print('Created new category .. {}'.format(topic)) self.rss[topic] = [link] - self.save_rss_on_fs(self.rss) - def remove_link(self, link): done = False for topic in self.topics: if link in self.rss[topic]: self.rss[topic] = [l for l in self.rss[topic] if l != link] - self.save_rss_on_fs(self.rss) print('removed: {}\nfrom: {}'.format(link, topic)) done = True @@ -108,7 +147,6 @@ def delete_topic(self, topic): exit() try: del self.rss[topic] - self.save_rss_on_fs(self.rss) print('Removed "{}" from your library.'.format(topic)) except KeyError: print('"{}" is not in your library!'.format(topic)) diff --git a/termfeed/feed.py b/termfeed/feed.py index 10244a8..1e1cacf 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -6,13 +6,13 @@ Usage: feed feed - feed -b [] - feed -a [] - feed -d - feed -t [] - feed -D - feed -R [] - feed --print-library + feed -b [ | --debug] + feed -a [ | --debug] + feed -d [--debug] + feed -t [ | --debug] + feed -D [--debug] + feed -R [ | --debug] + feed --print-library [--debug] feed (-h | --help) feed --version @@ -26,6 +26,7 @@ -D TOPIC Remove entire cateogry (and its urls) from your library. -R Rebuild the library from the rss.yaml -h --help Show this screen. + --debug Enable debug messages """ @@ -38,6 +39,13 @@ import arrow import dateutil.parser from plumbum import colors as c +import click +import logging +logger = logging.getLogger(__name__) + +log_info = logger.info + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) try: from urllib import urlopen @@ -284,7 +292,9 @@ def main(): rebuild = args['-R'] file = args[''] print_library = args['--print-library'] + debug_flag = args['--debug'] + dbop.debug_flag = debug_flag fetch = True @@ -326,6 +336,40 @@ def main(): if print_library: print(dbop.as_yaml) + dbop.save_on_fs_if_changed() + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + + +@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS) +@click.pass_context +@click.option('--debug/--no-debug', default=False) +def cli(ctx, debug): + if ctx.invoked_subcommand is None: + print('I was invoked without subcommand') + else: + print('I am about to invoke %s' % ctx.invoked_subcommand) + print('Debug mode is %s' % ('on' if debug else 'off')) + +@cli.command() +@click.argument('url', nargs=1) +@click.argument('label', nargs=-1) # , help='One or more labels for this url.' +@click.option('--title', default='') +#@click.option('-l', '--label', default='General', multiple=True) +def add(url, label, **kwargs): + print('Synching', kwargs) + pass + +@cli.command() +@click.argument('name') # , help='namme must be a url or a label' +def remove(**kwargs): + pass + +@cli.command() +def show(**kwargs): + print(dbop.as_yaml) + print(dbop.as_yaml_v2) + # start if __name__ == '__main__': @@ -333,4 +377,6 @@ def main(): print('No Internet Connection!') exit() + # cli() + main() From 7576d270eb10f212a6989788b3c405e2aaf933c4 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Thu, 23 Mar 2017 16:47:45 +0800 Subject: [PATCH 12/23] new: dev: add cached-property package --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e52f69e..792751d 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,8 @@ license = "MIT", author_email='iamaziz.alto@gmail.com', version='0.0.11', - install_requires=['feedparser', 'pyyaml', 'docopt', 'plumbum', 'arrow'], + install_requires=[ + 'feedparser', 'pyyaml', 'docopt', 'plumbum', 'arrow', 'cached-property==1.3.0'], packages=['termfeed'], scripts=[], entry_points={ From 36e5f4e7db7b82a2c883d305c0c9ae56dec1348e Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Thu, 23 Mar 2017 16:48:21 +0800 Subject: [PATCH 13/23] new: dev: add __main__ file. --- termfeed/__main__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 termfeed/__main__.py diff --git a/termfeed/__main__.py b/termfeed/__main__.py new file mode 100644 index 0000000..7037693 --- /dev/null +++ b/termfeed/__main__.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- + +from .feed import _connected, main + +if __name__ == '__main__': + + if not _connected(): + print('No Internet Connection!') + exit() + + main() From 920274c40504289525a6f7dc38a634690e4c5bad Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Fri, 24 Mar 2017 09:15:34 +0800 Subject: [PATCH 14/23] fix: dev: fix UnboundLocalError on file var --- termfeed/database.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/termfeed/database.py b/termfeed/database.py index 4de0610..ac79845 100644 --- a/termfeed/database.py +++ b/termfeed/database.py @@ -76,7 +76,12 @@ def rss(self): self.debug('Load library') if not self.__rss: if not self.file.exists(): - file = local.path(file) if file else local.path(__file__).dirname / 'rss.yaml' + try: + file = local.path(file) if file else local.path(__file__).dirname / 'rss.yaml' + except UnboundLocalError: + file = local.path(self.file) \ + if self.file else local.path(__file__).dirname / 'rss.yaml' + if not file.exists(): raise FileNotFoundError(file) with open(file, 'r') as f: From e0437bcbbe7a77e90e2074d10e78146afc8c4377 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Fri, 24 Mar 2017 09:33:35 +0800 Subject: [PATCH 15/23] chg: dev: add option to create json setting --- termfeed/database.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/termfeed/database.py b/termfeed/database.py index ac79845..097d3a9 100644 --- a/termfeed/database.py +++ b/termfeed/database.py @@ -83,7 +83,7 @@ def rss(self): if self.file else local.path(__file__).dirname / 'rss.yaml' if not file.exists(): - raise FileNotFoundError(file) + self.create_file(file) with open(file, 'r') as f: return yaml.load(f) else: @@ -91,6 +91,18 @@ def rss(self): self.__rss = json.load(f) return self.__rss.copy() # ensure copy, for comp in __del__ + @staticmethod + def create_file(file): + """create file.""" + input_res = input("Do you want to create setting file ([y]/n):") + if input_res == 'y' or input_res == '': + pass + else: + sys.exit(1) + with open(file, 'w') as f: + yaml.dump({'General': []}, f) + print('Setting created at: {}'.format(file)) + @property def topics(self): return self.rss.keys() From 356c2b123a74fae95f3fa11f5b61020ae1ba96a2 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Fri, 24 Mar 2017 09:40:42 +0800 Subject: [PATCH 16/23] chg: usr: exit when no rss found --- termfeed/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/termfeed/database.py b/termfeed/database.py index 097d3a9..ce090d3 100644 --- a/termfeed/database.py +++ b/termfeed/database.py @@ -89,7 +89,11 @@ def rss(self): else: with open(self.file, 'r') as f: self.__rss = json.load(f) - return self.__rss.copy() # ensure copy, for comp in __del__ + try: + return self.__rss.copy() # ensure copy, for comp in __del__ + except AttributeError: + print('No rss found. Please add rss url.') + sys.exit(1) @staticmethod def create_file(file): From 6e61cee9178df932c03fb9083d6ceef1b9fd6540 Mon Sep 17 00:00:00 2001 From: Christoph Date: Fri, 24 Mar 2017 11:30:54 +0100 Subject: [PATCH 17/23] pep8 --- setup.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 792751d..60b4f5b 100644 --- a/setup.py +++ b/setup.py @@ -8,15 +8,22 @@ setup( name='TermFeed', - description=('Browse, read, and open your favorite rss feed in the terminal (without curses).'), + description=( + 'Browse, read, and open your favorite rss feed in the terminal (without curses).'), author='Aziz Alto', url='https://github.com/iamaziz/TermFeed', download_url='https://github.com/iamaziz/TermFeed/archive/master.zip', - license = "MIT", + license="MIT", author_email='iamaziz.alto@gmail.com', version='0.0.11', install_requires=[ - 'feedparser', 'pyyaml', 'docopt', 'plumbum', 'arrow', 'cached-property==1.3.0'], + 'feedparser', + 'pyyaml', + 'docopt', + 'plumbum', + 'arrow', + 'cached-property>=1.3.0', + ], packages=['termfeed'], scripts=[], entry_points={ From 2a8894664a4289a41ba58d276a9266b2f4dd3891 Mon Sep 17 00:00:00 2001 From: Christoph Date: Fri, 24 Mar 2017 11:36:47 +0100 Subject: [PATCH 18/23] bumpversion patch --- .bumpversion.cfg | 9 +++++++++ setup.py | 2 +- termfeed/feed.py | 4 ++-- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..02c1e6d --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,9 @@ +[bumpversion] +current_version = 0.0.12 +commit = False +tag = False + +[bumpversion:file:setup.py] + +[bumpversion:file:termfeed/feed.py] + diff --git a/setup.py b/setup.py index 60b4f5b..12ca621 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ download_url='https://github.com/iamaziz/TermFeed/archive/master.zip', license="MIT", author_email='iamaziz.alto@gmail.com', - version='0.0.11', + version='0.0.12', install_requires=[ 'feedparser', 'pyyaml', diff --git a/termfeed/feed.py b/termfeed/feed.py index 1e1cacf..9a14519 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -1,7 +1,7 @@ #!/usr/bin/env python #-*- coding: utf-8 -*- -"""TermFeed 0.0.11 +"""TermFeed 0.0.12 Usage: feed @@ -279,7 +279,7 @@ def validate_feed(url): def main(): args = docopt( - __doc__, version="TermFeed 0.0.11 (with pleasure by: Aziz Alto)") + __doc__, version="TermFeed 0.0.12 (with pleasure by: Aziz Alto)") # parse args browse = args['-b'] From 4b52387284c61ef907b2fe5061c54659317b1f09 Mon Sep 17 00:00:00 2001 From: Christoph Date: Fri, 24 Mar 2017 12:15:21 +0100 Subject: [PATCH 19/23] gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ba74660..a8c42eb 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ docs/_build/ # PyBuilder target/ + +#IDE's +.idea/ From 917d9be6b9cc9c9c7e8ddc1cfe5eac6241ae254d Mon Sep 17 00:00:00 2001 From: Christoph Date: Fri, 24 Mar 2017 12:15:48 +0100 Subject: [PATCH 20/23] revert some changes --- termfeed/__main__.py | 6 +- termfeed/database.py | 251 +++++++++++----------- termfeed/feed.py | 482 +++++++++++++++++++++++-------------------- 3 files changed, 394 insertions(+), 345 deletions(-) diff --git a/termfeed/__main__.py b/termfeed/__main__.py index 7037693..d4ddb77 100644 --- a/termfeed/__main__.py +++ b/termfeed/__main__.py @@ -1,12 +1,8 @@ #!/usr/bin/env python #-*- coding: utf-8 -*- -from .feed import _connected, main +from .feed import main if __name__ == '__main__': - if not _connected(): - print('No Internet Connection!') - exit() - main() diff --git a/termfeed/database.py b/termfeed/database.py index ce090d3..7f2d553 100644 --- a/termfeed/database.py +++ b/termfeed/database.py @@ -4,176 +4,187 @@ """ database operations. -dbop.py manipulate database add, update, delete +database.py manipulate database add, update, delete """ import yaml, json, re from plumbum import local +from plumbum import colors as c from os import path from cached_property import cached_property from pathlib import Path +import logging +logger = logging.getLogger(__name__) +log_info = logger.info +log_debug = logger.debug class DataBase: file = local.env.home / '.termfeed.json' - - debug_flag = False - - def debug(self, *args, **kwargs): - if self.debug_flag: - print(*args, **kwargs) - - def rebuild_library(self, file = None): - # force type local.path - file = local.path(file) if file else local.path(__file__).dirname / 'rss.yaml' - if not file.exists(): - raise FileNotFoundError(file) - with open(file, 'r') as f: - self.rss = yaml.load(f) - - print('created ".termfeed" in {}'.format(file.dirname)) - - @property - def as_yaml(self): - return yaml.dump(self.rss, default_flow_style=False) - - @property - def as_yaml_v2(self): - import collections - data = collections.defaultdict(dict) - for topic in self.rss: - for link in self.rss[topic]: - data[link].setdefault('label', []).append(topic) - - r = re.compile(r'(\w+)(?:/commits/\w+\.atom|\.git)') - - for link in data: - if 'git' in link: - data[link].setdefault('flag', []).append('git') - - # ToDo: catch exeption - title, = r.findall(link)#summary_detail.base) - data[link]['title'] = title - - return yaml.dump(dict(data)) - - #def __init__(self): - # print(self.rss == self.rss) - - def save_on_fs_if_changed(self): - if not self.__rss == self.rss: - print('Backup changed library in {}.'.format(self.file)) - with open(self.file, 'w') as f: - json.dump(self.__rss, f) - - __rss = None + __data = None + dry_run = False @cached_property - def rss(self): + def data(self): # The following will only called once - self.debug('Load library') - if not self.__rss: + log_info('Load library') + if not self.__data: if not self.file.exists(): - try: - file = local.path(file) if file else local.path(__file__).dirname / 'rss.yaml' - except UnboundLocalError: - file = local.path(self.file) \ - if self.file else local.path(__file__).dirname / 'rss.yaml' - + file = local.path(__file__).dirname / 'db.yaml' if not file.exists(): - self.create_file(file) + raise FileNotFoundError(file) with open(file, 'r') as f: + log_info('Open yaml') return yaml.load(f) else: with open(self.file, 'r') as f: - self.__rss = json.load(f) - try: - return self.__rss.copy() # ensure copy, for comp in __del__ - except AttributeError: - print('No rss found. Please add rss url.') - sys.exit(1) - - @staticmethod - def create_file(file): - """create file.""" - input_res = input("Do you want to create setting file ([y]/n):") - if input_res == 'y' or input_res == '': - pass + log_info('Open json') + self.__data = json.load(f) + return self.__data.copy() # ensure copy, for comp in __del__ + + # def set_data(self, data): + # verify_data(data) + # self.__data = data + # del self.data + # with self: + # pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # type, value, traceback + verify_data(self.data) + if not self.dry_run: + print('Backup changed library in {}.'.format(self.file)) + with open(self.file, 'w') as f: + json.dump(self.data, f) + log_info('Write to json') + log_debug(json.dumps(self.data)) + log_debug(self.as_yaml) else: - sys.exit(1) - with open(file, 'w') as f: - yaml.dump({'General': []}, f) - print('Setting created at: {}'.format(file)) + print('Normaly would write the following to file {}: '.format(self.file)) + print(c.highlight | c.black | json.dumps(self.data)) + + debug_flag = False @property - def topics(self): - return self.rss.keys() - # return list(self.db.keys()) + def labels(self): + labels = set() + for _, val in self.data.items(): + labels |= set(val['label']) + return labels - def read(self, topic): - if topic in self.topics: - return self.rss[topic] - else: - return None + @property + def as_yaml(self): + return yaml.dump(self.data) # , default_flow_style=False + + def link_for_label(self, *labels): + return [link for link, val in self.data.items() if any(True for l in labels if l in val['label'])] - def browse_links(self, topic): - if topic in self.topics: - links = self.rss[topic] - print('{} resources:'.format(topic)) + @property + def links(self): + return self.data.keys() + + def browse_links(self, label): + if label in self.labels: + links = self(label) + print('{} resources:'.format(label)) for link in links: print('\t{}'.format(link)) else: - print('no category named {}'.format(topic)) + print('no category named {}'.format(label)) print(self) def __str__(self): - out = 'available topics: \n\t' + '\n\t'.join(self.topics) + out = 'available lables: \n\t' + '\n\t'.join(self.labels) return(out) - def print_topics(self): + def print_labels(self): print(self) - def add_link(self, link, topic='General'): - if topic in self.topics: - if link not in self.rss[topic]: - # to add a new url: copy, mutates, store back - #temp = self.rss[topic] - #temp.append(link) - #self.rss[topic] = temp - self.rss.append(link) - print('Updated .. {}'.format(topic)) - else: - print('{} already exists in {}!!'.format(link, topic)) + def link_as_yaml(self, link): + return yaml.dump({link:self.data[link]}) + + def add_link(self, link, *labels, flag = None, title=''): + if link not in self.data: + if not flag: + flag = [] + template = dict(title=title, flag=flag, label=list(labels)) + self.data[link] = verify_entry(template, link) + print('Added:') + print(self.link_as_yaml(link)) + if logger.level <= logging.INFO: + print(self.as_yaml) + elif not title == '' or not title == self.data[link]['title']: + self.data[link]['label'] = list(set(self.data[link]['label']) | set(labels)) + self.data[link]['title'] = title + log_info('Title has changed') + print(self.as_yaml) + + elif set(labels) | set(self.data[link]['label']) == set(self.data[link]['label']): + print('{} already exists and has all labels: {}!!'.format(link, labels)) + print(self.as_yaml) else: - print('Created new category .. {}'.format(topic)) - self.rss[topic] = [link] + self.data[link]['label'] = list(set(self.data[link]['label']) | set(labels)) + # print('Created new category .. {}'.format(topic)) + print(self.link_as_yaml(link)) + if logger.level <= logging.INFO: + print(self.as_yaml) def remove_link(self, link): done = False - for topic in self.topics: - if link in self.rss[topic]: - self.rss[topic] = [l for l in self.rss[topic] if l != link] - print('removed: {}\nfrom: {}'.format(link, topic)) - done = True - - if not done: + if link in self.data: + del self.data[link] + print('removed: {}'.format(link)) + else: print('URL not found: {}'.format(link)) - - def delete_topic(self, topic): - if topic == 'General': + def delete_topic(self, label): + if label == '': print('Default topic "General" cannot be removed.') exit() try: - del self.rss[topic] - print('Removed "{}" from your library.'.format(topic)) + for link in self.data: + if label in self.data[link]['label']: + self.data[link]['label'] = list(set(self.data[link]['label']) - set(label)) + print('Removed "{}" from your library.'.format(label)) except KeyError: - print('"{}" is not in your library!'.format(topic)) + print('"{}" is not in your library!'.format(label)) exit() +def verify_entry(entry, link): + allowed_keys = {'label', 'flag', 'title'} + if not entry: + entry = dict() + if not (entry.keys() | allowed_keys) == allowed_keys: + print('The url {} has invalid keys: '.format(link), entry) + exit() + if not isinstance(entry.setdefault('title', ''), str): + print('The url {} has invalid title member: {}'.format(link, entry['title'])) + exit() + if not isinstance(entry.setdefault('flag', []), list): + print('The url {} has invalid flag type: {}'.format(link, entry['flag'])) + exit() + if not all([isinstance(f, str) for f in entry['flag']]): + print('The url {} has invalid flag member: {}'.format(link, entry['flag'])) + exit() + if not isinstance(entry.setdefault('label', []), list): + print('The url {} has invalid flag type: {}'.format(link, entry['label'])) + exit() + if not all([isinstance(l, str) for l in entry['label']]): + print('The url {} has invalid flag member: {}'.format(link, entry['label'])) + exit() + return entry + + +def verify_data(data): + for link in data: + data[link] = verify_entry(data[link], link) + # if __name__ == '__main__': # for l in read('News'): diff --git a/termfeed/feed.py b/termfeed/feed.py index 9a14519..39a2074 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -1,60 +1,33 @@ #!/usr/bin/env python -#-*- coding: utf-8 -*- - -"""TermFeed 0.0.12 - -Usage: - feed - feed - feed -b [ | --debug] - feed -a [ | --debug] - feed -d [--debug] - feed -t [ | --debug] - feed -D [--debug] - feed -R [ | --debug] - feed --print-library [--debug] - feed (-h | --help) - feed --version - -Options: - List feeds from the default category 'General' of your library. - List feeds from the provided url source. - -b Browse feed by category avaialble in the database file. - -a URL Add new url to database under [] (or 'General' otherwise). - -d URL Delete from the database file. - -t See the stored categories in your library, or list the URLs stored under in your library. - -D TOPIC Remove entire cateogry (and its urls) from your library. - -R Rebuild the library from the rss.yaml - -h --help Show this screen. - --debug Enable debug messages - -""" - - -from __future__ import print_function -import sys -import webbrowser -import feedparser + +import logging +import os import re +import webbrowser + import arrow +import click import dateutil.parser +import feedparser +import yaml +yaml.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) +yaml.add_representer(feedparser.FeedParserDict, + yaml.representer.SafeRepresenter.represent_dict) + +from bs4 import BeautifulSoup from plumbum import colors as c -import click -import logging +from tabulate import tabulate +from urllib.request import urlopen +import termfeed.database + logger = logging.getLogger(__name__) log_info = logger.info +log_debug = logger.debug -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) - -try: - from urllib import urlopen -except ImportError: - from urllib.request import urlopen +dbop = termfeed.database.DataBase() -import termfeed.database -dbop = termfeed.database.DataBase() def _connected(): """check internet connect""" host = 'http://google.com' @@ -65,18 +38,19 @@ def _connected(): except: return False + def open_page(url, title): with c.info: print('\topening ... {}\n'.format(title.encode('utf8'))) # open page in browser webbrowser.open(url) -from tabulate import tabulate + def print_feed(zipped): - #keys() - #dict_keys(['id', 'links', 'summary', 'author', 'guidislink', - #'author_detail', 'link', 'summary_detail', 'published', 'content', - #'authors', 'published_parsed', 'title', 'title_detail']) + # keys() + # dict_keys(['id', 'links', 'summary', 'author', 'guidislink', + # 'author_detail', 'link', 'summary_detail', 'published', 'content', + # 'authors', 'published_parsed', 'title', 'title_detail']) def parse_time(post): try: @@ -85,77 +59,100 @@ def parse_time(post): return c.info | arrow.get(dateutil.parser.parse(post.updated)).humanize() def parse_author(post): - if post.author_detail.name: - return post.author_detail.name - else: - return post.author_detail.email.split('@')[0] + try: + if post.author_detail.name: + return post.author_detail.name + else: + return post.author_detail.email.split('@')[0] + except AttributeError: + return 'unknown' + except BaseException as e: + print(c.red | yaml.dump(post)) + import sys + raise e - r = re.compile(r'(\w+)(?:/commits/\w+\.atom|\.git)') + # r = re.compile(r'(\w+)(?:/commits/\w+\.atom|\.git)') def repo(post): - # print(post.keys()) - repos = r.findall(post.content[0].base)#summary_detail.base) - if repos: - return repos[0] - else: - return '' - - table = [[ c.green | '[{}]'.format(num), - repo(post), - parse_time(post), - c.dark_gray | parse_author(post), - post.title, - ] for num, post in reversed(list(zipped.items()))] - - print(tabulate(table, tablefmt="plain"))#, tablefmt="plain")) - - -def print_desc(topic, txt): + return dbop.data[post.title_detail.base]['title'] + + # try: + table = [[c.green | '[{}]'.format(num), + repo(post), + parse_time(post), + c.dark_gray | parse_author(post), + post.title, + ] for num, post in reversed(list(zipped.items()))] + # except AttributeError as e: + # print('Bug:', post.keys()) + # print(post.published) + # print(c.magenta | yaml.dump(post)) + # print(post.title) + # print(post.title_detail) + # raise e + # else: + # print('Bug:', dir(post), post.keys()) + + print(tabulate(table, tablefmt="plain")) # , tablefmt="plain")) + + +def print_desc(topic, txt, post): with c.info: try: - print('\n\n{}:'.format(topic)) + + print('\n\nTitle : {}:'.format(topic)) + except UnicodeEncodeError: pass - with c.bold: - print('\n\t{}'.format(txt)) + with c.dark_gray: + yaml.add_representer( + tuple, yaml.representer.SafeRepresenter.represent_list) + yaml.add_representer(feedparser.FeedParserDict, + yaml.representer.SafeRepresenter.represent_dict) + + import copy + post_copy = copy.deepcopy(post) + + # post_copy = post.copy() + for key in list(post_copy.keys()): + if '_parsed' in key \ + or '_detail' in key \ + or key in ('guidislink', 'link', 'links', 'id', 'summary', 'authors', 'title'): + del post_copy[key] + if 'content' is key: + for i in range(len(post_copy[key])): + post_copy[key][i] = clean_txt(post_copy[key][i]['value']) + print(yaml.dump(post_copy)) def open_it(): - try: - txt = '\n\n\t Open it in browser ? [y/n] ' - try: - q = raw_input(txt) # python 2 - except NameError: - q = input(txt) # python 3 + if os.environ.get('DISPLAY'): + return click.confirm('\n\n\t Open it in browser ?') - print('\n') - if q == 'y': - return True - except KeyboardInterrupt: - print('\n') + elif click.confirm('No display aviable, do you want to continue?', default=True, show_default=False): + log_info('Confirm True') return False + else: + log_info('Confirm False') + exit() + return False + + def clean_txt(txt): """clean txt from e.g. html tags""" - cleaned = re.sub(r'<.*?>', '', txt) # remove html - cleaned = cleaned.replace('<', '<').replace('>', '>') # retain html code tags - cleaned = cleaned.replace('"', '"') - cleaned = cleaned.replace('’', "'") - cleaned = cleaned.replace(' ', ' ') # italized text - return cleaned + clean_text = BeautifulSoup(txt, "html.parser").text + return clean_text + def _continue(): try: - msg = """\nPress: Enter to continue, ... [NUM] for short description / open a page, ... or CTRL-C to exit: """ + msg = """\nPress: Enter to continue, ... [NUM] for short description / open a page, ... or CTRL-C to exit""" with c.warn: - print(msg, end='') - # kb is the pressed keyboard key - try: - kb = raw_input() - except NameError: - kb = input() - return kb + return click.prompt(msg, type=str, default='', show_default=False) + # click.confirm('No display aviaable, do you want to continue?', + # default=True, show_default=False): except KeyboardInterrupt: # return False @@ -163,7 +160,6 @@ def _continue(): def parse_feed(url): - d = feedparser.parse(url) # validate rss URL @@ -171,40 +167,35 @@ def parse_feed(url): return d else: print("INVALID URL feed: {}".format(url)) + # exit() return None def fetch_feeds(urls): - - feeds = [] - for i, url in enumerate(urls): - - feeds += [parse_feed(url)] - - #if d is None: - # continue # to next url - # feeds source l = len(urls) - 1 - for i, d in enumerate(feeds): - title = url if d.feed.title else d.feed.title - print(c.magenta | " {}/{} SOURCE>> {}".format(i, l, d.feed.title) ) - - # print out feeds + feeds = [parse_feed(url) for url in urls] + for f, u in zip(feeds, urls): + if not f: + print(c.warn | 'ERROR with {}'.format(u)) + feeds.remove(f) zipped = [] - for d in feeds: + for i, d in enumerate(feeds): + # title = url if d.feed.title else d.feed.title + print(c.magenta | " {}/{} SOURCE>> {}".format(i, l, d.feed.title)) zipped += d.entries # https://wiki.python.org/moin/HowTo/Sorting#The_Old_Way_Using_Decorate-Sort-Undecorate try: - decorated = [(dateutil.parser.parse(post.published), i, post) for i, post in enumerate(zipped)] + decorated = [(dateutil.parser.parse(post.published), i, post) + for i, post in enumerate(zipped)] except: - decorated = [(dateutil.parser.parse(post.updated), i, post) for i, post in enumerate(zipped)] - + decorated = [(dateutil.parser.parse(post.updated), i, post) + for i, post in enumerate(zipped)] decorated.sort(reverse=True) - zipped = [post for time, i, post in decorated] # undecorate + zipped = [post for time, i, post in decorated] # undecorate zipped = dict(enumerate(zipped)) @@ -223,7 +214,7 @@ def recurse(zipped): try: desc = zipped[int(kb)].description desc = clean_txt(desc) - print_desc(title, desc) + print_desc(title, desc, zipped[int(kb)]) except AttributeError: print('\n\tNo description available!!') @@ -238,145 +229,196 @@ def recurse(zipped): recurse(zipped) -def topic_choice(browse, topic): +def topic_choice(browse, *labels): + log_info(labels) + + labels = set(labels) & dbop.labels - if not topic in dbop.topics: + log_info(labels) + + if not labels: if browse: tags = {} - for i, tag in enumerate(dbop.topics): + for i, tag in enumerate(dbop.labels | {'all'}): tags[i] = tag print("{}) {}".format(i, tags[i])) + uin = click.prompt('\nChoose the topic (number)? ', type=int) try: - m = '\nChoose the topic (number)? : ' - try: # python 2 - uin = raw_input(m) - except NameError: # python 3 - uin = input(m) - uin = int(uin) - topic = tags[uin] - except: # catch all exceptions + labels = tags[uin] + except: # catch all exceptions print('\nInvalid choice!') - topic = 'General' + labels = [] else: - topic = 'General' - urls = dbop.read(topic) + labels = [] + if labels == 'all': + urls = dbop.links + else: + urls = dbop.link_for_label(labels) return urls def validate_feed(url): if parse_feed(url): - return url + return True else: - exit() + return False -# from .support.docopt import docopt -from docopt import docopt +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) -def main(): - args = docopt( - __doc__, version="TermFeed 0.0.12 (with pleasure by: Aziz Alto)") - - # parse args - browse = args['-b'] - external = args[''] - add_link = args['-a'] - category = args[''] - delete = args['-d'] - remove = args['-D'] - tags = args['-t'] - rebuild = args['-R'] - file = args[''] - print_library = args['--print-library'] - debug_flag = args['--debug'] - - dbop.debug_flag = debug_flag - - fetch = True - - # get rss urls - if external: - urls = [validate_feed(external)] - else: - urls = topic_choice(browse, category) - # if not listing feeds - if add_link or delete or tags or rebuild or remove or print_library: - fetch = False +@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS) +@click.pass_context +@click.option('-l', '--label', multiple=True) +@click.option('-n', '--dry-run', is_flag=True) +# @click.option('--debug/--no-debug', default=False) +@click.option('-v', '--verbose', count=True, help='Levels: -v:INFO, -vvv:DEBUG') +def cli(ctx, verbose, label, dry_run): + logging.basicConfig(format=(c.dark_gray | logging.BASIC_FORMAT)) + + logger_pkg = logging.getLogger(__package__) + if verbose >= 3: + logger_pkg.setLevel(logging.DEBUG) + elif verbose >= 1: + logger_pkg.setLevel(logging.INFO) + + if dry_run: + dbop.dry_run = True - # updating URLs library - if add_link: - url = validate_feed(add_link) - if category: - dbop.add_link(url, category) - else: - dbop.add_link(url) - if delete: - dbop.remove_link(delete) - - if remove: - dbop.delete_topic(remove) - # display resource contents - if tags: - if category: - dbop.browse_links(category) + if ctx.invoked_subcommand is None: + log_info('I was invoked without subcommand') + if not label: + fetch_feeds(dbop.links) else: - print(dbop) + fetch_feeds(dbop.link_for_label(*label)) + else: + log_info('I am about to invoke %s' % ctx.invoked_subcommand) - if rebuild: - dbop.rebuild_library(file) - if fetch: - fetch_feeds(urls) +@cli.command() +@click.argument('url', nargs=1) +@click.argument('label', nargs=-1) # , help='One or more labels for this url.' +@click.option('--title', default='') +@click.option('-f', '--flag', multiple=True) +def add(url, label, title, flag): + + regex = re.compile( + r'(https://github\.com/|git@github\.com:)(\w+)/([\w-]+)\.git') + if regex.match(url): + (r_prefix, r_user, r_name), = regex.findall(url) + + url_offer = 'https://github.com/' + r_user + \ + '/' + r_name + '/commits/master.atom' + + with c.info: + if click.confirm(''' + It seams, that you try to add a github repo. + But the url seams to be wrong. + Do you mean {}?'''.format(url_offer), default=True): + url = url_offer + flag = list(set(flag) | {'github', 'git'}) + elif 'git' in url and click.confirm(''' + It seams, that your url is a git url, add Flag? + '''.format(url_offer), default=True): + flag = list(set(flag) | set('git')) + if title == '': + title = r_name + + if validate_feed(url): + with dbop as db: + db.add_link(url, *label, title=title, flag=flag) + print("Add URL feed: {}".format(url)) - if print_library: - print(dbop.as_yaml) - dbop.save_on_fs_if_changed() +@cli.command() +@click.argument('name') # , help='namme must be a url or a label' +@click.argument('url', nargs=1) +def remove(url): + if url in dbop.links: + # with dbop as dbop: + dbop.remove_link(url) + print("Removed URL feed: {}".format(url)) + else: + print("Could not find URL feed: {}".format(url)) -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +@cli.command() +@click.option('--label/--no-label', default=False) +def show(label): + if label: + print('Labels: ', dbop.labels) + print(dbop.as_yaml) -@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS) -@click.pass_context -@click.option('--debug/--no-debug', default=False) -def cli(ctx, debug): - if ctx.invoked_subcommand is None: - print('I was invoked without subcommand') - else: - print('I am about to invoke %s' % ctx.invoked_subcommand) - print('Debug mode is %s' % ('on' if debug else 'off')) @cli.command() -@click.argument('url', nargs=1) -@click.argument('label', nargs=-1) # , help='One or more labels for this url.' -@click.option('--title', default='') -#@click.option('-l', '--label', default='General', multiple=True) -def add(url, label, **kwargs): - print('Synching', kwargs) - pass +@click.argument('label', nargs=-1) # , help='One or more labels for this url.' +def browse(label): + urls = topic_choice(browse, *label) + fetch_feeds(urls) + @cli.command() -@click.argument('name') # , help='namme must be a url or a label' -def remove(**kwargs): - pass +@click.pass_context +def edit(ctx): + import tempfile + from contextlib import suppress + from subprocess import call + + EDITOR = os.environ.get('EDITOR', 'dav') # that easy! + #EDITOR = 'dav' + if EDITOR == 'suplemon': + os.environ['TERM'] = 'xterm-256color' + + initial_message = dbop.as_yaml + with tempfile.NamedTemporaryFile(suffix=".tmp", mode='w+') as tf: + tf.write(initial_message) + tf.flush() + with suppress(KeyboardInterrupt): + call([EDITOR, tf.name]) + + # do the parsing with `tf` using regular File operations. + # for instance: + if False: + tf.seek(0) + edited_message = tf.read() + print(edited_message) + ctx.invoke(load, file=tf.name) + @cli.command() -def show(**kwargs): - print(dbop.as_yaml) - print(dbop.as_yaml_v2) +@click.argument('file', type=click.File('r')) +def load(file): + with open(file) as f: + data = yaml.load(f.read()) + print('Loaded: ', file, os.path.exists(file)) + print(yaml.dump(data)) + #log_debug('data keys: ', data.keys()) + #log_debug('data keys: ', data) + + verification = [validate_feed(link) for link in data] + if not all(verification): + for link in data: + if not validate_feed(link): + print("INVALID URL feed: {}".format(link)) + exit() -# start -if __name__ == '__main__': + with dbop as db: + db.data = data + print(yaml.dump(data)) + + +def main(): if not _connected(): print('No Internet Connection!') exit() - # cli() + cli() +# start +if __name__ == '__main__': main() From 855e2de530ce63b9abbaf67e64d883350dcc695a Mon Sep 17 00:00:00 2001 From: Christoph Date: Fri, 24 Mar 2017 12:15:53 +0100 Subject: [PATCH 21/23] test --- tests/feed_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/feed_test.py diff --git a/tests/feed_test.py b/tests/feed_test.py new file mode 100644 index 0000000..2f21983 --- /dev/null +++ b/tests/feed_test.py @@ -0,0 +1,13 @@ +import click +from click.testing import CliRunner + + +from termfeed.feed import * + +def test_feed(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(cli, input='y') + #print(result.output) + #assert result.output == 'Hello World!\n' + # assert result.exit_code == 0 From 3103c213e0ad59e3917dafaa71e3986eb22aac68 Mon Sep 17 00:00:00 2001 From: Christoph Date: Fri, 24 Mar 2017 12:15:59 +0100 Subject: [PATCH 22/23] makefile --- Makefile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4476759 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ + + +help: + cat Makefile + +install: + pip install . + +.PHONY: tests +tests: + nosetests tests/ + +feed: + python -m feed + +bumpversion_patch: + bumpversion patch + +bumpversion_minor: + bumpversion minor + +bumpversion_mayor: + bumpversion mayor From ffd7c05113b327e2ea927bf8cb2c5843c68c7471 Mon Sep 17 00:00:00 2001 From: Christoph Date: Fri, 24 Mar 2017 12:46:35 +0100 Subject: [PATCH 23/23] add example db.yaml --- README.md | 24 ++++++++++++++++++++++++ termfeed/db.yaml | 13 +++++++++++++ termfeed/feed.py | 6 +++++- 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 termfeed/db.yaml diff --git a/README.md b/README.md index 754247e..c037269 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,31 @@ To read, preview, open, store, or delete your favorite RSS feeds from the comman If 1) you are a terminal addict, and 2) you want to stay up to date with the outside world by reading quick feed and summaries WITHOUT having to leave your terminal; then TermFeed is for you. These are the main reasons I created TermFeed. +# Changes agains original +```Usage: feed [OPTIONS] COMMAND [ARGS]... + +Options: + -l, --label TEXT + -n, --dry-run + -v, --verbose Levels: -v:INFO, -vvv:DEBUG + -h, --help Show this message and exit. + +Commands: + add + browse + edit + load + remove + show + ``` + +Config file is a termdeef/db.yaml +Modifications are made to support git repos. + + + +# Old Readme ### Usage diff --git a/termfeed/db.yaml b/termfeed/db.yaml new file mode 100644 index 0000000..e808937 --- /dev/null +++ b/termfeed/db.yaml @@ -0,0 +1,13 @@ +https://github.com/boeddeker/TermFeed/commits/master.atom: + flag: [git, github] + label: [github] + title: TermFeed +https://github.com/boeddeker/TermFeed/commits/master.atom: + flag: [git, github] + label: [github] + title: TermFeed +http://blog.jupyter.org/rss/: + flag: [] + label: [github] + title: blog.jupyter + diff --git a/termfeed/feed.py b/termfeed/feed.py index 39a2074..44f8c33 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -74,7 +74,11 @@ def parse_author(post): # r = re.compile(r'(\w+)(?:/commits/\w+\.atom|\.git)') def repo(post): - return dbop.data[post.title_detail.base]['title'] + try: + return dbop.data[post.title_detail.base]['title'] + except: + print('Keys: ', dbop.data.keys()) + raise # try: table = [[c.green | '[{}]'.format(num),