diff --git a/buildozer.spec b/buildozer.spec index 872469c..dcc45ef 100644 --- a/buildozer.spec +++ b/buildozer.spec @@ -52,7 +52,7 @@ requirements = python3,android,pyjnius,beautifulsoup4,brotlipy,cached-property,c #icon.filename = %(source.dir)s/data/icon.png # (str) Supported orientation (one of landscape, sensorLandscape, portrait or all) -orientation = all +orientation = portrait # (list) List of service to declare #services = NAME:ENTRYPOINT_TO_PY,NAME2:ENTRYPOINT2_TO_PY diff --git a/main.py b/main.py index 0e87b3b..9611d5a 100644 --- a/main.py +++ b/main.py @@ -54,20 +54,13 @@ import ssl from src.app import MyApp -from src.db import connection from src.lib.helpers import get_root_path -if __name__ == '__main__': - if 'ANDROID_STORAGE' in os.environ: - try: - from android import loadingscreen - - loadingscreen.hide_loading_screen() - except Exception as e: - print("Loading screen is not removed", e) +def main(): ssl._create_default_https_context = ssl._create_unverified_context MyApp().run() + MyApp().db_connection.close() # Delete Files on exit. root_path = get_root_path() @@ -76,4 +69,6 @@ os.remove(f) os.remove(root_path + 'output.apkg') - connection.close() + +if __name__ == '__main__': + main() diff --git a/src/__pycache__/app.cpython-38.pyc b/src/__pycache__/app.cpython-38.pyc index b18b517..4595088 100644 Binary files a/src/__pycache__/app.cpython-38.pyc and b/src/__pycache__/app.cpython-38.pyc differ diff --git a/src/app.kv b/src/app.kv index c3f3ee2..d02970b 100644 --- a/src/app.kv +++ b/src/app.kv @@ -48,22 +48,14 @@ ScreenManager: font_style: 'Button' pos_hint: {"center_x": 0.5, 'center_y': 0.89} on_release: root.open_dictionary_dropdown() - MDTextField: - id: word_input - text: get_text("sample_url") + ClickableTextFieldRound: + id: url_field padding: "10dp" - multiline: False + text: get_text("sample_url") hint_text: get_text("paste_here") helper_text: f"ex: {get_text('sample_url')}" - helper_text_mode: "on_focus" - # icon_right: get_text('paste_icon') - # icon_right_color: app.theme_cls.primary_color size_hint: (.75, .1) - pos_hint: {'center_x': 0.45, 'center_y': 0.72} - MDIconButton: - icon: get_text('paste_icon') - pos_hint: {"center_x": .9, "center_y": .72} - on_release: word_input.text = Clipboard.paste() + pos_hint: {"center_x": .45, "center_y": .72} # MDLabel: # text: get_text("chose_accent") # theme_text_color: 'Primary' @@ -92,11 +84,11 @@ ScreenManager: on_focus: root.tags_input_focus_mode(self.focus) BoxLayout: orientation: 'horizontal' - size_hint: (.9, .3) - pos_hint: {"center_x": .54, "center_y": .42} + size_hint: (.88, .3) + pos_hint: {"center_x": .5, "center_y": .42} Check: id: check_uk - width: 36 + width: 48 pos_hint: {"center_x": .12, "center_y": .4} on_active: root.checkbox_click(self, self.active, "co.uk") MDLabel: @@ -107,7 +99,7 @@ ScreenManager: on_ref_press: root.check_it_down(*args) Check: id: check_us - width: 36 + width: 48 pos_hint: {"center_x": .25, "center_y": .4} active: True on_active: root.checkbox_click(self, self.active, "com") @@ -182,4 +174,24 @@ ScreenManager: IconLeftWidget: - icon: root.icon \ No newline at end of file + icon: root.icon + +: + size_hint_y: None + height: word_input.height + + MDTextField: + id: word_input + multiline: False + hint_text: root.hint_text + text: root.text + helper_text: root.helper_text + helper_text_mode: "on_focus" + icon_left: get_text("tag_icon") + + MDIconButton: + icon: get_text('paste_icon') + pos_hint: {"center_y": .5} + pos: word_input.width - self.width + dp(8), 0 + theme_text_color: "Hint" + on_release: word_input.text = Clipboard.paste() diff --git a/src/app.py b/src/app.py index 677f6c9..34124f1 100644 --- a/src/app.py +++ b/src/app.py @@ -23,14 +23,14 @@ from bs4 import BeautifulSoup from kivy.animation import Animation -from kivy import require, platform +from kivy import require from kivy.metrics import dp from kivy.properties import StringProperty from kivymd.app import MDApp from kivymd.toast import toast from kivymd.uix.list import OneLineListItem, TwoLineListItem, OneLineIconListItem from kivymd.uix.dialog import MDDialog -from kivymd.uix.boxlayout import MDBoxLayout +from kivymd.uix.relativelayout import MDRelativeLayout from kivymd.uix.button import MDFlatButton, MDRaisedButton, MDRectangleFlatButton, MDIconButton, MDFloatingActionButton from kivymd.uix.menu import MDDropdownMenu from kivy.lang.builder import Builder @@ -51,8 +51,9 @@ from kivy.uix.button import Button from kivy.utils import get_color_from_hex +from src.db import create_connection, create_table, create_tag, select_all_tags, select_tags_which_contains from src.dict_scraper.spiders import cambridge -from src.lib.helpers import get_root_path +from src.lib.helpers import get_root_path, is_platform, check_android_permissions, request_android_permissions from src.lib.json_to_apkg import JsonToApkg from src.lib.strings import get_text @@ -60,7 +61,7 @@ # os.environ["KIVY_NO_CONSOLELOG"] = "1" # require('2.1.0') -CONTAINER = {'current_url': '', 'requests': [], 'tags': [], 'tags_input_text': '', +CONTAINER = {'current_url': '', 'requests': [], 'tags': [], 'm_checkboxes': [], 'm_checkboxes_selected': 0, 'm_checkboxes_total': 0} DICTIONARIES = { get_text("cambridge"): "dictionary.cambridge.org/dictionary/english/", @@ -70,7 +71,7 @@ get_text("vocabulary_com"): "vocabulary.com/dictionary/", } HEADERS = { - 'User-Agent': generate_user_agent(device_type='smartphone' if 'ANDROID_STORAGE' in os.environ else 'desktop'), + 'User-Agent': generate_user_agent(device_type='smartphone' if is_platform('android') else 'desktop'), 'Referer': 'https://www.google.com' } @@ -148,6 +149,12 @@ def clear_request(word_url=None): # Window.size = (500, 400) +class ClickableTextFieldRound(MDRelativeLayout): + text = StringProperty() + hint_text = StringProperty() + helper_text = StringProperty() + + class MeaningsPanelContent(MDGridLayout): def __init__(self, *args, **kwargs): super().__init__() @@ -355,7 +362,7 @@ def open_dictionary_dropdown(self): self.dictionary_menu = MDDropdownMenu( caller=self.ids.dict_dropdown, items=menu_items, - position='bottom', + position='center', width_mult=4, max_height=dp(248), ) @@ -460,7 +467,6 @@ def checkbox_click(self, instance, value, tld): self.tld = tld def generate_flashcards(self, btn): - from src.db import connection, cursor selected_checkboxes = [] for checkbox in CONTAINER['m_checkboxes']: if type(checkbox) is list: @@ -483,17 +489,12 @@ def generate_flashcards(self, btn): # print(CONTAINER['tags']) for tag in CONTAINER['tags']: try: - cursor.execute(""" - INSERT OR REPLACE INTO tags (tag) VALUES (?) - """, (tag,)) + create_tag(MDApp.get_running_app().db_connection, (tag,)) except Exception as e: print(e) print(traceback.format_exc()) # print("inserted") - # print('committing') - connection.commit() - # print('committed') - jta.generate_apkg(notes) + apkg_filename = jta.generate_apkg(notes) MDApp.get_running_app().soft_restart() self.dialog_popup( @@ -527,10 +528,8 @@ def change_checkbox_state(self, checkbox): checkbox.active = True def show_meanings(self): - word_url = self.ids.word_input.text.split('#')[0].split('?')[0] + word_url = self.ids.url_field.text.split('#')[0].split('?')[0] CONTAINER['tags'] = self.ids.tags_input.text.split() - if not CONTAINER['tags']: - CONTAINER['tags'] = [''] dict_name = None if not validators.url(word_url): self.toast(get_text("url_not_found")) @@ -557,6 +556,11 @@ def show_meanings(self): dict_name = name break if dict_name: + if not check_android_permissions(): + self.dialog.dismiss() + MDApp.get_running_app().soft_restart() + return False + MDApp.get_running_app().create_tables() # d = runner.crawl( # CambridgeSpider, # url=word_url, @@ -717,6 +721,7 @@ class MyApp(MDApp): def __init__(self, **kwargs): super().__init__(**kwargs) self.screen = Builder.load_file("src/app.kv") + self.db_connection = None def build(self): Window.bind(on_keyboard=self._on_keyboard_handler) @@ -731,25 +736,9 @@ def build(self): # return MyLayout() def on_start(self): + print('Starting...') print("Size:", Window.size) - if platform == 'android': - try: - from android.storage import app_storage_path - settings_path = app_storage_path() - print("settings_path", settings_path) - - from android.storage import primary_external_storage_path - primary_ext_storage = primary_external_storage_path() - print("primary_ext_storage", primary_ext_storage) - - from android.storage import secondary_external_storage_path - secondary_ext_storage = secondary_external_storage_path() - print("secondary_ext_storage", secondary_ext_storage) - except Exception as e: - print("Error printing paths", e) - - from android.permissions import request_permissions, Permission - request_permissions([Permission.READ_EXTERNAL_STORAGE, Permission.WRITE_EXTERNAL_STORAGE]) + request_android_permissions() def restart(self): self.root.clear_widgets() @@ -774,12 +763,14 @@ def change_screen(self): self.root.current = 'menu_screen' def soft_restart(self): + print('Restarting..') global CONTAINER CONTAINER['current_url'] = '' CONTAINER['tags'] = [] CONTAINER['m_checkboxes'] = [] CONTAINER['m_checkboxes_selected'] = 0 CONTAINER['m_checkboxes_total'] = 0 + request_android_permissions() meanings_screen = self.root.get_screen("meanings_screen") meanings_screen.ids.toolbar.title = get_text("app_title") @@ -789,10 +780,37 @@ def soft_restart(self): # meanings_screen.ids.toolbar.right_action_items = \ # [[get_text("select_all_icon"), lambda x: self.get_running_app().meanings_screen_instance.select_all()]] meanings_screen.ids.meanings_selection_list.clear_widgets() - # self.root.transition.direction = 'right' - # self.root.transition.duration = 0.5 # 0.5 second - # self.root.current = 'menu_screen' - self.change_screen() + self.root.transition.direction = 'right' + self.root.transition.duration = 0.5 # 0.5 second + self.root.current = 'menu_screen' + # self.change_screen() + + def create_tables(self): + if self.db_connection is not None: + return True + db_path = f'{get_root_path()}data.db' + self.db_connection = create_connection(db_path) + # https://stackoverflow.com/a/44951682 + sql_create_tags_table = """ + CREATE TABLE IF NOT EXISTS tags( + tag TEXT NOT NULL COLLATE NOCASE, + PRIMARY KEY(tag) + ) + """ + create_table(self.db_connection, sql_create_tags_table) + + # https://www.designcise.com/web/tutorial/how-to-do-case-insensitive-comparisons-in-sqlite + # Without a COLLATE INDEX our queries will do a full table scan + # EXPLAIN QUERY PLAN SELECT * FROM tags WHERE tag = 'some-tag; + # output: SCAN TABLE tags + # When COLLATE NOCASE index is present, the query does not scan all rows. + # EXPLAIN QUERY PLAN SELECT * FROM tags WHERE tag = 'some-tag'; + # output: SEARCH TABLE tags USING INDEX idx_nocase_tags (tag=?) + sql_create_tags_index = """ + CREATE INDEX IF NOT EXISTS idx_nocase_tags ON tags (tag COLLATE NOCASE) + """ + create_table(self.db_connection, sql_create_tags_index) + return True # def callback(self, button): # Snackbar(text="Hello World").open() @@ -838,7 +856,10 @@ def _on_keyboard_handler(self, instance, key, *args): # CONTAINER['tags_input_text'] += chr(key) def open_tags_dropdown(self, key=None): - from src.db import cursor + if not check_android_permissions(): + self.soft_restart() + return False + self.create_tables() menu_screen = self.root.get_screen("menu_screen") menu_screen_instance = self.get_running_app().menu_screen_instance menu_items = [] @@ -849,10 +870,10 @@ def open_tags_dropdown(self, key=None): typed_tag = '' # print("typed_tag:", typed_tag) if not typed_tag: - cursor.execute(f"SELECT tag FROM tags ORDER BY tag DESC LIMIT 5") + rows = select_all_tags(self.db_connection) else: - cursor.execute(f"SELECT tag FROM tags WHERE tag LIKE '%{typed_tag}%' ORDER BY tag DESC LIMIT 5") - for row in cursor.fetchall(): + rows = select_tags_which_contains(self.db_connection, typed_tag) + for row in rows: some_dict = { "viewclass": "IconListItem", "icon": get_text("tag_icon"), @@ -864,7 +885,8 @@ def open_tags_dropdown(self, key=None): menu_screen_instance.tags_menu = MDDropdownMenu( caller=menu_screen.ids.tags_input, items=menu_items, - position='bottom', + position='auto', + ver_growth='up' if is_platform('android') else 'down', width_mult=4, ) menu_screen_instance.tags_menu.open() diff --git a/src/db.py b/src/db.py index af51c9a..fd29696 100644 --- a/src/db.py +++ b/src/db.py @@ -1,28 +1,80 @@ import sqlite3 -from src.lib.helpers import get_root_path - -connection = sqlite3.connect(f'{get_root_path()}data.db') -cursor = connection.cursor() - -# https://stackoverflow.com/a/44951682 -cursor.execute(""" -CREATE TABLE IF NOT EXISTS tags( -tag TEXT NOT NULL COLLATE NOCASE, -PRIMARY KEY(tag) -) -""") -# dt datetime default current_timestamp - -# Without a COLLATE INDEX our queries will do a full table scan -# EXPLAIN QUERY PLAN SELECT * FROM tags WHERE tag = 'some-tag; -# output: SCAN TABLE tags - -# When COLLATE NOCASE index is present, the query does not scan all rows. -# EXPLAIN QUERY PLAN SELECT * FROM tags WHERE tag = 'some-tag'; -# output: SEARCH TABLE tags USING INDEX idx_nocase_tags (tag=?) - -# Refer: https://www.designcise.com/web/tutorial/how-to-do-case-insensitive-comparisons-in-sqlite -cursor.execute(""" -CREATE INDEX IF NOT EXISTS idx_nocase_tags ON tags (tag COLLATE NOCASE) -""") + +def create_connection(db_path): + """ create a database connection to the SQLite database + :param db_path: Path to db + :return: Connection object or None + """ + conn = None + try: + conn = sqlite3.connect(db_path) + return conn + except sqlite3.Error as e: + print(e) + + return conn + + +def create_table(conn, create_table_sql): + """ create a table from the create_table_sql statement + :param conn: Connection object + :param create_table_sql: a CREATE TABLE statement + :return: + """ + try: + c = conn.cursor() + c.execute(create_table_sql) + except sqlite3.Error as e: + print(e) + + +def create_tag(conn, tag): + """ + Create a new tag + :param conn: + :param tag: + :return: + """ + + sql = ''' INSERT OR REPLACE INTO tags (tag) VALUES (?) ''' + cur = conn.cursor() + cur.execute(sql, tag) + conn.commit() + + return cur.lastrowid + + +def select_all_tags(conn): + """ + Query all rows in the tags table + :param conn: the Connection object + :return: + """ + cur = conn.cursor() + cur.execute("SELECT tag FROM tags ORDER BY tag DESC LIMIT 5") + + rows = cur.fetchall() + + # for row in rows: + # print(row) + + return rows + + +def select_tags_which_contains(conn, tag): + """ + Query tags by tag + :param conn: the Connection object + :param tag: + :return: + """ + cur = conn.cursor() + cur.execute(f"SELECT tag FROM tags WHERE tag LIKE '%{tag}%' ORDER BY tag DESC LIMIT 5") + + rows = cur.fetchall() + + # for row in rows: + # print(row) + + return rows diff --git a/src/dict_scraper/spiders/cambridge.py b/src/dict_scraper/spiders/cambridge.py index 3b82d07..fea1021 100644 --- a/src/dict_scraper/spiders/cambridge.py +++ b/src/dict_scraper/spiders/cambridge.py @@ -1,63 +1,15 @@ import os import re -import requests from gtts import gTTS -from bs4.element import ResultSet, Tag -from src.lib.helpers import get_root_path +from src.lib.helpers import extract_text, get_root_path, get_tree, get_valid_filename allowed_domains = ['dictionary.cambridge.org'] start_urls = ['https://dictionary.cambridge.org/'] -class SuspiciousOperation(Exception): - """The user did something suspicious""" - - -def get_valid_filename(name): - """ - Return the given string converted to a string that can be used for a clean - filename. Remove leading and trailing spaces; convert other spaces to - underscores; and remove anything that is not an alphanumeric, dash, - underscore, or dot. - >>> get_valid_filename("john's portrait in 2004.jpg") - 'johns_portrait_in_2004.jpg' - """ - s = str(name).strip().replace(" ", "-") - s = re.sub(r"(?u)[^-\w.]", "", s) - if s in {"", ".", ".."}: - raise SuspiciousOperation("Could not derive file name from '%s'" % name) - return s - - -def get_tree(branch, seen, *args, **kwargs): - out = [] - for d in branch.find_all("div", class_="cid"): - if d not in seen: - seen.add(d) - out.append(d["id"]) - t = get_tree(d, seen) - if t: - out.append(t) - return out - - -def extract_text(data, join_char=''): - strings = [] - if type(data) is ResultSet: - if data: - for element in data: - for string in element.strings: - strings.append(repr(string)[1:-1]) - elif type(data) is Tag: - if data: - for string in data.strings: - strings.append(repr(string)[1:-1]) - return join_char.join(strings) - - class MeaningsSpider: def __init__(self, soup, *args, **kwargs): self.soup = soup diff --git a/src/lib/__pycache__/json_to_apkg.cpython-38.pyc b/src/lib/__pycache__/json_to_apkg.cpython-38.pyc index 3e67f96..ba21e80 100644 Binary files a/src/lib/__pycache__/json_to_apkg.cpython-38.pyc and b/src/lib/__pycache__/json_to_apkg.cpython-38.pyc differ diff --git a/src/lib/helpers.py b/src/lib/helpers.py index 76ae602..62c28c7 100644 --- a/src/lib/helpers.py +++ b/src/lib/helpers.py @@ -1,11 +1,18 @@ import os +import re + +from bs4.element import ResultSet, Tag +from kivy import platform + + +def is_platform(os_name) -> bool: + return os_name == platform def get_root_path() -> str: - if 'ANDROID_STORAGE' in os.environ: + if is_platform('android'): # if 'ANDROID_STORAGE' in os.environ: from android.storage import app_storage_path # path = f'{app_storage_path()}/' - package_name = app_storage_path().split('/')[-2] path = f'/storage/emulated/0/Android/data/{package_name}/files/' else: # platform == 'win' @@ -13,3 +20,81 @@ def get_root_path() -> str: if not os.path.exists(path + 'media/'): os.makedirs(path + 'media/') return path + + +class SuspiciousOperation(Exception): + """The user did something suspicious""" + + +def get_valid_filename(name): + """ + Return the given string converted to a string that can be used for a clean + filename. Remove leading and trailing spaces; convert other spaces to + underscores; and remove anything that is not an alphanumeric, dash, + underscore, or dot. + >>> get_valid_filename("john's portrait in 2004.jpg") + 'johns_portrait_in_2004.jpg' + """ + s = str(name).strip().replace(" ", "-") + s = re.sub(r"(?u)[^-\w.]", "", s) + if s in {"", ".", ".."}: + raise SuspiciousOperation("Could not derive file name from '%s'" % name) + return s + + +def get_tree(branch, seen, *args, **kwargs): + out = [] + for d in branch.find_all("div", class_="cid"): + if d not in seen: + seen.add(d) + out.append(d["id"]) + t = get_tree(d, seen) + if t: + out.append(t) + return out + + +def extract_text(data, join_char=''): + strings = [] + if type(data) is ResultSet: + if data: + for element in data: + for string in element.strings: + strings.append(repr(string)[1:-1]) + elif type(data) is Tag: + if data: + for string in data.strings: + strings.append(repr(string)[1:-1]) + return join_char.join(strings) + + +def check_android_permissions() -> bool: + if is_platform('android'): + from android.permissions import check_permission + return check_permission('android.permission.WRITE_EXTERNAL_STORAGE') + else: + return True + + +def request_android_permissions(): + if is_platform('android'): + from android.permissions import request_permissions, Permission + request_permissions([Permission.READ_EXTERNAL_STORAGE, Permission.WRITE_EXTERNAL_STORAGE]) + + +def print_android_storage_path(): + if is_platform('android'): + try: + from android.storage import app_storage_path + settings_path = app_storage_path() + print("settings_path", settings_path) + + from android.storage import primary_external_storage_path + primary_ext_storage = primary_external_storage_path() + print("primary_ext_storage", primary_ext_storage) + + from android.storage import secondary_external_storage_path + secondary_ext_storage = secondary_external_storage_path() + print("secondary_ext_storage", secondary_ext_storage) + except Exception as e: + print("Error printing paths", e) diff --git a/src/lib/json_to_apkg.py b/src/lib/json_to_apkg.py index 3d398ac..1ff8afe 100644 --- a/src/lib/json_to_apkg.py +++ b/src/lib/json_to_apkg.py @@ -88,7 +88,7 @@ class JsonToApkg: def __init__(self): pass - def generate_apkg(self, notes): + def generate_apkg(self, notes) -> str: # print('Before my_deck') my_deck = genanki.Deck( 1646145285163, # todo: change id and name