From 5ae60b602e48d0711757a06df834fd59974a285b Mon Sep 17 00:00:00 2001 From: newt-sc <47229722+newt-sc@users.noreply.github.com> Date: Sun, 25 Aug 2024 01:05:31 +0300 Subject: [PATCH] release: v1.60.0 --- CHANGELOG.md | 3 ++ a4kStreaming/core.py | 6 +++ a4kStreaming/explorer.py | 2 +- a4kStreaming/lib/database.py | 84 +++++++++++++++++------------------ a4kStreaming/lib/kodi_mock.py | 2 +- a4kStreaming/lib/request.py | 32 ++++++++++--- a4kStreaming/lib/utils.py | 7 ++- addon.xml | 5 ++- packages/addons.xml | 5 ++- packages/addons.xml.crc | 2 +- 10 files changed, 93 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93af3ba..f2cc606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +* [v1.60.0](https://github.com/newt-sc/a4kStreaming/releases/tag/plugin.video.a4kstreaming%2Fplugin.video.a4kstreaming-1.60.0): + * Add generic requests caching to speedup browsing + * [v1.59.0](https://github.com/newt-sc/a4kStreaming/releases/tag/plugin.video.a4kstreaming%2Fplugin.video.a4kstreaming-1.59.0): * Remove deprecated usages ListItem methods diff --git a/a4kStreaming/core.py b/a4kStreaming/core.py index 34765a2..5dd61e2 100644 --- a/a4kStreaming/core.py +++ b/a4kStreaming/core.py @@ -21,9 +21,15 @@ goto ) +from .lib.database import db + core = sys.modules[__name__] utils.core = core +removed_bytes = db.cleanup() +if removed_bytes: + logger.notice('Removed %s bytes from cache' % removed_bytes) + api_mode_enabled = True url = '' handle = None diff --git a/a4kStreaming/explorer.py b/a4kStreaming/explorer.py index dae2d65..fcddcd1 100644 --- a/a4kStreaming/explorer.py +++ b/a4kStreaming/explorer.py @@ -2815,7 +2815,7 @@ def delete_magnet(): item.setArt({ 'poster': provider_params.title.poster }) info_tag = core.kodi.set_info_tag(item, video_meta) - info_tag.set_stream_details('video', { 'codec': result['videocodec'], 'duration': result['ref'].duration }) + info_tag.add_stream_info('video', { 'codec': result['videocodec'], 'duration': result['ref'].duration }) core.utils.end_action(core, True, item) return link diff --git a/a4kStreaming/lib/database.py b/a4kStreaming/lib/database.py index 8ba4d24..f302bf1 100644 --- a/a4kStreaming/lib/database.py +++ b/a4kStreaming/lib/database.py @@ -1,61 +1,61 @@ # -*- coding: utf-8 -*- -import hashlib -from . import kodi -from .utils import open_file_wrapper, os, re, json, time, provider_temp_dir +from .utils import open_file_wrapper, os, time, provider_temp_dir def __cache_key_path(key): - path = ''.join([x if x.isalnum() else '_' for x in key]) + '.json' + path = ''.join([x if x.isalnum() else '_' for x in key]) return os.path.join(provider_temp_dir, path) def __cache_save(key, data): path = __cache_key_path(key) - with open_file_wrapper(path, mode='w')() as f: - f.write(json.dumps(data, indent=4)) + with open_file_wrapper(path, mode='wb')() as f: + f.write(data) def __cache_get(key): path = __cache_key_path(key) if not os.path.exists(path): return {} try: - with open_file_wrapper(path)() as f: - return json.load(f) + with open_file_wrapper(path, mode='rb')() as f: + return f.read() except: return {} -def __generate_md5(*args): - md5_hash = hashlib.md5() +def __cache_check(key): + path = __cache_key_path(key) + return os.path.exists(path) + +def __cache_cleanup(): try: - [md5_hash.update(str(arg)) for arg in args] + # while temp dir bigger than 500MiB, remove files sorted by age (oldest first) + max_size = 500 * 1024 * 1024 + files = [] + size = 0 + + for file in os.listdir(provider_temp_dir): + path = os.path.join(provider_temp_dir, file) + if os.path.isfile(path): + files.append((path, os.path.getmtime(path))) + size += os.path.getsize(path) + + if size < max_size: + return + + original_size = size + + files.sort(key=lambda x: x[1]) + for file, _ in files: + if size < max_size: + break + size -= os.path.getsize(file) + os.remove(file) + + return original_size - size except: - [md5_hash.update(str(arg).encode('utf-8')) for arg in args] - return str(md5_hash.hexdigest()) - -def __get_function_name(function_instance): - return re.sub(r'.+?\s*method\s*|.+function\s*|\sat\s*?.+|\s*?of\s*?.+', '', repr(function_instance)) - -def __hash_function(function_instance, *args): - return __get_function_name(function_instance) + __generate_md5(args) - -def __get_or_add(key, value, fn, duration, *args, **kwargs): - key = __hash_function(fn, *args) if not key else key - if not value: - data = __cache_get(key) - if data: - if not duration or time.time() - data['t'] < (duration * 60): - return data['v'] - - if not value and not fn: - return None - - value = fn(*args, **kwargs) if not value else value - data = { 't': time.time(), 'v': value } - __cache_save(key, data) - return value - -database = lambda: None -def db_get(fn, duration, *args, **kwargs): - return __get_or_add(None, None, fn, duration, *args, **kwargs) -database.get = db_get -database.cache_get = lambda key: __get_or_add(key, None, None, None) -database.cache_insert = lambda key, value: __get_or_add(key, value, None, None) + pass + +db = lambda: None +db.set = lambda key, value: __cache_save(key, value) +db.get = lambda key: __cache_get(key) +db.check = lambda key: __cache_check(key) +db.cleanup = lambda: __cache_cleanup() diff --git a/a4kStreaming/lib/kodi_mock.py b/a4kStreaming/lib/kodi_mock.py index f9513f9..ac1f3ee 100644 --- a/a4kStreaming/lib/kodi_mock.py +++ b/a4kStreaming/lib/kodi_mock.py @@ -101,7 +101,7 @@ def __get_video_info_tag(*args, **kwargs): return __video_info_tag __list_info_tag = lambda: None __list_info_tag.set_cast = lambda _: None -__list_info_tag.set_stream_details = lambda _, __: None +__list_info_tag.add_stream_info = lambda _, __: None def set_info_tag(*args, **kwargs): return __list_info_tag def __create_listitem(*args, **kwargs): return __listitem diff --git a/a4kStreaming/lib/request.py b/a4kStreaming/lib/request.py index d85d7cb..4efd5c5 100644 --- a/a4kStreaming/lib/request.py +++ b/a4kStreaming/lib/request.py @@ -20,11 +20,10 @@ def __retry_on_503(core, request, response, retry=True): request['validate'] = lambda response: __retry_on_503(core, request, response, retry=False) return request -def execute(core, request, session=None): - if not session: - session = requests.Session() - retries = Retry(total=3, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) - session.mount("https://", HTTPAdapter(max_retries=retries, pool_maxsize=100)) +def execute(core, request, cache=True): + session = requests.Session() + retries = Retry(total=3, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) + session.mount("https://", HTTPAdapter(max_retries=retries, pool_maxsize=100)) request.setdefault('timeout', 60) headers = request.setdefault('headers', {}) @@ -41,7 +40,28 @@ def execute(core, request, session=None): logger.debug('%s ^ - %s' % (request['method'], request['url'])) try: - response = session.request(verify=False, **request) + # hash request object for checking a cache file + request_hash = core.utils.hash({ 'url': request['url'], 'method': request['method'], 'data': request.get('data', '') }) + if cache and core.db.check(request_hash): + core.logger.debug('Cache hit: %s' % request_hash) + response = lambda: None + response.text = '' + response.content = core.db.get(request_hash) + response.status_code = 200 + + def refresh(): + response = session.request(verify=False, **request) + if response.status_code == 200: + core.logger.debug('Cache refresh: %s' % request_hash) + core.db.set(request_hash, response.content) + + core.threading.Thread(target=refresh).start() + else: + if cache: + core.logger.debug('Cache miss: %s' % request_hash) + response = session.request(verify=False, **request) + if response.status_code == 200 and cache: + core.db.set(request_hash, response.content) exc = '' except: exc = core.traceback.format_exc() diff --git a/a4kStreaming/lib/utils.py b/a4kStreaming/lib/utils.py index 9904238..63f618a 100644 --- a/a4kStreaming/lib/utils.py +++ b/a4kStreaming/lib/utils.py @@ -75,12 +75,15 @@ def versiontuple(v): version_parts = [(int(v_part) if v_part.isdigit() else 0) for v_part in version_parts] return tuple(map(int, version_parts)) +def hash(data): + return hashlib.sha1(json.dumps(data, sort_keys=True).encode('utf-8')).hexdigest() + def chunk(it, size): it = iter(it) return iter(lambda: tuple(islice(it, size)), ()) def open_file_wrapper(file, mode='r', encoding='utf-8'): - if py2: + if py2 or mode.endswith('b'): return lambda: open(file, mode) return lambda: open(file, mode, encoding=encoding) @@ -138,7 +141,7 @@ def download_zip(core, zip_url, zip_name): 'timeout': 15 } - with core.request.execute(core, request) as r: + with core.request.execute(core, request, False) as r: with open(filepath, 'wb') as f: shutil.copyfileobj(r.raw, f) diff --git a/addon.xml b/addon.xml index f80125d..e6d41e5 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ @@ -32,6 +32,9 @@ Designed for low-end devices and Estuary skin. screenshot-06.jpg +[v1.60.0]: + * Add generic requests caching to speedup browsing + [v1.59.0]: * Remove deprecated usages ListItem methods diff --git a/packages/addons.xml b/packages/addons.xml index a5fa951..457416e 100644 --- a/packages/addons.xml +++ b/packages/addons.xml @@ -4,7 +4,7 @@ @@ -35,6 +35,9 @@ Designed for low-end devices and Estuary skin. screenshot-06.jpg +[v1.60.0]: + * Add generic requests caching to speedup browsing + [v1.59.0]: * Remove deprecated usages ListItem methods diff --git a/packages/addons.xml.crc b/packages/addons.xml.crc index 82ea773..a281acc 100644 --- a/packages/addons.xml.crc +++ b/packages/addons.xml.crc @@ -1 +1 @@ -04b4bc63dac4924a998aefa868552532a8d67443 \ No newline at end of file +94ed66018b0d476f20cd1368dd61e6c3180dea6b \ No newline at end of file