From 3f2422500112a0870c945d519b67d7a6a40e4f59 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 4 Sep 2024 19:34:48 +0200 Subject: [PATCH] Several fixes and linting following tests with wopi test suite --- src/core/cs3iface.py | 47 ++++++++++++++++++++++++--------------- src/core/localiface.py | 2 +- src/core/wopi.py | 1 - test/test_storageiface.py | 4 ++-- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 3964384b..af506801 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -7,6 +7,7 @@ """ import os +from configparser import ConfigParser import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr import cs3.gateway.v1beta1.gateway_api_pb2 as cs3gw @@ -14,6 +15,7 @@ from cs3client.cs3client import CS3Client from cs3client.cs3resource import Resource from cs3client.auth import Auth +import cs3client.exceptions import core.commoniface as common # key used if the `lockasattr` option is true, in order to store the lock payload without ensuring any lock semantic @@ -27,18 +29,19 @@ def init(inconfig, inlog): """Init module-level variables""" - global config # pylint: disable=global-statement - config = {} global log # pylint: disable=global-statement log = inlog + global config # pylint: disable=global-statement + config = ConfigParser() + config["cs3client"] = {} config["cs3client"]["host"] = inconfig.get("cs3", "revagateway") - config["cs3client"]["chunk_size"] = inconfig.getint("io", "chunksize") - config["cs3client"]["ssl_verify"] = inconfig.getboolean("cs3", "sslverify", fallback=True) - config["cs3client"]["lock_expiration"] = inconfig.getint("general", "wopilockexpiration") - config["cs3client"]["lock_as_attr"] = inconfig.getboolean("cs3", "lockasattr", fallback=False) - config["cs3client"]["lock_not_impl"] = False - config["cs3client"]["grpc_timeout"] = inconfig.getint("cs3", "grpctimeout", fallback=10) - config["cs3client"]["http_timeout"] = inconfig.getint("cs3", "httptimeout", fallback=10) + config["cs3client"]["chunk_size"] = inconfig.get("io", "chunksize") + config["cs3client"]["ssl_verify"] = inconfig.get("cs3", "sslverify", fallback='True') + config["cs3client"]["lock_expiration"] = inconfig.get("general", "wopilockexpiration") + config["cs3client"]["lock_as_attr"] = inconfig.get("cs3", "lockasattr", fallback='False') + config["cs3client"]["lock_not_impl"] = 'False' + config["cs3client"]["grpc_timeout"] = inconfig.get("cs3", "grpctimeout", fallback='10') + config["cs3client"]["http_timeout"] = inconfig.get("cs3", "httptimeout", fallback='10') global client # pylint: disable=global-statement client = CS3Client(config, "cs3client", log) @@ -46,7 +49,7 @@ def init(inconfig, inlog): def healthcheck(): """Probes the storage and returns a status message. For cs3 storage, we execute a call to ListAuthProviders""" try: - client.auth.ListAuthProviders() + Auth(client).ListAuthProviders() log.debug('msg="Executed ListAuthProviders as health check" endpoint="%s"' % (ctx["revagateway"])) return "OK" except ConnectionError as e: @@ -74,9 +77,8 @@ def authenticate_for_test(userid, userpwd): def stat(endpoint, fileref, userid): - resource = Resource(endpoint, fileref) - client.auth.set_token(userid) - statInfo = client.file.stat(resource) + resource = Resource.from_file_ref_and_endpoint(fileref, endpoint) + statInfo = client.file.stat(Auth.check_token(userid), resource) if statInfo.type == cs3spr.RESOURCE_TYPE_CONTAINER: log.info( 'msg="Invoked stat" endpoint="%s" fileref="%s" trace="%s" result="ISDIR"' @@ -118,14 +120,18 @@ def statx(endpoint, fileref, userid): def setxattr(endpoint, filepath, userid, key, value, lockmd): """Set the extended attribute to using the given userid as access token""" - _, lock_id = lockmd + lock_id = None + if lockmd: + _, lock_id = lockmd resource = Resource.from_file_ref_and_endpoint(filepath, endpoint) client.file.set_xattr(Auth.check_token(userid), resource, key, value, lock_id) def rmxattr(endpoint, filepath, userid, key, lockmd): """Remove the extended attribute using the given userid as access token""" - _, lock_id = lockmd + lock_id = None + if lockmd: + _, lock_id = lockmd resource = Resource.from_file_ref_and_endpoint(filepath, endpoint) client.file.remove_xattr(Auth.check_token(userid), resource, key, lock_id) @@ -149,7 +155,9 @@ def writefile(endpoint, filepath, userid, content, size, lockmd): def renamefile(endpoint, filepath, newfilepath, userid, lockmd): """Rename a file from origfilepath to newfilepath using the given userid as access token.""" - _, lock_id = lockmd + lock_id = None + if lockmd: + _, lock_id = lockmd resource = Resource.from_file_ref_and_endpoint(filepath, endpoint) new_resource = Resource.from_file_ref_and_endpoint(newfilepath, endpoint) client.file.rename_file(Auth.check_token(userid), resource, new_resource, lock_id) @@ -158,7 +166,7 @@ def renamefile(endpoint, filepath, newfilepath, userid, lockmd): def removefile(endpoint, filepath, userid): """Remove a file using the given userid as access token. The force argument is ignored for now for CS3 storage.""" - resource = Resource.from_file_ref_and_endpoint(endpoint, filepath) + resource = Resource.from_file_ref_and_endpoint(filepath, endpoint) client.file.remove_file(Auth.check_token(userid), resource) @@ -179,7 +187,10 @@ def refreshlock(endpoint, filepath, userid, appname, value, oldvalue=None): def getlock(endpoint, filepath, userid): """Get the lock metadata for the given filepath""" resource = Resource.from_file_ref_and_endpoint(filepath, endpoint) - return client.file.get_lock(Auth.check_token(userid), resource) + try: + return client.file.get_lock(Auth.check_token(userid), resource) + except cs3client.exceptions.NotFoundException: + return None def unlock(endpoint, filepath, userid, appname, value): diff --git a/src/core/localiface.py b/src/core/localiface.py index 86b7f19e..ffc8a13b 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -104,7 +104,7 @@ def stat(_endpoint, filepath, _userid): try: xattrs = { k.strip('user.'): os.getxattr(_getfilepath(filepath), k).decode() - for k in os.listxattr(_getfilepath(filepath)) + for k in os.listxattr(_getfilepath(filepath)) } except OSError as e: log.info('msg="Failed to invoke listxattr/getxattr" inode="%d" filepath="%s" exception="%s"' % diff --git a/src/core/wopi.py b/src/core/wopi.py index 371bbecd..c87aba09 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -15,7 +15,6 @@ from urllib.parse import unquote_plus as url_unquote from urllib.parse import quote_plus as url_quote from urllib.parse import urlparse -from more_itertools import peekable import flask import core.wopiutils as utils import core.commoniface as common diff --git a/test/test_storageiface.py b/test/test_storageiface.py index 3bb17aae..97df0062 100644 --- a/test/test_storageiface.py +++ b/test/test_storageiface.py @@ -178,7 +178,7 @@ def test_write_remove_specialchars(self): def test_write_islock(self): '''Test double write with the islock flag''' if self.storagetype == 'cs3': - self.log.warn('Skipping test_write_islock for storagetype cs3') + self.log.warning('Skipping test_write_islock for storagetype cs3') return try: self.storage.removefile(self.endpoint, self.homepath + '/testoverwrite', self.userid) @@ -195,7 +195,7 @@ def test_write_islock(self): def test_write_race(self): '''Test multithreaded double write with the islock flag. Might fail as it relies on tight timing''' if self.storagetype == 'cs3': - self.log.warn('Skipping test_write_race for storagetype cs3') + self.log.warning('Skipping test_write_race for storagetype cs3') return try: self.storage.removefile(self.endpoint, self.homepath + '/testwriterace', self.userid)