From 45c3a320419acbd34c5a32725a3779967a148db8 Mon Sep 17 00:00:00 2001 From: k9ert Date: Tue, 7 Mar 2023 18:56:39 +0100 Subject: [PATCH] Bugfix: logging improvements, spectrum socket fix and swan plugin deactivation (#2284) * remove lots of wallet_manager logging * make tiny perf-logging a bit more usefull * bump spectrum * limit logging in Spectrum * deactivating swan plugin * set devstatus to beta --- requirements.in | 2 +- requirements.txt | 60 +++++++++---------- src/cryptoadvance/specter/config.py | 2 + .../specter/managers/wallet_manager.py | 20 +------ .../specter/server_endpoints/controller.py | 4 +- .../specterext/spectrum/config.py | 9 ++- src/cryptoadvance/specterext/swan/service.py | 9 ++- 7 files changed, 50 insertions(+), 56 deletions(-) diff --git a/requirements.in b/requirements.in index 48943c4442..e40aea80b6 100644 --- a/requirements.in +++ b/requirements.in @@ -33,5 +33,5 @@ psycopg2-binary==2.9.5 cryptoadvance-liquidissuer==0.2.4 specterext-exfund==0.1.7 specterext-faucet==0.1.2 -cryptoadvance.spectrum==0.6.1 +cryptoadvance.spectrum==0.6.2 specterext-stacktrack==0.3.0 diff --git a/requirements.txt b/requirements.txt index 46324969f5..95a2f2328c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile --generate-hashes requirements.in # @@ -119,9 +119,9 @@ cryptoadvance-liquidissuer==0.2.4 \ --hash=sha256:5a2c531801854c5a4a46daf184877e22f731cdb42d2cfb840785bda7371ba6fb \ --hash=sha256:9e468f3e35ecc566b3f74a2263677cf26632548abb194521dba15ad37acd1e9b # via -r requirements.in -cryptoadvance.spectrum==0.6.1 \ - --hash=sha256:593f4f0715d6a0d04725eca08853773cb7ae55ed90f18a920343ee4eaf746730 \ - --hash=sha256:c1b9150bb932098214bad48104f3bc3a33e8541b2d8bc809f5355f00250c8279 +cryptoadvance-spectrum==0.6.2 \ + --hash=sha256:0b1cc71eaa8a14551b790cfc8ddb28e04ead07247edca54b60ac9d082b298de8 \ + --hash=sha256:26031c7d3bbbd6963fa281682ae3a6450677b3b56d16c1622a353fc2ab5e2b2e # via -r requirements.in cryptography==3.4.7 \ --hash=sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d \ @@ -153,7 +153,21 @@ embit==0.6.1 \ --hash=sha256:16a84c6668dc9ffc907594457a46f7142cee379646bc009a5a9b77b0d2cb4e12 # via # -r requirements.in - # cryptoadvance.spectrum + # cryptoadvance-spectrum +flask==2.1.1 \ + --hash=sha256:8a4cf32d904cf5621db9f0c9fbcd7efabf3003f22a04e4d0ce790c7137ec5264 \ + --hash=sha256:a8c9bd3e558ec99646d177a9739c41df1ded0629480b4c8d2975412f3c9519c8 + # via + # -r requirements.in + # cryptoadvance-spectrum + # flask-apscheduler + # flask-babel + # flask-cors + # flask-httpauth + # flask-login + # flask-restful + # flask-sqlalchemy + # flask-wtf flask-apscheduler==1.12.4 \ --hash=sha256:681dae34dc6cc9403ce674795e53abd0bff540472129cfd3d3c93e0e1d502da8 # via -r requirements.in @@ -180,22 +194,8 @@ flask-restful==0.3.9 \ flask-sqlalchemy==2.5.1 \ --hash=sha256:2bda44b43e7cacb15d4e05ff3cc1f8bc97936cc464623424102bfc2c35e95912 \ --hash=sha256:f12c3d4cc5cc7fdcc148b9527ea05671718c3ea45d50c7e732cceb33f574b390 - # via cryptoadvance.spectrum -flask==2.1.1 \ - --hash=sha256:8a4cf32d904cf5621db9f0c9fbcd7efabf3003f22a04e4d0ce790c7137ec5264 \ - --hash=sha256:a8c9bd3e558ec99646d177a9739c41df1ded0629480b4c8d2975412f3c9519c8 - # via - # -r requirements.in - # cryptoadvance.spectrum - # flask-apscheduler - # flask-babel - # flask-cors - # flask-httpauth - # flask-login - # flask-restful - # flask-sqlalchemy - # flask-wtf -flask_wtf==0.15.1 \ + # via cryptoadvance-spectrum +flask-wtf==0.15.1 \ --hash=sha256:6ff7af73458f182180906a37a783e290bdc8a3817fe4ad17227563137ca285bf \ --hash=sha256:ff177185f891302dc253437fe63081e7a46a4e99aca61dfe086fb23e54fff2dc # via -r requirements.in @@ -606,7 +606,7 @@ psycopg2-binary==2.9.5 \ --hash=sha256:f95b8aca2703d6a30249f83f4fe6a9abf2e627aa892a5caaab2267d56be7ab69 # via # -r requirements.in - # cryptoadvance.spectrum + # cryptoadvance-spectrum pyaes==1.6.1 \ --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f # via hwi @@ -636,7 +636,7 @@ pysocks==1.7.1 \ --hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0 # via # -r requirements.in - # cryptoadvance.spectrum + # cryptoadvance-spectrum python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 @@ -651,10 +651,6 @@ pytimeparse==1.1.8 \ --hash=sha256:04b7be6cc8bd9f5647a6325444926c3ac34ee6bc7e69da4367ba282f076036bd \ --hash=sha256:e86136477be924d7e670646a98561957e8ca7308d44841e21f5ddea757556a0a # via -r requirements.in -pytz-deprecation-shim==0.1.0.post0 \ - --hash=sha256:8314c9692a636c8eb3bda879b9f119e350e93223ae83e70e80c31675a0fdc1a6 \ - --hash=sha256:af097bae1b616dde5c5744441e2ddc69e74dfdcb0c263129610d85b87445a59d - # via tzlocal pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a @@ -664,12 +660,16 @@ pytz==2022.7.1 \ # flask-babel # flask-restful # pandas +pytz-deprecation-shim==0.1.0.post0 \ + --hash=sha256:8314c9692a636c8eb3bda879b9f119e350e93223ae83e70e80c31675a0fdc1a6 \ + --hash=sha256:af097bae1b616dde5c5744441e2ddc69e74dfdcb0c263129610d85b87445a59d + # via tzlocal requests==2.26.0 \ --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \ --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7 # via # -r requirements.in - # cryptoadvance.spectrum + # cryptoadvance-spectrum semver==2.13.0 \ --hash=sha256:ced8b23dceb22134307c1b8abfa523da14198793d9787ac838e70e29e77458d4 \ --hash=sha256:fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f @@ -743,7 +743,7 @@ sqlalchemy==1.4.46 \ --hash=sha256:e3c1808008124850115a3f7e793a975cfa5c8a26ceeeb9ff9cbb4485cac556df \ --hash=sha256:f8cb80fe8d14307e4124f6fad64dfd87ab749c9d275f82b8b4ec84c84ecebdbe # via - # cryptoadvance.spectrum + # cryptoadvance-spectrum # flask-sqlalchemy stem==1.8.0 \ --hash=sha256:a0b48ea6224e95f22aa34c0bc3415f0eb4667ddeae3dfb5e32a6920c185568c2 diff --git a/src/cryptoadvance/specter/config.py b/src/cryptoadvance/specter/config.py index 70237aaef5..95f6b9c43f 100644 --- a/src/cryptoadvance/specter/config.py +++ b/src/cryptoadvance/specter/config.py @@ -247,6 +247,8 @@ class TestConfig(BaseConfig): BITCOIN_RPC_TIMEOUT = 60 LIQUID_RPC_TIMEOUT = 60 + SERVICES_DEVSTATUS_THRESHOLD = "beta" + class CypressTestConfig(TestConfig): SPECTER_URL_PREFIX = "" diff --git a/src/cryptoadvance/specter/managers/wallet_manager.py b/src/cryptoadvance/specter/managers/wallet_manager.py index 97f5c95538..96e7cc7978 100644 --- a/src/cryptoadvance/specter/managers/wallet_manager.py +++ b/src/cryptoadvance/specter/managers/wallet_manager.py @@ -67,7 +67,7 @@ def update( The _update internal method will resync the internal status with Bitcoin Core use_threading : for the _update method which is heavily communicating with Bitcoin Core """ - logger.debug( + logger.info( f"starting update of wallet_manager (threading: {use_threading} , comment: {comment})" ) if self.is_loading: @@ -97,9 +97,6 @@ def update( if self.working_folder is not None: wallets_files = load_jsons(self.working_folder, key="name") - logger.info( - f"Iterating over {len(wallets_files.values())} wallet files in {self.working_folder}" - ) for wallet in wallets_files: wallet_name = wallets_files[wallet]["name"] wallets_update_list[wallet_name] = wallets_files[wallet] @@ -120,18 +117,15 @@ def update( ): if "pytest" in sys.modules: if self.allow_threading_for_testing: - logger.info("Using threads in updating the wallet manager.") t = FlaskThread( target=self._update, args=(wallets_update_list,), ) t.start() else: - logger.info("Not using threads in updating the wallet manager.") self._update(wallets_update_list) else: if use_threading: - logger.info("Using threads in updating the wallet manager.") t = FlaskThread( target=self._update, args=(wallets_update_list,), @@ -139,7 +133,6 @@ def update( t.start() else: - logger.info("Not using threads in updating the wallet manager.") self._update(wallets_update_list) else: self.is_loading = False @@ -157,9 +150,6 @@ def _update(self, wallets_update_list: Dict): * the unloaded wallets are loaded in Bitcoin Core * and, on the Specter side, the wallet objects of those unloaded wallets are reinitialised """ - logger.info( - f"Started updating wallets with {len(wallets_update_list.values())} wallets" - ) timestamp = datetime.now() # list of wallets in the dict existing_names = list(self.wallets.keys()) @@ -172,11 +162,9 @@ def _update(self, wallets_update_list: Dict): wallet_alias = wallets_update_list[wallet]["alias"] wallet_name = wallets_update_list[wallet]["name"] - logger.info(f"Updating wallet {wallet_name}") # wallet from json not yet loaded in Bitcoin Core?! if os.path.join(self.rpc_path, wallet_alias) not in loaded_wallets: try: - logger.debug(f"Loading {wallet_name} to Bitcoin Core") self.rpc.loadwallet( os.path.join(self.rpc_path, wallet_alias) ) @@ -217,9 +205,6 @@ def _update(self, wallets_update_list: Dict): # % wallets_update_list[wallet]["alias"] # ) except RpcError as e: - logger.warning( - f"Couldn't load wallet {wallet_alias} into Bitcoin Core. Silently ignored! RPC error: {e}" - ) self._failed_load_wallets.append( { **wallets_update_list[wallet], @@ -227,9 +212,6 @@ def _update(self, wallets_update_list: Dict): } ) except Exception as e: - logger.warning( - f"Couldn't load wallet {wallet_alias}. Silently ignored! Wallet error: {e}" - ) logger.exception(e) self._failed_load_wallets.append( { diff --git a/src/cryptoadvance/specter/server_endpoints/controller.py b/src/cryptoadvance/specter/server_endpoints/controller.py index b4bcb3bed6..1c3ebbc0c1 100644 --- a/src/cryptoadvance/specter/server_endpoints/controller.py +++ b/src/cryptoadvance/specter/server_endpoints/controller.py @@ -234,8 +234,8 @@ def slow_request_detection_stop(response: Response): # Original looks like this: # [2023-02-10 14:15:49,716] INFO in _internal: 127.0.0.1 - - [10/Feb/2023 14:15:49] "GET /ext/devhelp/static/devhelp/img/orange-wrench.png HTTP/1.1" 304 - url: ParseResult = urlparse(request.url) - logger.info( - "{} {: <40} ({}) took {: >4} ms".format( + logger.info( # commas for being able to split up more easily afterwards + "{}, {: <40}, {}, took, {: >4}, ms".format( request.method, url.path + url.params + url.query + url.fragment, response.status_code, diff --git a/src/cryptoadvance/specterext/spectrum/config.py b/src/cryptoadvance/specterext/spectrum/config.py index f14f0feaf7..af70280d33 100644 --- a/src/cryptoadvance/specterext/spectrum/config.py +++ b/src/cryptoadvance/specterext/spectrum/config.py @@ -36,6 +36,10 @@ class BaseConfig(object): # The one which is chosen at startup ELECTRUM_DEFAULT_OPTION = "electrum.emzy.de" + SUPPRESS_JSONRPC_LOGGING = _get_bool_env_var( + "SUPPRESS_JSONRPC_LOGGING", default="false" + ) + # Level 1: How does persistence work? # Convention: BlaConfig @@ -79,7 +83,9 @@ class TestConfig(NigiriLocalElectrumLiteConfig): class DevelopmentConfig(EmzyElectrumLiteConfig): - pass + SUPPRESS_JSONRPC_LOGGING = _get_bool_env_var( + "SUPPRESS_JSONRPC_LOGGING", default="true" + ) class Development2Config(EmzyElectrumLiteConfig): @@ -89,6 +95,5 @@ class Development2Config(EmzyElectrumLiteConfig): class ProductionConfig(EmzyElectrumLiteConfig): - """Not sure whether we're production ready, though""" pass diff --git a/src/cryptoadvance/specterext/swan/service.py b/src/cryptoadvance/specterext/swan/service.py index 357a285ddf..2a7b24f624 100644 --- a/src/cryptoadvance/specterext/swan/service.py +++ b/src/cryptoadvance/specterext/swan/service.py @@ -11,7 +11,12 @@ from cryptoadvance.specter.user import User -from cryptoadvance.specter.services.service import Service, devstatus_prod +from cryptoadvance.specter.services.service import ( + Service, + devstatus_alpha, + devstatus_beta, + devstatus_prod, +) from cryptoadvance.specter.addresslist import Address from cryptoadvance.specter.wallet import Wallet from urllib.parse import urlparse @@ -28,7 +33,7 @@ class SwanService(Service): has_blueprint = True blueprint_module = "cryptoadvance.specterext.swan.controller" isolated_client = False - devstatus = devstatus_prod + devstatus = devstatus_beta encrypt_data = True # TODO: As more Services are integrated, we'll want more robust categorization and sorting logic