From 4d4a0e8f058bfaa50b7131c4473acf52f8fae898 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Fri, 29 Sep 2023 07:08:38 -0700 Subject: [PATCH 1/6] print error message from new backend --- pyscicat/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index f43a52d..fd3317c 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -113,16 +113,16 @@ def _call_endpoint( response = self._send_to_scicat(cmd=cmd, endpoint=endpoint, data=data) result = response.json() if not response.ok: - err = result.get("error", {}) + # err = result.get("error", {}) if ( allow_404 and response.status_code == 404 - and re.match(r"Unknown (.+ )?id", err.get("message", "")) + and re.match(r"Unknown (.+ )?id", result) ): # The operation failed but because the object does not exist in SciCat. - logger.error("Error in operation %s: %s", operation, err) + logger.error("Error in operation %s: %s", operation, result) return None - raise ScicatCommError(f"Error in operation {operation}: {err}") + raise ScicatCommError(f"Error in operation {operation}: {result}") logger.info( "Operation '%s' successful%s", operation, From 132df1edde52b398cd1b08656d2e3ce9102b6090 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Fri, 29 Sep 2023 14:51:30 -0700 Subject: [PATCH 2/6] add CreateDatasetOrigDatablockDto --- pyscicat/client.py | 14 +++++++++----- pyscicat/model.py | 26 ++++++++++++++++++-------- tests/test_pyscicat/test_client.py | 30 +++++++++++++----------------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index fd3317c..9f46761 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -15,9 +15,9 @@ from pyscicat.model import ( Attachment, + CreateDatasetOrigDatablockDto, Dataset, Instrument, - OrigDatablock, Proposal, Sample, ) @@ -111,7 +111,8 @@ def _call_endpoint( allow_404=False, ) -> Optional[dict]: response = self._send_to_scicat(cmd=cmd, endpoint=endpoint, data=data) - result = response.json() + + result = response.json() if len(response.content) > 0 else None if not response.ok: # err = result.get("error", {}) if ( @@ -123,6 +124,7 @@ def _call_endpoint( logger.error("Error in operation %s: %s", operation, result) return None raise ScicatCommError(f"Error in operation {operation}: {result}") + logger.info( "Operation '%s' successful%s", operation, @@ -200,7 +202,9 @@ def datasets_update(self, dataset: Dataset, pid: str) -> str: """ update_dataset = datasets_update - def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: + def datasets_origdatablock_create( + self, dataset_id: str, datablockDto: CreateDatasetOrigDatablockDto + ) -> dict: """ Create a new SciCat Dataset OrigDatablock This function has been renamed. @@ -223,11 +227,11 @@ def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: Raises if a non-20x message is returned """ - endpoint = f"Datasets/{quote_plus(origdatablock.datasetId)}/origdatablocks" + endpoint = f"Datasets/{quote_plus(dataset_id)}/origdatablocks" return self._call_endpoint( cmd="post", endpoint=endpoint, - data=origdatablock, + data=datablockDto, operation="datasets_origdatablock_create", ) diff --git a/pyscicat/model.py b/pyscicat/model.py index efb9919..4cc43e2 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -90,7 +90,9 @@ class Job(MongoQueryable): executionTime: Optional[str] = None jobParams: Optional[dict] = None jobStatusMessage: Optional[str] = None - datasetList: Optional[dict] = None # documentation says dict, but should maybe be list? + datasetList: Optional[ + dict + ] = None # documentation says dict, but should maybe be list? jobResultObject: Optional[dict] = None # ibid. @@ -115,7 +117,9 @@ class Dataset(Ownable): creationTime: str # datetime datasetName: Optional[str] = None description: Optional[str] = None - history: Optional[List[dict]] = None # list of foreigh key ids to the Messages table + history: Optional[ + List[dict] + ] = None # list of foreigh key ids to the Messages table instrumentId: Optional[str] = None isPublished: Optional[bool] = False keywords: Optional[List[str]] = None @@ -187,25 +191,31 @@ class Datablock(Ownable): id: Optional[str] = None # archiveId: str = None listed in catamel model, but comes back invalid? - size: int + packedSize: Optional[int] = None chkAlg: Optional[int] = None version: str = None instrumentGroup: Optional[str] = None - dataFileList: List[DataFile] datasetId: str -class OrigDatablock(Ownable): +class CreateDatasetOrigDatablockDto(BaseModel): + """ + DTO for creating a new dataset with an original datablock + """ + + chkAlg: Optional[int] = None + dataFileList: List[DataFile] + size: int + + +class OrigDatablock(Ownable, CreateDatasetOrigDatablockDto): """ An Original Datablock maps between a Dataset and contains DataFiles """ id: Optional[str] = None - # archiveId: str = None listed in catamel model, but comes back invalid? - size: int instrumentGroup: Optional[str] = None - dataFileList: List[DataFile] datasetId: str diff --git a/tests/test_pyscicat/test_client.py b/tests/test_pyscicat/test_client.py index 46584f1..8040f76 100644 --- a/tests/test_pyscicat/test_client.py +++ b/tests/test_pyscicat/test_client.py @@ -14,7 +14,7 @@ from pyscicat.model import ( Attachment, - Datablock, + CreateDatasetOrigDatablockDto, DataFile, Instrument, Proposal, @@ -138,14 +138,13 @@ def test_scicat_ingest(): # Datablock with DataFiles data_file = DataFile(path="/foo/bar", size=42) - data_block = Datablock( + data_block_dto = CreateDatasetOrigDatablockDto( size=42, - version="1", datasetId=dataset_id, dataFileList=[data_file], - **ownable.dict() + ) - scicat.upload_dataset_origdatablock(data_block) + scicat.upload_dataset_origdatablock(dataset_id, data_block_dto) # Attachment attachment = Attachment( @@ -187,17 +186,9 @@ def test_get_dataset(): def test_get_nonexistent_dataset(): with requests_mock.Mocker() as mock_request: mock_request.get( - local_url + "Datasets/74", - status_code=404, - reason="Not Found", - json={ - "error": { - "statusCode": 404, - "name": "Error", - "message": 'Unknown "Dataset" id "74".', - "code": "MODEL_NOT_FOUND", - } - }, + local_url + "datasets/74", + status_code=200, + content=b'' ) client = from_token(base_url=local_url, token="a_token") assert client.datasets_get_one("74") is None @@ -206,7 +197,7 @@ def test_get_nonexistent_dataset(): def test_get_dataset_bad_url(): with requests_mock.Mocker() as mock_request: mock_request.get( - "http://localhost:3000/api/v100/Datasets/53", + "http://localhost:3000/api/v100/datasets/53", status_code=404, reason="Not Found", json={ @@ -228,3 +219,8 @@ def test_initializers(): client = from_token(local_url, "let me in!") assert client._token == "let me in!" + +def test_real(): + client = from_token(base_url="http://localhost:3000/api/wer3", token="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJfaWQiOiI2NTE1ZGYwNWM4OGNhMzNkODkzNjA3NjkiLCJ1c2VybmFtZSI6ImluZ2VzdG9yIiwiZW1haWwiOiJzY2ljYXRpbmdlc3RvckB5b3VyLnNpdGUiLCJhdXRoU3RyYXRlZ3kiOiJsb2NhbCIsIl9fdiI6MCwiaWQiOiI2NTE1ZGYwNWM4OGNhMzNkODkzNjA3NjkiLCJpYXQiOjE2OTU5OTMyMTcsImV4cCI6MTY5NTk5NjgxN30.Dc-K39ikfSixMGXzURrJ0Z4lHwZGOzRWlpeU2u5fdIA") + response = client.datasets_get_one("sdfsdfl") + print(response) From f5d6c26cfc8aa9c380c2313fd3adaa8064485283 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Fri, 29 Sep 2023 14:53:24 -0700 Subject: [PATCH 3/6] remove allow_404 --- pyscicat/client.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 9f46761..8b3b48b 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -108,21 +108,11 @@ def _call_endpoint( endpoint: str, data: BaseModel = None, operation: str = "", - allow_404=False, ) -> Optional[dict]: response = self._send_to_scicat(cmd=cmd, endpoint=endpoint, data=data) result = response.json() if len(response.content) > 0 else None if not response.ok: - # err = result.get("error", {}) - if ( - allow_404 - and response.status_code == 404 - and re.match(r"Unknown (.+ )?id", result) - ): - # The operation failed but because the object does not exist in SciCat. - logger.error("Error in operation %s: %s", operation, result) - return None raise ScicatCommError(f"Error in operation {operation}: {result}") logger.info( @@ -522,7 +512,6 @@ def datasets_find( cmd="get", endpoint=f"Datasets/fullquery?{query}", operation="datasets_find", - allow_404=True, ) """ @@ -563,7 +552,7 @@ def datasets_get_many(self, filter_fields: Optional[dict] = None) -> Optional[di filter_fields = json.dumps(filter_fields) endpoint = f'Datasets?filter={{"where":{filter_fields}}}' return self._call_endpoint( - cmd="get", endpoint=endpoint, operation="datasets_get_many", allow_404=True + cmd="get", endpoint=endpoint, operation="datasets_get_many" ) """ @@ -599,7 +588,6 @@ def published_data_get_many(self, filter=None) -> Optional[dict]: cmd="get", endpoint=endpoint, operation="published_data_get_many", - allow_404=True, ) """ @@ -624,7 +612,6 @@ def datasets_get_one(self, pid: str) -> Optional[dict]: cmd="get", endpoint=f"Datasets/{quote_plus(pid)}", operation="datasets_get_one", - allow_404=True, ) get_dataset_by_pid = datasets_get_one @@ -663,7 +650,6 @@ def instruments_get_one(self, pid: str = None, name: str = None) -> Optional[dic cmd="get", endpoint=endpoint, operation="instruments_get_one", - allow_404=True, ) get_instrument = instruments_get_one @@ -689,7 +675,6 @@ def samples_get_one(self, pid: str) -> Optional[dict]: cmd="get", endpoint=f"Samples/{quote_plus(pid)}", operation="samples_get_one", - allow_404=True, ) get_sample = samples_get_one @@ -711,7 +696,7 @@ def proposals_get_one(self, pid: str = None) -> Optional[dict]: The proposal with the requested pid """ return self._call_endpoint( - cmd="get", endpoint=f"Proposals/{quote_plus(pid)}", allow_404=True + cmd="get", endpoint=f"Proposals/{quote_plus(pid)}" ) get_proposal = proposals_get_one @@ -736,7 +721,6 @@ def datasets_origdatablocks_get_one(self, pid: str) -> Optional[dict]: cmd="get", endpoint=f"Datasets/{quote_plus(pid)}/origdatablocks", operation="datasets_origdatablocks_get_one", - allow_404=True, ) get_dataset_origdatablocks = datasets_origdatablocks_get_one @@ -761,7 +745,6 @@ def datasets_delete(self, pid: str) -> Optional[dict]: cmd="delete", endpoint=f"Datasets/{quote_plus(pid)}", operation="datasets_delete", - allow_404=True, ) delete_dataset = datasets_delete From 9644ea6ea8b3caa30986327960c6a652570cf679 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Sun, 1 Oct 2023 16:59:48 -0700 Subject: [PATCH 4/6] fix logger message --- pyscicat/client.py | 5 ++--- tests/test_pyscicat/test_client.py | 5 ----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 8b3b48b..61f8b64 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -5,7 +5,6 @@ import hashlib import logging import json -import re from typing import Optional from urllib.parse import urljoin, quote_plus @@ -114,11 +113,11 @@ def _call_endpoint( result = response.json() if len(response.content) > 0 else None if not response.ok: raise ScicatCommError(f"Error in operation {operation}: {result}") - + logger.info( "Operation '%s' successful%s", operation, - f"pid={result['pid']}" if "pid" in result else "", + f"pid={result['pid']}" if result and "pid" in result else "", ) return result diff --git a/tests/test_pyscicat/test_client.py b/tests/test_pyscicat/test_client.py index 8040f76..1bd704e 100644 --- a/tests/test_pyscicat/test_client.py +++ b/tests/test_pyscicat/test_client.py @@ -219,8 +219,3 @@ def test_initializers(): client = from_token(local_url, "let me in!") assert client._token == "let me in!" - -def test_real(): - client = from_token(base_url="http://localhost:3000/api/wer3", token="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJfaWQiOiI2NTE1ZGYwNWM4OGNhMzNkODkzNjA3NjkiLCJ1c2VybmFtZSI6ImluZ2VzdG9yIiwiZW1haWwiOiJzY2ljYXRpbmdlc3RvckB5b3VyLnNpdGUiLCJhdXRoU3RyYXRlZ3kiOiJsb2NhbCIsIl9fdiI6MCwiaWQiOiI2NTE1ZGYwNWM4OGNhMzNkODkzNjA3NjkiLCJpYXQiOjE2OTU5OTMyMTcsImV4cCI6MTY5NTk5NjgxN30.Dc-K39ikfSixMGXzURrJ0Z4lHwZGOzRWlpeU2u5fdIA") - response = client.datasets_get_one("sdfsdfl") - print(response) From 85b28b81c78d3194b810308701567cd9ae570bfc Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Mon, 2 Oct 2023 09:12:12 -0700 Subject: [PATCH 5/6] fix user/pw logins --- pyscicat/client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 61f8b64..ae27cd5 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -809,9 +809,8 @@ def _log_in_via_auth_msad(base_url, username, password): verify=True, ) if not response.ok: - err = response.json()["error"] logger.error( - f'Error retrieving token for user: {err["name"]}, {err["statusCode"]}: {err["message"]}' + f'Error retrieving token for user: {response.json()}' ) raise ScicatLoginError(response.content) @@ -834,6 +833,6 @@ def get_token(base_url, username, password): err = response.json()["error"] logger.error( - f' Failed log in: {err["name"]}, {err["statusCode"]}: {err["message"]}' + f' Failed log in: {response.json()}' ) raise ScicatLoginError(response.content) From 4e7ab3c0b2709903ab4619e20fc97ea0876b515b Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Mon, 2 Oct 2023 09:14:37 -0700 Subject: [PATCH 6/6] removed unused variable --- pyscicat/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index ae27cd5..9c31942 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -831,7 +831,6 @@ def get_token(base_url, username, password): if response.ok: return response.json()["access_token"] - err = response.json()["error"] logger.error( f' Failed log in: {response.json()}' )