From 15008c5a89a99da0d2789469f758d290373a3fad Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Fri, 20 Dec 2024 16:44:08 -0500 Subject: [PATCH 01/12] Add facility to be specified as a sync parameter --- src/nsls2api/api/v1/jobs_api.py | 11 ++++++----- src/nsls2api/services/background_service.py | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/nsls2api/api/v1/jobs_api.py b/src/nsls2api/api/v1/jobs_api.py index ea9183d1..511587ce 100644 --- a/src/nsls2api/api/v1/jobs_api.py +++ b/src/nsls2api/api/v1/jobs_api.py @@ -73,8 +73,9 @@ async def sync_proposal_types(facility: FacilityName = FacilityName.nsls2): include_in_schema=SYNC_ROUTES_IN_SCHEMA, tags=["sync"], ) -async def sync_proposals_for_cycle(request: Request, cycle: str) -> BackgroundJob: - sync_params = JobSyncParameters(cycle=cycle) +async def sync_proposals_for_cycle(request: Request, cycle: str, + facility: FacilityName = FacilityName.nsls2) -> BackgroundJob: + sync_params = JobSyncParameters(cycle=cycle, facility=facility) job = await background_service.create_background_job( JobActions.synchronize_proposals_for_cycle, sync_parameters=sync_params, @@ -94,9 +95,9 @@ async def sync_cycles(facility: FacilityName = FacilityName.nsls2): @router.get("/sync/update-cycles/{facility}", include_in_schema=SYNC_ROUTES_IN_SCHEMA, tags=["sync"]) async def sync_update_cycles( - request: fastapi.Request, - facility: FacilityName = FacilityName.nsls2, - cycle: Optional[str] = None, + request: fastapi.Request, + facility: FacilityName = FacilityName.nsls2, + cycle: Optional[str] = None, ): sync_params = JobSyncParameters( facility=facility, sync_source=JobSyncSource.PASS diff --git a/src/nsls2api/services/background_service.py b/src/nsls2api/services/background_service.py index b5fc523d..49fdfaf0 100644 --- a/src/nsls2api/services/background_service.py +++ b/src/nsls2api/services/background_service.py @@ -105,9 +105,7 @@ async def worker_function(): try: match job.action: case JobActions.synchronize_admins: - logger.info( - f"Processing job {job.id} to synchronize admins." - ) + logger.info(f"Processing job {job.id} to synchronize admins.") await sync_service.worker_synchronize_dataadmins() case JobActions.update_cycle_information: logger.info( @@ -132,7 +130,7 @@ async def worker_function(): ) case JobActions.synchronize_proposals_for_cycle: logger.info( - f"Processing job {job.id} to synchronize proposals for cycle {job.sync_parameters.cycle} (from {job.sync_parameters.sync_source})." + f"Processing job {job.id} to synchronize proposals for the {job.sync_parameters.facility} facilities cycle {job.sync_parameters.cycle} (from {job.sync_parameters.sync_source})." ) await sync_service.worker_synchronize_proposals_for_cycle_from_pass( job.sync_parameters.cycle From 723a3f9fa32657afb8a2ecac41ebd847fdacdf8e Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Fri, 20 Dec 2024 16:47:24 -0500 Subject: [PATCH 02/12] propagate facility to worker function --- src/nsls2api/services/background_service.py | 2 +- src/nsls2api/services/sync_service.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nsls2api/services/background_service.py b/src/nsls2api/services/background_service.py index 49fdfaf0..2eb80c6a 100644 --- a/src/nsls2api/services/background_service.py +++ b/src/nsls2api/services/background_service.py @@ -133,7 +133,7 @@ async def worker_function(): f"Processing job {job.id} to synchronize proposals for the {job.sync_parameters.facility} facilities cycle {job.sync_parameters.cycle} (from {job.sync_parameters.sync_source})." ) await sync_service.worker_synchronize_proposals_for_cycle_from_pass( - job.sync_parameters.cycle + job.sync_parameters.cycle, job.sync_paramters.facility ) case JobActions.synchronize_proposal_types: logger.info( diff --git a/src/nsls2api/services/sync_service.py b/src/nsls2api/services/sync_service.py index 5381b5b7..3f1c3a38 100644 --- a/src/nsls2api/services/sync_service.py +++ b/src/nsls2api/services/sync_service.py @@ -323,7 +323,8 @@ async def worker_synchronize_proposal_from_pass(proposal_id: str) -> None: ) -async def worker_synchronize_proposals_for_cycle_from_pass(cycle: str) -> None: +async def worker_synchronize_proposals_for_cycle_from_pass(cycle: str, + facility_name: FacilityName = FacilityName.nsls2) -> None: start_time = datetime.datetime.now() cycle_year = await facility_service.cycle_year(cycle) From b19fa379cf499ac7f113fb15390206e65a8959c9 Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Fri, 20 Dec 2024 16:53:46 -0500 Subject: [PATCH 03/12] propagate facility to services functions --- src/nsls2api/services/proposal_service.py | 38 +++++++++++------------ src/nsls2api/services/sync_service.py | 12 +++---- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/nsls2api/services/proposal_service.py b/src/nsls2api/services/proposal_service.py index e6b9eb3c..b919ebac 100644 --- a/src/nsls2api/services/proposal_service.py +++ b/src/nsls2api/services/proposal_service.py @@ -74,8 +74,8 @@ async def recently_updated(count=5, beamline: str | None = None): # return result -async def fetch_proposals_for_cycle(cycle_name: str) -> list[str]: - cycle = await Cycle.find_one(Cycle.name == cycle_name) +async def fetch_proposals_for_cycle(cycle_name: str, facility_name: FacilityName = FacilityName.nsls2) -> list[str]: + cycle = await Cycle.find_one(Cycle.name == cycle_name, Cycle.facility == facility_name) if cycle is None: raise LookupError(f"Cycle {cycle} not found in local database.") return cycle.proposals @@ -151,13 +151,13 @@ async def search_proposals(search_text: str) -> Optional[list[Proposal]]: # Get a list of proposals that match the given criteria async def fetch_proposals( - proposal_id: list[str] | None = None, - beamline: list[str] | None = None, - cycle: list[str] | None = None, - facility: list[str] | None = None, - page_size: int = 10, - page: int = 1, - include_directories: bool = False, + proposal_id: list[str] | None = None, + beamline: list[str] | None = None, + cycle: list[str] | None = None, + facility: list[str] | None = None, + page_size: int = 10, + page: int = 1, + include_directories: bool = False, ) -> Optional[list[ProposalFullDetails]]: query = [] @@ -202,7 +202,7 @@ async def fetch_proposals( async def proposal_type_description_from_pass_type_id( - pass_type_id: int, + pass_type_id: int, ) -> Optional[str]: proposal_type = await ProposalType.find_one( ProposalType.pass_id == str(pass_type_id) @@ -245,7 +245,7 @@ async def fetch_users_on_proposal(proposal_id: str) -> Optional[list[User]]: async def fetch_usernames_from_proposal( - proposal_id: str, + proposal_id: str, ) -> Optional[list[str]]: proposal = await proposal_by_id(proposal_id) @@ -304,18 +304,18 @@ async def has_valid_cycle(proposal: Proposal): # If we don't have any cycles listed and this is not a commissioning # proposal then the cycle information is invalid return not ( - (len(proposal.cycles) == 0) - and ( - proposal.pass_type_id != 300005 - or proposal.type == "Beamline Commissioning (beamline staff only)" - ) + (len(proposal.cycles) == 0) + and ( + proposal.pass_type_id != 300005 + or proposal.type == "Beamline Commissioning (beamline staff only)" + ) ) async def is_commissioning(proposal: Proposal): return ( - proposal.pass_type_id == "300005" - or proposal.type == "Beamline Commissioning (beamline staff only)" + proposal.pass_type_id == "300005" + or proposal.type == "Beamline Commissioning (beamline staff only)" ) @@ -439,7 +439,7 @@ async def generate_fake_proposal_id() -> int: async def generate_fake_test_proposal( - facility_name: FacilityName = FacilityName.nsls2, add_specific_user=None + facility_name: FacilityName = FacilityName.nsls2, add_specific_user=None ) -> Optional[Proposal]: """ Generates a fake test proposal. diff --git a/src/nsls2api/services/sync_service.py b/src/nsls2api/services/sync_service.py index 3f1c3a38..356ad3d7 100644 --- a/src/nsls2api/services/sync_service.py +++ b/src/nsls2api/services/sync_service.py @@ -288,7 +288,7 @@ async def synchronize_proposal_from_pass(proposal_id: str) -> None: logger.debug(f"Response: {response}") -async def update_proposals_with_cycle(cycle_name: str) -> None: +async def update_proposals_with_cycle(cycle_name: str, facility_name: FacilityName = FacilityName.nsls2) -> None: """ Update the cycle <-> proposals mapping for the given cycle. @@ -296,9 +296,9 @@ async def update_proposals_with_cycle(cycle_name: str) -> None: :type cycle_name: str """ - proposal_list = await proposal_service.fetch_proposals_for_cycle(cycle_name) + proposal_list = await proposal_service.fetch_proposals_for_cycle(cycle_name, facility_name=facility_name) - logger.info(f"Found {len(proposal_list)} proposals for cycle {cycle_name}.") + logger.info(f"Found {len(proposal_list)} proposals for {facility_name} cycle {cycle_name}.") for proposal_id in proposal_list: # Add the cycle to the Proposal object @@ -327,10 +327,10 @@ async def worker_synchronize_proposals_for_cycle_from_pass(cycle: str, facility_name: FacilityName = FacilityName.nsls2) -> None: start_time = datetime.datetime.now() - cycle_year = await facility_service.cycle_year(cycle) + cycle_year = await facility_service.cycle_year(cycle, facility_name=facility_name) - proposals = await proposal_service.fetch_proposals_for_cycle(cycle) - logger.info(f"Synchronizing {len(proposals)} proposals for {cycle} cycle.") + proposals = await proposal_service.fetch_proposals_for_cycle(cycle, facility_name=facility_name) + logger.info(f"Synchronizing {len(proposals)} proposals for facility {facility_name} in {cycle} cycle.") for proposal_id in proposals: logger.info(f"Synchronizing proposal {proposal_id}.") From ff453430e3365e4e0117d1455684fadd239afc3f Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Fri, 20 Dec 2024 16:56:24 -0500 Subject: [PATCH 04/12] Add facility to commissioning proposals query --- src/nsls2api/services/pass_service.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nsls2api/services/pass_service.py b/src/nsls2api/services/pass_service.py index e9c666fc..ecead96c 100644 --- a/src/nsls2api/services/pass_service.py +++ b/src/nsls2api/services/pass_service.py @@ -121,11 +121,11 @@ async def get_saf_from_proposal( return saf_list -async def get_commissioning_proposals_by_year(year: str, facility: FacilityName = FacilityName.nsls2) -> Optional[list[PassProposal]]: +async def get_commissioning_proposals_by_year(year: str, facility_name: FacilityName = FacilityName.nsls2) -> Optional[list[PassProposal]]: - pass_facility = await facility_service.pass_id_for_facility(facility) + pass_facility = await facility_service.pass_id_for_facility(facility_name) if not pass_facility: - error_message: str = f"Facility {facility} does not have a PASS ID." + error_message: str = f"Facility {facility_name} does not have a PASS ID." logger.error(error_message) raise PassException(error_message) @@ -139,11 +139,11 @@ async def get_commissioning_proposals_by_year(year: str, facility: FacilityName for commissioning_proposal in pass_commissioning_proposals: commissioning_proposal_list.append(PassProposal(**commissioning_proposal)) except ValidationError as error: - error_message = f"Error validating commissioning proposal data received from PASS for year {str(year)} at {facility} facility." + error_message = f"Error validating commissioning proposal data received from PASS for year {str(year)} at {facility_name} facility." logger.error(error_message) raise PassException(error_message) from error except Exception as error: - error_message = f"Error retrieving commissioning proposal information from PASS for year {str(year)} at {facility} facility." + error_message = f"Error retrieving commissioning proposal information from PASS for year {str(year)} at {facility_name} facility." logger.exception(error_message) raise PassException(error_message) from error From e02fa09f803cb8a9c0b88d98c09a69a156b95ef9 Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Fri, 20 Dec 2024 16:56:57 -0500 Subject: [PATCH 05/12] Add facility to sync service --- src/nsls2api/services/sync_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nsls2api/services/sync_service.py b/src/nsls2api/services/sync_service.py index 356ad3d7..9432d937 100644 --- a/src/nsls2api/services/sync_service.py +++ b/src/nsls2api/services/sync_service.py @@ -338,7 +338,7 @@ async def worker_synchronize_proposals_for_cycle_from_pass(cycle: str, commissioning_proposals: list[ PassProposal - ] = await pass_service.get_commissioning_proposals_by_year(cycle_year) + ] = await pass_service.get_commissioning_proposals_by_year(cycle_year, facility_name=facility_name) logger.info( f"Synchronizing {len(proposals)} commissioning proposals for the year {cycle_year}." ) @@ -347,7 +347,7 @@ async def worker_synchronize_proposals_for_cycle_from_pass(cycle: str, await synchronize_proposal_from_pass(str(proposal.Proposal_ID)) # Now update the cycle information for each proposal - await update_proposals_with_cycle(cycle) + await update_proposals_with_cycle(cycle, facility_name=facility_name) time_taken = datetime.datetime.now() - start_time logger.info( From 195dcd603ecb5bdf388ab54cc2de987367dc71cb Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Thu, 9 Jan 2025 09:56:32 -0500 Subject: [PATCH 06/12] Pass facility into proposal_service function call. --- src/nsls2api/api/v1/facility_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nsls2api/api/v1/facility_api.py b/src/nsls2api/api/v1/facility_api.py index 89002d01..f966f52a 100644 --- a/src/nsls2api/api/v1/facility_api.py +++ b/src/nsls2api/api/v1/facility_api.py @@ -69,10 +69,10 @@ async def get_proposals_for_cycle(facility: FacilityName, cycle: str): status_code=501, ) - proposal_list = await proposal_service.fetch_proposals_for_cycle(cycle) + proposal_list = await proposal_service.fetch_proposals_for_cycle(cycle, facility) if proposal_list is None: return fastapi.responses.JSONResponse( - {"error": f"No proposals were found for cycle {cycle}"}, + {"error": f"No proposals were found for cycle {cycle} for facility {facility.name}"}, status_code=404, ) model = CycleProposalList( From aba9245377c417cb9c68a2816213561f1b171d01 Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Thu, 9 Jan 2025 09:58:18 -0500 Subject: [PATCH 07/12] Fix misuse of english. --- src/nsls2api/services/background_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nsls2api/services/background_service.py b/src/nsls2api/services/background_service.py index e131a627..2bb1fd0e 100644 --- a/src/nsls2api/services/background_service.py +++ b/src/nsls2api/services/background_service.py @@ -130,7 +130,7 @@ async def worker_function(): ) case JobActions.synchronize_proposals_for_cycle: logger.info( - f"Processing job {job.id} to synchronize proposals for the {job.sync_parameters.facility} facilities cycle {job.sync_parameters.cycle} (from {job.sync_parameters.sync_source})." + f"Processing job {job.id} to synchronize proposals for the {job.sync_parameters.facility} facility's cycle {job.sync_parameters.cycle} (from {job.sync_parameters.sync_source})." ) await sync_service.worker_synchronize_proposals_for_cycle_from_pass( job.sync_parameters.cycle, job.sync_paramters.facility From cf936f01cde8600861446c60ca9ba9fd17edcecb Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Thu, 9 Jan 2025 16:16:54 -0500 Subject: [PATCH 08/12] Remove unnecessary check - the pydantic model will not validate if the facility name is not correct --- src/nsls2api/api/v1/facility_api.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/nsls2api/api/v1/facility_api.py b/src/nsls2api/api/v1/facility_api.py index f966f52a..b6bed2c4 100644 --- a/src/nsls2api/api/v1/facility_api.py +++ b/src/nsls2api/api/v1/facility_api.py @@ -62,13 +62,6 @@ async def get_facility_cycles(facility: FacilityName): include_in_schema=True, ) async def get_proposals_for_cycle(facility: FacilityName, cycle: str): - if facility.name != "nsls2": - # TODO: Add other facilities - return fastapi.responses.JSONResponse( - {"message": f"Not implemented for the {facility.name} facility."}, - status_code=501, - ) - proposal_list = await proposal_service.fetch_proposals_for_cycle(cycle, facility) if proposal_list is None: return fastapi.responses.JSONResponse( From 6cc9b9ce84ff5044451f4b5f33a539e7908ca239 Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Thu, 9 Jan 2025 16:29:09 -0500 Subject: [PATCH 09/12] Added new endpoint to (hopefully) be more consistent. Deprecated the existing one --- src/nsls2api/api/v1/jobs_api.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/nsls2api/api/v1/jobs_api.py b/src/nsls2api/api/v1/jobs_api.py index 86bab76a..48e68c95 100644 --- a/src/nsls2api/api/v1/jobs_api.py +++ b/src/nsls2api/api/v1/jobs_api.py @@ -13,7 +13,8 @@ ) from nsls2api.services import background_service -SYNC_ROUTES_IN_SCHEMA = False +# TODO: This flag should be automatically set to be True for development but False for production. +SYNC_ROUTES_IN_SCHEMA = True router = fastapi.APIRouter(tags=["jobs"]) @@ -81,7 +82,7 @@ async def sync_proposal_types(facility: FacilityName = FacilityName.nsls2): "/sync/proposals/cycle/{cycle}", dependencies=[Depends(get_current_user)], include_in_schema=SYNC_ROUTES_IN_SCHEMA, - tags=["sync"], + tags=["sync"], deprecated=True, ) async def sync_proposals_for_cycle(request: Request, cycle: str, facility: FacilityName = FacilityName.nsls2) -> BackgroundJob: @@ -93,6 +94,22 @@ async def sync_proposals_for_cycle(request: Request, cycle: str, return job +@router.get( + "/sync/facility/{facility}/cycle/{cycle}/proposals", + dependencies=[Depends(get_current_user)], + include_in_schema=SYNC_ROUTES_IN_SCHEMA, + tags=["sync"], +) +async def sync_proposals_for_facility_cycle(request: Request, + facility: FacilityName, cycle: str) -> BackgroundJob: + sync_params = JobSyncParameters(cycle=cycle, facility=facility) + job = await background_service.create_background_job( + JobActions.synchronize_proposals_for_cycle, + sync_parameters=sync_params, + ) + return job + + @router.get( "/sync/cycles/{facility}", include_in_schema=SYNC_ROUTES_IN_SCHEMA, tags=["sync"] ) @@ -108,7 +125,7 @@ async def sync_cycles(facility: FacilityName = FacilityName.nsls2): @router.get( "/sync/update-cycles/{facility}", include_in_schema=SYNC_ROUTES_IN_SCHEMA, - tags=["sync"], + tags=["sync"], summary="Updates the local (nsls2core DB) cycle <-> proposal mapping" ) async def sync_update_cycles( request: fastapi.Request, From 3c603361cb792d5d77087a499f206e84407ed4f4 Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Thu, 9 Jan 2025 16:39:40 -0500 Subject: [PATCH 10/12] Applied ruff code formatting --- src/nsls2api/api/v1/facility_api.py | 4 ++- src/nsls2api/api/v1/jobs_api.py | 22 +++++++----- src/nsls2api/api/v1/proposal_api.py | 1 - src/nsls2api/services/pass_service.py | 5 +-- src/nsls2api/services/proposal_service.py | 42 +++++++++++++---------- src/nsls2api/services/sync_service.py | 29 +++++++++++----- 6 files changed, 63 insertions(+), 40 deletions(-) diff --git a/src/nsls2api/api/v1/facility_api.py b/src/nsls2api/api/v1/facility_api.py index b6bed2c4..0246d28f 100644 --- a/src/nsls2api/api/v1/facility_api.py +++ b/src/nsls2api/api/v1/facility_api.py @@ -65,7 +65,9 @@ async def get_proposals_for_cycle(facility: FacilityName, cycle: str): proposal_list = await proposal_service.fetch_proposals_for_cycle(cycle, facility) if proposal_list is None: return fastapi.responses.JSONResponse( - {"error": f"No proposals were found for cycle {cycle} for facility {facility.name}"}, + { + "error": f"No proposals were found for cycle {cycle} for facility {facility.name}" + }, status_code=404, ) model = CycleProposalList( diff --git a/src/nsls2api/api/v1/jobs_api.py b/src/nsls2api/api/v1/jobs_api.py index 48e68c95..0a6961c1 100644 --- a/src/nsls2api/api/v1/jobs_api.py +++ b/src/nsls2api/api/v1/jobs_api.py @@ -82,10 +82,12 @@ async def sync_proposal_types(facility: FacilityName = FacilityName.nsls2): "/sync/proposals/cycle/{cycle}", dependencies=[Depends(get_current_user)], include_in_schema=SYNC_ROUTES_IN_SCHEMA, - tags=["sync"], deprecated=True, + tags=["sync"], + deprecated=True, ) -async def sync_proposals_for_cycle(request: Request, cycle: str, - facility: FacilityName = FacilityName.nsls2) -> BackgroundJob: +async def sync_proposals_for_cycle( + request: Request, cycle: str, facility: FacilityName = FacilityName.nsls2 +) -> BackgroundJob: sync_params = JobSyncParameters(cycle=cycle, facility=facility) job = await background_service.create_background_job( JobActions.synchronize_proposals_for_cycle, @@ -100,8 +102,9 @@ async def sync_proposals_for_cycle(request: Request, cycle: str, include_in_schema=SYNC_ROUTES_IN_SCHEMA, tags=["sync"], ) -async def sync_proposals_for_facility_cycle(request: Request, - facility: FacilityName, cycle: str) -> BackgroundJob: +async def sync_proposals_for_facility_cycle( + request: Request, facility: FacilityName, cycle: str +) -> BackgroundJob: sync_params = JobSyncParameters(cycle=cycle, facility=facility) job = await background_service.create_background_job( JobActions.synchronize_proposals_for_cycle, @@ -125,12 +128,13 @@ async def sync_cycles(facility: FacilityName = FacilityName.nsls2): @router.get( "/sync/update-cycles/{facility}", include_in_schema=SYNC_ROUTES_IN_SCHEMA, - tags=["sync"], summary="Updates the local (nsls2core DB) cycle <-> proposal mapping" + tags=["sync"], + summary="Updates the local (nsls2core DB) cycle <-> proposal mapping", ) async def sync_update_cycles( - request: fastapi.Request, - facility: FacilityName = FacilityName.nsls2, - cycle: Optional[str] = None, + request: fastapi.Request, + facility: FacilityName = FacilityName.nsls2, + cycle: Optional[str] = None, ): sync_params = JobSyncParameters(facility=facility, sync_source=JobSyncSource.PASS) diff --git a/src/nsls2api/api/v1/proposal_api.py b/src/nsls2api/api/v1/proposal_api.py index 8ffbb0ae..b53f2e77 100644 --- a/src/nsls2api/api/v1/proposal_api.py +++ b/src/nsls2api/api/v1/proposal_api.py @@ -1,4 +1,3 @@ -from http.client import HTTPException from typing import Annotated import fastapi diff --git a/src/nsls2api/services/pass_service.py b/src/nsls2api/services/pass_service.py index 9ceac027..4bc55a91 100644 --- a/src/nsls2api/services/pass_service.py +++ b/src/nsls2api/services/pass_service.py @@ -121,8 +121,9 @@ async def get_saf_from_proposal( return saf_list -async def get_commissioning_proposals_by_year(year: str, facility_name: FacilityName = FacilityName.nsls2) -> Optional[list[PassProposal]]: - +async def get_commissioning_proposals_by_year( + year: str, facility_name: FacilityName = FacilityName.nsls2 +) -> Optional[list[PassProposal]]: pass_facility = await facility_service.pass_id_for_facility(facility_name) if not pass_facility: error_message: str = f"Facility {facility_name} does not have a PASS ID." diff --git a/src/nsls2api/services/proposal_service.py b/src/nsls2api/services/proposal_service.py index b919ebac..88e7f778 100644 --- a/src/nsls2api/services/proposal_service.py +++ b/src/nsls2api/services/proposal_service.py @@ -74,8 +74,12 @@ async def recently_updated(count=5, beamline: str | None = None): # return result -async def fetch_proposals_for_cycle(cycle_name: str, facility_name: FacilityName = FacilityName.nsls2) -> list[str]: - cycle = await Cycle.find_one(Cycle.name == cycle_name, Cycle.facility == facility_name) +async def fetch_proposals_for_cycle( + cycle_name: str, facility_name: FacilityName = FacilityName.nsls2 +) -> list[str]: + cycle = await Cycle.find_one( + Cycle.name == cycle_name, Cycle.facility == facility_name + ) if cycle is None: raise LookupError(f"Cycle {cycle} not found in local database.") return cycle.proposals @@ -151,13 +155,13 @@ async def search_proposals(search_text: str) -> Optional[list[Proposal]]: # Get a list of proposals that match the given criteria async def fetch_proposals( - proposal_id: list[str] | None = None, - beamline: list[str] | None = None, - cycle: list[str] | None = None, - facility: list[str] | None = None, - page_size: int = 10, - page: int = 1, - include_directories: bool = False, + proposal_id: list[str] | None = None, + beamline: list[str] | None = None, + cycle: list[str] | None = None, + facility: list[str] | None = None, + page_size: int = 10, + page: int = 1, + include_directories: bool = False, ) -> Optional[list[ProposalFullDetails]]: query = [] @@ -202,7 +206,7 @@ async def fetch_proposals( async def proposal_type_description_from_pass_type_id( - pass_type_id: int, + pass_type_id: int, ) -> Optional[str]: proposal_type = await ProposalType.find_one( ProposalType.pass_id == str(pass_type_id) @@ -245,7 +249,7 @@ async def fetch_users_on_proposal(proposal_id: str) -> Optional[list[User]]: async def fetch_usernames_from_proposal( - proposal_id: str, + proposal_id: str, ) -> Optional[list[str]]: proposal = await proposal_by_id(proposal_id) @@ -304,18 +308,18 @@ async def has_valid_cycle(proposal: Proposal): # If we don't have any cycles listed and this is not a commissioning # proposal then the cycle information is invalid return not ( - (len(proposal.cycles) == 0) - and ( - proposal.pass_type_id != 300005 - or proposal.type == "Beamline Commissioning (beamline staff only)" - ) + (len(proposal.cycles) == 0) + and ( + proposal.pass_type_id != 300005 + or proposal.type == "Beamline Commissioning (beamline staff only)" + ) ) async def is_commissioning(proposal: Proposal): return ( - proposal.pass_type_id == "300005" - or proposal.type == "Beamline Commissioning (beamline staff only)" + proposal.pass_type_id == "300005" + or proposal.type == "Beamline Commissioning (beamline staff only)" ) @@ -439,7 +443,7 @@ async def generate_fake_proposal_id() -> int: async def generate_fake_test_proposal( - facility_name: FacilityName = FacilityName.nsls2, add_specific_user=None + facility_name: FacilityName = FacilityName.nsls2, add_specific_user=None ) -> Optional[Proposal]: """ Generates a fake test proposal. diff --git a/src/nsls2api/services/sync_service.py b/src/nsls2api/services/sync_service.py index fb8fc0a8..08180128 100644 --- a/src/nsls2api/services/sync_service.py +++ b/src/nsls2api/services/sync_service.py @@ -317,7 +317,9 @@ async def synchronize_proposal_from_pass(proposal_id: str) -> None: logger.debug(f"Response: {response}") -async def update_proposals_with_cycle(cycle_name: str, facility_name: FacilityName = FacilityName.nsls2) -> None: +async def update_proposals_with_cycle( + cycle_name: str, facility_name: FacilityName = FacilityName.nsls2 +) -> None: """ Update the cycle <-> proposals mapping for the given cycle. @@ -325,9 +327,13 @@ async def update_proposals_with_cycle(cycle_name: str, facility_name: FacilityNa :type cycle_name: str """ - proposal_list = await proposal_service.fetch_proposals_for_cycle(cycle_name, facility_name=facility_name) + proposal_list = await proposal_service.fetch_proposals_for_cycle( + cycle_name, facility_name=facility_name + ) - logger.info(f"Found {len(proposal_list)} proposals for {facility_name} cycle {cycle_name}.") + logger.info( + f"Found {len(proposal_list)} proposals for {facility_name} cycle {cycle_name}." + ) for proposal_id in proposal_list: # Add the cycle to the Proposal object @@ -352,14 +358,19 @@ async def worker_synchronize_proposal_from_pass(proposal_id: str) -> None: ) -async def worker_synchronize_proposals_for_cycle_from_pass(cycle: str, - facility_name: FacilityName = FacilityName.nsls2) -> None: +async def worker_synchronize_proposals_for_cycle_from_pass( + cycle: str, facility_name: FacilityName = FacilityName.nsls2 +) -> None: start_time = datetime.datetime.now() cycle_year = await facility_service.cycle_year(cycle, facility_name=facility_name) - proposals = await proposal_service.fetch_proposals_for_cycle(cycle, facility_name=facility_name) - logger.info(f"Synchronizing {len(proposals)} proposals for facility {facility_name} in {cycle} cycle.") + proposals = await proposal_service.fetch_proposals_for_cycle( + cycle, facility_name=facility_name + ) + logger.info( + f"Synchronizing {len(proposals)} proposals for facility {facility_name} in {cycle} cycle." + ) for proposal_id in proposals: logger.info(f"Synchronizing proposal {proposal_id}.") @@ -367,7 +378,9 @@ async def worker_synchronize_proposals_for_cycle_from_pass(cycle: str, commissioning_proposals: list[ PassProposal - ] = await pass_service.get_commissioning_proposals_by_year(cycle_year, facility_name=facility_name) + ] = await pass_service.get_commissioning_proposals_by_year( + cycle_year, facility_name=facility_name + ) logger.info( f"Synchronizing {len(proposals)} commissioning proposals for the year {cycle_year}." ) From 452d7a584a4e9dae2788d10db2d997455389aea6 Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Thu, 9 Jan 2025 16:59:20 -0500 Subject: [PATCH 11/12] Added LBMS proposals per cycle stats This really needs a rethink - but for now just add the LBMS stats. --- src/nsls2api/api/models/stats_model.py | 3 ++- src/nsls2api/api/v1/stats_api.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/nsls2api/api/models/stats_model.py b/src/nsls2api/api/models/stats_model.py index 5b0b735e..101cbd52 100644 --- a/src/nsls2api/api/models/stats_model.py +++ b/src/nsls2api/api/models/stats_model.py @@ -13,7 +13,8 @@ class StatsModel(pydantic.BaseModel): beamline_count: int commissioning_proposal_count: int facility_data_health: bool - nsls2_proposals_per_cycle: Optional[list[ProposalsPerCycleModel]] = [] + nsls2_proposals_per_cycle: Optional[list[ProposalsPerCycleModel]] + lbms_proposals_per_cycle: Optional[list[ProposalsPerCycleModel]] class AboutModel(pydantic.BaseModel): diff --git a/src/nsls2api/api/v1/stats_api.py b/src/nsls2api/api/v1/stats_api.py index 490e9c84..8a62f1b9 100644 --- a/src/nsls2api/api/v1/stats_api.py +++ b/src/nsls2api/api/v1/stats_api.py @@ -24,8 +24,8 @@ async def stats(): facility_data_health = await facility_service.is_healthy("nsls2") + # Get the NSLS-II proposals per cycle nsls2_proposals_per_cycle: list[ProposalsPerCycleModel] = [] - nsls2_cycle_list = await facility_service.facility_cycles("nsls2") for cycle in nsls2_cycle_list: proposal_list = await proposal_service.fetch_proposals_for_cycle(cycle) @@ -35,6 +35,17 @@ async def stats(): ) nsls2_proposals_per_cycle.append(model) + # Get the LBMS proposals per cycle + lbms_proposals_per_cycle: list[ProposalsPerCycleModel] = [] + lbms_cycle_list = await facility_service.facility_cycles("lbms") + for cycle in lbms_cycle_list: + proposal_list = await proposal_service.fetch_proposals_for_cycle(cycle) + if proposal_list is not None: + model = ProposalsPerCycleModel( + cycle=cycle, proposal_count=len(proposal_list) + ) + lbms_proposals_per_cycle.append(model) + model = StatsModel( facility_count=facilities, beamline_count=beamlines, @@ -42,6 +53,7 @@ async def stats(): commissioning_proposal_count=commissioning, facility_data_health=facility_data_health, nsls2_proposals_per_cycle=nsls2_proposals_per_cycle, + lbms_proposals_per_cycle=lbms_proposals_per_cycle, ) return model From dd545638f1fea5079ed670ec7d8ccfecd4d76004 Mon Sep 17 00:00:00 2001 From: Stuart Campbell Date: Thu, 9 Jan 2025 17:13:02 -0500 Subject: [PATCH 12/12] Update src/nsls2api/services/background_service.py Co-authored-by: Jun Aishima --- src/nsls2api/services/background_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nsls2api/services/background_service.py b/src/nsls2api/services/background_service.py index 2bb1fd0e..bb2f8b84 100644 --- a/src/nsls2api/services/background_service.py +++ b/src/nsls2api/services/background_service.py @@ -133,7 +133,7 @@ async def worker_function(): f"Processing job {job.id} to synchronize proposals for the {job.sync_parameters.facility} facility's cycle {job.sync_parameters.cycle} (from {job.sync_parameters.sync_source})." ) await sync_service.worker_synchronize_proposals_for_cycle_from_pass( - job.sync_parameters.cycle, job.sync_paramters.facility + job.sync_parameters.cycle, job.sync_parameters.facility ) case JobActions.synchronize_proposal_types: logger.info(