From ed1b6850af632e31f4128088c093d0d241388094 Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Fri, 22 Nov 2024 14:45:42 +0200 Subject: [PATCH 01/14] finalise release --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2aacd390b0..168105ba592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 1.6.0 (TBD) +## 1.6.0 ## Improvements @@ -44,7 +44,7 @@ - Only render units/postfix when field has a value [#7055](https://github.com/opencrvs/opencrvs-core/issues/7055) - Only show items with values in review [#5192](https://github.com/opencrvs/opencrvs-core/pull/5192) - Fix prefix text overlap issue in form text inputs -- **Staged files getting reset on precommit hook failure** We were running lint-staged separately on each package using lerna which potentially created a race condition causing staged changes to get lost on failure. Now we are running lint-staged directly without depending on lerna. ***This is purely a DX improvement without affecting any functionality of the system*** +- **Staged files getting reset on precommit hook failure** We were running lint-staged separately on each package using lerna which potentially created a race condition causing staged changes to get lost on failure. Now we are running lint-staged directly without depending on lerna. **_This is purely a DX improvement without affecting any functionality of the system_** - Fix `informantType` missing in template object which prevented rendering informant relationship data in the certificates [#5952](https://github.com/opencrvs/opencrvs-core/issues/5952) - Fix users hitting rate limit when multiple users authenticated the same time with different usernames [#7728](https://github.com/opencrvs/opencrvs-core/issues/7728) From 7ddcb31c5754889804994949721f5c6b6bb216cd Mon Sep 17 00:00:00 2001 From: Niko Kurtti Date: Fri, 22 Nov 2024 15:41:36 +0200 Subject: [PATCH 02/14] Hotfix builds on publish to match lower env builds --- .github/workflows/publish-release.yml | 40 +++++++++++++++++++++------ 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 443566e7cd0..c40a17314f2 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -48,12 +48,29 @@ jobs: - name: Get list of services id: get-services run: | - services=$(grep "^ [^ ]" docker-compose.yml | grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//') + services=$(grep "^ [^ ]" docker-compose.yml | grep -v base| grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//') services_json=$(echo $services | tr '\n' ',' | sed 's/,$//' | jq -R 'split(" ")' | tr -d '\n') # Set the list of service names as an output variable echo "services=$services_json" >> $GITHUB_OUTPUT echo "services=$services_json" + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push base image + uses: docker/build-push-action@v6 + with: + file: packages/Dockerfile.base + context: . + push: true + tags: | + opencrvs/ocrvs-base:${{ steps.set-version.outputs.version }} + cache-from: type=registry,ref=opencrvs/ocrvs-base:${{ steps.set-version.outputs.version }} + cache-to: type=inline + outputs: services: ${{ steps.get-services.outputs.services }} version: ${{ steps.set-version.outputs.version }} @@ -61,6 +78,7 @@ jobs: build: needs: base strategy: + fail-fast: false matrix: service: ${{ fromJSON(needs.base.outputs.services) }} runs-on: ubuntu-22.04 @@ -74,17 +92,23 @@ jobs: if: github.event_name == 'push' - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build ${{ matrix.service }} - run: | - echo ${{ matrix.service }} - export VERSION=${{ github.event.inputs.release_version }} - docker compose build ${{ matrix.service }} - docker compose push ${{ matrix.service }} + - name: Build and push + uses: docker/build-push-action@v6 + with: + file: packages/${{ matrix.service }}/Dockerfile + build-args: | + VERSION=${{ needs.base.outputs.version }} + push: true + context: . + tags: | + opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }} + cache-from: type=registry,ref=opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version}} + cache-to: type=inline security-scans: needs: [base, build] From b2add4c404b6a3e33b767534eca30267bcff89d5 Mon Sep 17 00:00:00 2001 From: euanmillar Date: Fri, 29 Nov 2024 14:32:10 +0000 Subject: [PATCH 03/14] Fix spacing of Heading3 --- packages/components/src/Headings/Headings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/src/Headings/Headings.tsx b/packages/components/src/Headings/Headings.tsx index dd8454c5952..9104f2b2528 100644 --- a/packages/components/src/Headings/Headings.tsx +++ b/packages/components/src/Headings/Headings.tsx @@ -23,6 +23,6 @@ export const Heading2 = styled.h2` export const Heading3 = styled.h3` ${({ theme }) => theme.fonts.h3}; color: ${({ theme }) => theme.colors.grey600}; - padding: 8px 0px; + padding: 18px 0px 8px 0px; border-top: 1px solid ${({ theme }) => theme.colors.grey200}; ` From 905308a877003ea6ce7ce3b22e35a19a88135c2c Mon Sep 17 00:00:00 2001 From: tahmidrahman-dsi Date: Fri, 29 Nov 2024 14:53:31 +0600 Subject: [PATCH 04/14] chore: disable back button on a conditional expression - to support disabling navigation on fetch loading --- packages/client/src/views/RegisterForm/RegisterForm.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/client/src/views/RegisterForm/RegisterForm.tsx b/packages/client/src/views/RegisterForm/RegisterForm.tsx index 824440226a4..03f8768ddbf 100644 --- a/packages/client/src/views/RegisterForm/RegisterForm.tsx +++ b/packages/client/src/views/RegisterForm/RegisterForm.tsx @@ -1115,6 +1115,7 @@ class RegisterFormView extends React.Component { type="tertiary" size="small" onClick={this.props.goBack} + disabled={!canContinue} > {intl.formatMessage(buttonMessages.back)} From ef1308de2e429e449b64aa9541ca49fa80908c39 Mon Sep 17 00:00:00 2001 From: Tameem Bin Haider Date: Wed, 4 Dec 2024 16:45:58 +0600 Subject: [PATCH 05/14] docs: sync CHANGELOG with develop (#8153) --- CHANGELOG.md | 206 +++++++++++++++++++++++++-------------------------- 1 file changed, 99 insertions(+), 107 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d024d0e3419..8b2d038c8a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 1.6.1 (TBD) -## Bug fixes +### Bug fixes - Maximum upload file size limit is now based on the size of the uploaded files after compression and not before. [#7840](https://github.com/opencrvs/opencrvs-core/issues/7840) @@ -10,61 +10,14 @@ - Add an optional configurable field in section `canContinue` which takes an expression. Falsy value of this expression will disable the continue button in forms. This can be used to work with fetch field which has a loading state and prevent the user to get past the section while the request is still in progress. -## 1.6.0 - -## Improvements - -- Fetch child identifier in view record - -### New features - -- Certificate handlebar for registration fees `registrationFees` [#6817](https://github.com/opencrvs/opencrvs-core/issues/6817) -- Logged in user details handlebar `loggedInUser` [#6529](https://github.com/opencrvs/opencrvs-core/issues/6529) -- Supporting document fields can now be made required -- If there is only one option in the document uploader select, then it stays hidden and only the upload button is showed with the only option being selected by default -- Allow configuring the default search criteria for record search [#6924](https://github.com/opencrvs/opencrvs-core/issues/6924) -- Add checks to validate client and server are always on the same version. This prevents browsers with a cached or outdated client versions from making potentially invalid requests to the backend [#6695](https://github.com/opencrvs/opencrvs-core/issues/6695) -- Add http request creation ability to the form with a set of new form components (`HTTP`,`BUTTON`,`REDIRECT`) [#7489](https://github.com/opencrvs/opencrvs-core/issues/7489) - -- #### ElasticSearch reindexing - - Allows reindexing ElasticSearch via a new search-service endpoint `reindex`. We're replacing the original `ocrvs` index with timestamped ones. This is done automatically when upgrading and migrating, but this is an important architectural change that should be noted. More details in [#7033](https://github.com/opencrvs/opencrvs-core/pull/7033). - -- Introduce a new certificate handlebar "preview" which can be used to conditionally render some svg element when previewing the certificate e.g. background image similar to security paper - -### Improvements - -- Internally we were storing the `family` name field as a required property which was limiting what how you could capture the name of a person in the forms. Now we are storing it as an optional property which would make more flexible. -- Remove the leftover features from the application config pages, such as certificates and informant notification. [#7156](https://github.com/opencrvs/opencrvs-core/issues/7156) -- **PDF page size** The generated PDF used to be defaulted to A4 size. Now it respects the SVG dimensions if specified -- Support html content wrapped in `foreignObject` used in svg template in certificate PDF output - -## Bug fixes - -- Custom form field validators from country config will work offline. [#7478](https://github.com/opencrvs/opencrvs-core/issues/7478) -- Registrar had to retry from outbox every time they corrected a record. [#7583](https://github.com/opencrvs/opencrvs-core/issues/7583) -- Local environment setup command (`bash setup.sh`) could fail in machines that didn't have a unrelated `compose` binary. Fixed to check for Docker Compose. [#7609](https://github.com/opencrvs/opencrvs-core/pull/7609) -- Fix date validation not working correctly in Firefox [#7615](https://github.com/opencrvs/opencrvs-core/issues/7615) -- Fix layout issue that was causing the edit button on the AdvancedSearch's date range picker to not show on mobile view. [#7417](https://github.com/opencrvs/opencrvs-core/issues/7417) -- Fix hardcoded placeholder copy of input when saving a query in advanced search -- Handle label params used in form inputs when rendering in action details modal -- Fix health facilities missing from dropdown after correcting a record address [#7528](https://github.com/opencrvs/opencrvs-core/issues/7528) -- "Choose a new password" form now allows the user to submit the form using the "Enter/Return" key [#5502](https://github.com/opencrvs/opencrvs-core/issues/5502) -- Dropdown options now flow to multiple rows in forms [#7653](https://github.com/opencrvs/opencrvs-core/pull/7653) -- Only render units/postfix when field has a value [#7055](https://github.com/opencrvs/opencrvs-core/issues/7055) -- Only show items with values in review [#5192](https://github.com/opencrvs/opencrvs-core/pull/5192) -- Fix prefix text overlap issue in form text inputs -- **Staged files getting reset on precommit hook failure** We were running lint-staged separately on each package using lerna which potentially created a race condition causing staged changes to get lost on failure. Now we are running lint-staged directly without depending on lerna. **_This is purely a DX improvement without affecting any functionality of the system_** -- Fix `informantType` missing in template object which prevented rendering informant relationship data in the certificates [#5952](https://github.com/opencrvs/opencrvs-core/issues/5952) -- Fix users hitting rate limit when multiple users authenticated the same time with different usernames [#7728](https://github.com/opencrvs/opencrvs-core/issues/7728) +## [1.6.0](https://github.com/opencrvs/opencrvs-core/compare/v1.5.1...v1.6.0) ### Breaking changes - Remove informant notification configuration from the UI and read notification configuration settings from `record-notification` endpoint in countryconfig -- Remove `DEL /elasticIndex` endpoint due reindexing changes. +- Remove DEL /elasticIndex endpoint due to reindexing changes. - **Gateways searchEvents API updated** `operationHistories` only returns `operationType` & `operatedOn` due to the other fields being unused in OpenCRVS - **Config changes to review/preview and signatures** Core used to provide review/preview section by default which are now removed and need to be provided from countryconfig. The signature field definitions (e.g. informant signature, bride signature etc.) were hard coded in core which also have now been removed. The signatures can now be added through the review/preview sections defined in countryconfig just like any other field. You can use the following section definition as the default which is without any additional fields. We highly recommend checking out our reference country repository which has the signature fields in its review/preview sections -- `hasChildLocation` query has been removed from gateway. We have created the query `isLeafLevelLocation` instead which is more descriptive on its intended use. ``` { @@ -88,91 +41,136 @@ ] } ``` +- `hasChildLocation` query has been removed from gateway. We have created the query `isLeafLevelLocation` instead which is more descriptive on its intended use. ### New features - **Conditional filtering for document select options** The select options for the DOCUMENT_UPLOADER_WITH_OPTION field can now be conditionally filtered similar to the SELECT_WITH_OPTIONS field using the `optionCondition` field +- Supporting document fields can now be made required +- If there is only one option in the document uploader select, then it stays hidden and only the upload button is showed with the only option being selected by default +- A new certificate handlebar "preview" has been added which can be used to conditionally render some svg element when previewing the certificate e.g. background image similar to security paper +- Add HTTP request creation ability to the form with a set of new form components (HTTP, BUTTON, REDIRECT) [#7489](https://github.com/opencrvs/opencrvs-core/issues/7489) -## Bug fixes +### Improvements +- **ElasticSearch reindexing** Allows reindexing ElasticSearch via a new search-service endpoint `reindex`. We're replacing the original `ocrvs` index with timestamped ones. This is done automatically when upgrading and migrating, but this is an important architectural change that should be noted. More details in [#7033](https://github.com/opencrvs/opencrvs-core/pull/7033). +- Internally we were storing the `family` name field as a required property which was limiting what how you could capture the name of a person in the forms. Now we are storing it as an optional property which would make more flexible. +- Remove the leftover features from the application config pages, such as certificates and informant notification. [#7156](https://github.com/opencrvs/opencrvs-core/issues/7156) +- **PDF page size** The generated PDF used to be defaulted to A4 size. Now it respects the SVG dimensions if specified +- Support html content wrapped in `foreignObject` used in svg template in certificate PDF output + +### Bug fixes + +- Custom form field validators from country config will work offline. [#7478](https://github.com/opencrvs/opencrvs-core/issues/7478) +- Registrar had to retry from outbox every time they corrected a record. [#7583](https://github.com/opencrvs/opencrvs-core/issues/7583) +- Local environment setup command (`bash setup.sh`) could fail in machines that didn't have a unrelated `compose` binary. Fixed to check for Docker Compose. [#7609](https://github.com/opencrvs/opencrvs-core/pull/7609) - Fix wrong status shown in the Comparison View page of the duplicate record [#7439](https://github.com/opencrvs/opencrvs-core/issues/7439) +- Fix date validation not working correctly in Firefox [#7615](https://github.com/opencrvs/opencrvs-core/issues/7615) +- Fix layout issue that was causing the edit button on the AdvancedSearch's date range picker to not show on mobile view. [#7417](https://github.com/opencrvs/opencrvs-core/issues/7417) +- Fix hardcoded placeholder copy of input when saving a query in advanced search +- Handle label params used in form inputs when rendering in action details modal +- **Staged files getting reset on precommit hook failure** We were running lint-staged separately on each package using lerna which potentially created a race condition causing staged changes to get lost on failure. Now we are running lint-staged directly without depending on lerna. **_This is purely a DX improvement without affecting any functionality of the system_** +- Fix `informantType` missing in template object which prevented rendering informant relationship data in the certificates [#5952](https://github.com/opencrvs/opencrvs-core/issues/5952) +- Fix users hitting rate limit when multiple users authenticated the same time with different usernames [#7728](https://github.com/opencrvs/opencrvs-core/issues/7728) ## 1.5.1 -## Improvements +### Improvements - Fetch child identifier in view record - Home screen application’s name and icons are to be configured from country configuration package as manifest.json and app icon files are moved from core to country config (check `opencrvs-countryconfig/src/client-static` folder) -## Bug fixes +### Bug fixes - On slow connections or in rare corner cases, it was possible that the same record got saved to the database twice. This was caused by a bug in how the unique technical identifier we generate were stored as FHIR. The backend now ensures every record is submitted only once. [#7477](https://github.com/opencrvs/opencrvs-core/issues/7477) - Fixed an issue where address line fields (e.g., address line 1, address line 2, etc.) were not being updated correctly when a user attempted to update a record's event location, such as place of birth or place of death. [#7531](https://github.com/opencrvs/opencrvs-core/issues/7531) +- Handle label params used in form inputs when rendering in review section view +- Fix probable migration issues for countries migrating from 1.2 [#7464](https://github.com/opencrvs/opencrvs-core/issues/7464) - When a declaration(birth/death) is created the event location information was not being parsed to ElasticSearch which caused the Advanced search feature to not work when searching for records by event location.[7494](https://github.com/opencrvs/opencrvs-core/issues/7494) - When any user's role was updated, incorrect role was shown for that user's actions in the history section of a declaration's record audit page. [#7495](https://github.com/opencrvs/opencrvs-core/issues/7495) - Registration agent was unable to download declarations that were previously corrected by registrar. [#7582](https://github.com/opencrvs/opencrvs-core/issues/7582) -- The internal function we used to check if all the location references listed in the encounter are included in the bundle had incorrect logic which resulted in location details missing in ElasticSearch which broke Advanced search. [7494](https://github.com/opencrvs/opencrvs-core/issues/7494) - When a user updates a marriage declaration editing the signature of the bride, groom, witness one or witness two, handle the changed value of the signature properly. [#7462](https://github.com/opencrvs/opencrvs-core/issues/7462) +- Registration agent was unable to download declarations that were previously corrected by registrar. [#7582](https://github.com/opencrvs/opencrvs-core/issues/7582) +- The internal function we used to check if all the location references listed in the encounter are included in the bundle had incorrect logic which resulted in location details missing in ElasticSearch which broke Advanced search. [7494](https://github.com/opencrvs/opencrvs-core/issues/7494) -## 1.5.0 (TBD) - -### New features +## [1.5.0](https://github.com/opencrvs/opencrvs-core/compare/v1.4.1...v1.5.0) -- Certificate handlebar for registration fees `registrationFees` [#6817](https://github.com/opencrvs/opencrvs-core/issues/6817) -- Logged in user details handlebar `loggedInUser` [#6529](https://github.com/opencrvs/opencrvs-core/issues/6529) -- Supporting document fields can now be made required -- If there is only one option in the document uploader select, then it stays hidden and only the upload button is showed with the only option being selected by default +### Breaking changes -* **ElasticSearch reindexing** +- **Removed dependency on OpenHIM** -Allows reindexing ElasticSearch via a new search-service endpoint `reindex`. We're replacing the original `ocrvs` index with timestamped ones. This is done automatically when upgrading and migrating, but this is an important architectural change that should be noted. More details in [#7033](https://github.com/opencrvs/opencrvs-core/pull/7033). + The performance of OpenHIM added an unexpected burden of 200 m/s to every interaction. Cumulatively, this was negatively affecting user experience and therefore we decided to deprecate it. -- Introduce a new certificate handlebar "preview" which can be used to conditionally render some svg element when previewing the certificate e.g. background image similar to security paper + Interested implementers are free to re-introduce OpenHIM should they wish to use it as an interoperability layer without affecting the performance of OpenCRVS now that our architecture no longer depends on it. -## Breaking changes + The OpenHIM database is kept for backwards compatibility reasons and will be removed in v1.6. [OpenHIM](https://openhim.org/) is an Open Source middleware component designed for managing FHIR interoperability between disparate systems as part of the OpenHIE architectural specification. We had been using this component in a much more fundamental way to monitor microservice comms in a similar fashion to Amazon SQS. -- #### Upgrade node version to 18 +- **Upgrade node version to 18** This version enforces environment to have Node 18 installed (supported until April 2025) and removes support for Node 16 - - Supports node version `18.19.x` + - Use nvm to upgrade your local development environment to use node version `18.19.x.` - Specified operating systems in js modules as `darwin, linux` - Dev scripts and Vite run with an environment variable `NODE_OPTIONS=--dns-result-order=ipv4first` to resolve ipv4 addresses for `localhost` to support systems that resolves ipv6 addresses by default in Node versions >=17 -- #### Update the certificate preview mechanism - - In effort of minimizing JavaScript-bundle size, we have streamlined the way how review certificate -page renders certificates. In case the images in your certificates are previewing blurry, you need to update your SVG-certificates to print QR-codes and other images directly with `` instead of the more complicated `` -paradigm. This doesn't affect printed certificates as they are still created as previously. - -- #### Remove unused GraphQL resolvers locationById and locationsByParent - -- #### Remove unused GraphQL type `user.catchmentArea` in favor of `user.primaryOffice` +- **Update the certificate preview mechanism** In effort of minimizing JavaScript-bundle size, we have streamlined the way how review certificate -page renders certificates. In case the images in your certificates are previewing blurry, you need to update your SVG-certificates to print QR-codes and other images directly with `` instead of the more complicated `` -paradigm. This doesn't affect printed certificates as they are still created as previously. +- **Generate default address according to logged-in user's location** We have dropped support for the 'agentDefault' prop which was used as initial value for SELECT_WITH_DYNAMIC_OPTIONS fields. If you have not made any changes to address generation, then this should not affect you. If you have, you can refer to this PR to see how agentDefault has been deprecated in an example country: [https://github.com/opencrvs/opencrvs-farajaland/pull/978](https://github.com/opencrvs/opencrvs-farajaland/pull/978) +- **Remove system admin UI items: Application, User roles** We have now moved to configuring these items away from the UI in favour of directly editing these from country configuration repository in code - specifically in application-config-default.ts. +- **Set Metabase default credentials.** These must be configured via countryconfig repository environment variables and secrets otherwise the dashboard service won't start: OPENCRVS_METABASE_ADMIN_EMAIL & OPENCRVS_METABASE_ADMIN_PASSWORD +- **Check your Metabase map file.** For Metabase configuration, we renamed `farajaland-map.geojson` to `map.geojson` to not tie implementations into example country naming conventions. +- **Feature flags** In order to make application config settings more readable, we re-organised `src/api/application/application-config-default.ts` with a clear feature flag block like so. These are then used across the front and back end of the application to control configurable functionality. New feature flags DEATH_REGISTRATION allow you to optionally run off death registration if your country doesnt want to run its first pilot including death and PRINT_DECLARATION (see New Features) have been added. + `FEATURES: { + DEATH_REGISTRATION: true, + MARRIAGE_REGISTRATION: false, + ... +} ` +- **Improve rendering of addresses in review page where addresses match** When entering father's address details, some countries make use of a checkbox which says "Address is the same as the mothers. " which, when selected, makes the mother's address and fathers address the same. The checkbox has a programatic value of "Yes" or "No". As a result on the review page, the value "Yes" was displayed which didn't make grammatical sense as a response. We decided to use a custom label: "Same as mother's", which is what was asked on the form. This requires some code changes in the src/form/addresses/index.ts file to pull in the `hideInPreview` prop which will hide the value "Yes" on the review page and replace with a content managed label. Associated bug [#5086](https://github.com/opencrvs/opencrvs-core/issues/5086) + +### Infrastructure breaking changes + +More improvements have been made to the infrastructure provisioning and Github environment creation scripts and documentation. The complexity is somewhat reduced. + +- **We removed the example Wireguard VPN set up as it was confusing.** Our intention was to ensure that all implementers were aware that OpenCRVS should be installed behind a VPN and used Wireguard as an example. But the configuration requirements for Wireguard confused implementers who are not using it. Therefore we decided to remove Wireguard as an example. +- **We now have a "backup" Github environment and the backup server is automatically provisioned.** We moved the inventory file location to an explicit directory and removed parameters to scripts that can be automated. To migrate, move all inventory files (qa.yml, production.yml, staging.yml from `infrastructure/server-setup` to `infrastructure/server-setup/inventory` and configure `infrastructure/server-setup/inventory/backup.yml`. Run environment creator for your backup server `yarn environment:init --environment=backup` +- **You can configure the file path on the backup server where backups are stored.** We can also allow using staging to both periodically restore a production backup and also give it the capability if required to backup it's own data to a different location using `backup_server_remote_target_directory` and `backup_server_remote_source_directory` Ansible variables. This use case is mostly meant for OpenCRVS team internal use. +- **We now automate SSH key exchange between application and backup server.** For staging servers, automatically fetch production backup encryption key if periodic restore is enabled using `ansible_ssh_private_key_file` Ansible variables. Therefore documentation is simplified for a new server set-up. +- **In infrastructure Github workflows: SSH_PORT is new and required allowing you the ability to use a non-standard SSH port.** This Github Action environment variable must be added. +- **In infrastructure Github workflows: SSH_HOST** should be moved from being a Github Action environment secret to a Github Action environment variable before it is deprecated in 1.7.0 +- **No longer an assumption made that production server Docker replicas and Mongo replica-sets are necessary.** In our Docker Compose files, we had originally assumed that a production deployment would always be deployed on a cluster to enable load balancing. We applied a [Mongo replica set](https://github.com/opencrvs/opencrvs-countryconfig/blob/48cf278bab9d17e07b60b427294a26c8f35bcc1b/infrastructure/docker-compose.production-deploy.yml#L170C3-L201C19) by default on production and set [replicas: 2](https://github.com/opencrvs/opencrvs-countryconfig/blob/48cf278bab9d17e07b60b427294a26c8f35bcc1b/infrastructure/docker-compose.production-deploy.yml#L124) on each microservice. However after experience in multiple countries running small scale pilots, a production deployment usually starts off as 1 server node and then scales into a cluster over time in order to save costs and resources. Therefore these replicas are a waste of resources. So you will notice that this has been deleted. You can always manually add your desired replicas back into you Docker Compose configuration if you want. In Docker Compose files, search for REPLICAS and update accordingly as well as attending to the linked examples. + +Follow the descriptions in the migration notes to re-provision all servers safely. -- #### Move default address generation to country-config - - We are dropping support for 'agentDefault' to be used as initial value for SELECT_WITH_DYNAMIC_OPTIONS type of fields. The country config package now must return the form with prepopulated initial values to show default addresses. [#6871](https://github.com/opencrvs/opencrvs-core/issues/6871) - -- #### Remove system admin UI items: Application, Certificates, User roles, Informant notifications - - We have now moved to configuring these items directly from country configuration repository. - -## New features +### New features -- Add loading spinners before JavaScript bundle has loaded for both login and client -- Add loading bar before javaScript bundle has loaded for client and when fetching records [#6641](https://github.com/opencrvs/opencrvs-core/issues/6641) -- Support for landscape certificate templates -- Allow defining maxLength attribute for number type fields [#6356](https://github.com/opencrvs/opencrvs-core/issues/6356) -- Introduce two new properties to the form field `DOCUMENT_UPLOADER_WITH_OPTION` +- Introduced rate limiting to routes that could potentially be bruteforced or extracted PII from. +- The login and client application loading experience has improved. A loading bar appears before the javaScript bundle has loaded and this transitions when fetching records. +- Development time logs are now much tidier and errors easier to point out. Production logging will still remain as is. +- Masked emails and phone numbers from notification logs. +- Support for landscape certificate templates. +- Allow defining maxLength attribute for number type fields. +- A new certificate handlebar for registration fees has been added `registrationFees` +- A new certificate handlebar for logged-in user details has been added `loggedInUser` +- Add support for image compression configuration. Two new properties to this form field are available: `DOCUMENT_UPLOADER_WITH_OPTION` - `compressImagesToSizeMB` : An optional prop of number type to define a compressed size. Compression is ignored when the input file is already smaller or equal of the given value or a falsy given value. - `maxSizeMB`: An optional validation prop to prevent input of a file bigger than a defined value. -- Metabase default credentials now must be configured via countryconfig repository environment variables and secrets otherwise the dashboard service won't start [#6578](https://github.com/opencrvs/opencrvs-core/issues/6578) -- Introduce rate limiting to routes that could potentially be bruteforced or extracted PII from [#6145](https://github.com/opencrvs/opencrvs-core/pull/6145) +- If a country doesnt wish to use Sentry for logging errors, the SENTRY_DSN variable is now optional and the LogRocket option has been deprecated due to lack of demand. +- Given that upon an upgrade between versions of OpenCRVS, that users cache is cleared, it is important to inform staff to submit any draft applications before the upgrade date. We introduced an "Email all users" feature so that National System Admins can send all staff messages. This feature can be used for any other all staff comms that are deemed required. -## Improvements +
-- Development time logs are now much tidier and errors easier to point out. Production logging will still remain as is. [#7022](https://github.com/opencrvs/opencrvs-core/pull/7022) -- Mask emails and phone numbers from notification logs [#7204](https://github.com/opencrvs/opencrvs-core/pull/7204) +- Included an endpoint for serving individual certificates in development mode. This improves the developer experience when configuring certificates. +- Removed logrocket refrences. +- Enable gzip compression in client & login +- Use docker compose v2 in github workflows +- Added SMTP environment variables into the qa compose file to enable QA of SMTP servers. +- In the certificate, the 'Place of Certification' now accurately reflects the correct location. +- Groom's and Bride's name, printIssue translation variables updated [#124](https://github.com/opencrvs/opencrvs-countryconfig/pull/124) +- Add query mapper for International Postal Code field +- Provide env variables for metabase admin credentials +- Improved formatting of informant name for inProgress declaration emails +- There is now an option to print the review page of an event declaration form. The PRINT_DECLARATION feature flag in application config settings can enable this on or off. -## Bug fixes +### Bug fixes - Handle back button click after issuing a declaration [#6424](https://github.com/opencrvs/opencrvs-core/issues/6424) - Fix certificate verification QR code for a death declaration [#6230](https://github.com/opencrvs/opencrvs-core/issues/6230#issuecomment-1996766125) @@ -220,14 +218,9 @@ Allows reindexing ElasticSearch via a new search-service endpoint `reindex`. We' - Fix email of practitioner to be saved in hearth. A migration is added to correct the email of practitoiner in existing db. [7315](https://github.com/opencrvs/opencrvs-core/pull/7315) - Fix inaccessible and only partly visible "Edit" button in "Advanced Search" - feature's date range list [7485](https://github.com/opencrvs/opencrvs-core/pull/7485) -## Refactor - -- Remove dependency on openhim. The openhim db is kept for backwards compatibility reasons and will be removed in v1.6. It has brought some major changes - in how the microservices are communicating among them. More on this can be found on the updated [sequence diagrams](https://github.com/opencrvs/opencrvs-core/tree/develop/sequence-diagrams/backend) - ## [1.3.4](https://github.com/opencrvs/opencrvs-core/compare/v1.3.3...v1.3.4) -## Bug fixes +### Bug fixes - #### Include middlename when generating fullnames - Refactored out the scattered logic for generating fullnames and converged them into a single function @@ -239,7 +232,7 @@ Allows reindexing ElasticSearch via a new search-service endpoint `reindex`. We' - #### Fix system crash when opening the started action modal [#6551](https://github.com/opencrvs/opencrvs-core/issues/6551) - #### Convert eventDates to LocalDate before formatting [#6719](https://github.com/opencrvs/opencrvs-core/issues/6719) -## [1.4.1](https://github.com/opencrvs/opencrvs-core/compare/v1.3.3...v1.4.1) +## [1.4.1](https://github.com/opencrvs/opencrvs-core/compare/v1.4.0...v1.4.1) - Fix Metabase versions in Dashboards service. Previously the version used for local development wasn't the one built into the docker image, which caused the locally generated initialisation file to fail in deployed environments. - Fix a seeding script bug, where it failed when done too quickly [#6553](https://github.com/opencrvs/opencrvs-core/issues/6553) @@ -268,18 +261,17 @@ In the next OpenCRVS release v1.5.0, there will be two significant changes both ## [1.3.3](https://github.com/opencrvs/opencrvs-core/compare/v1.3.2...v1.3.3) -## Breaking changes +### New features -## New features +- **New handlebars serving the location ids of the admin level locations** -- #### New handlebars serving the location ids of the admin level locations Apart from the new handlebars, a couple more improvements were introduced: - stricter type for locations in client - **"location"** handlebar helper can now resolve offices & facilities - restrict the properties exposed through **"location"** handlebar helper - remove deprecated **DIVISION** & **UNION** from client -## Bug fixes +### Bug fixes - #### Fix location seeding scripts throwing error when there are too many source locations from the country config Locations are now seeded in smaller segments instead of one big collection. The newer approach has improved performance to a significant extent and also clears the interruption caused for a large number of country config locations @@ -287,8 +279,8 @@ In the next OpenCRVS release v1.5.0, there will be two significant changes both - Core not recognizing "occupation" as an optional field for deceased - Unassign declaration from a user if the declaration has already been proceeded through the workqueues by a separate user -## Dependency upgrades +### Dependency upgrades -- #### Metabase from v0.45.2.1 to v0.46.6.1 +- **Metabase from v0.45.2.1 to v0.46.6.1** See [Releases](https://github.com/opencrvs/opencrvs-core/releases) for release notes of older releases. From f684e121365b4127761b1141d74867c8726f6251 Mon Sep 17 00:00:00 2001 From: Siyasanga Mtshokotsha Date: Thu, 5 Dec 2024 12:10:15 +0200 Subject: [PATCH 06/14] fix: Stops local sys admin from creating national level staff (#8112) * Move access mgnt into the gateway service It is better to have in the gateway since most of access mgnt is handled there already https://github.com/opencrvs/opencrvs-core/issues/7698 * refactor: the getSystemRoles() to propery use filters The way we were building the criteria object was buggy especially for when we are filtering based on user roles https://github.com/opencrvs/opencrvs-core/issues/7698 * Filter User roles based on user that's requesting This is avoid users with lower roles creating or updating other users with higher roles https://github.com/opencrvs/opencrvs-core/issues/7698 * Record changes in the CHANGELOG https://github.com/opencrvs/opencrvs-core/issues/7698 * Revert "Filter User roles based on user that's requesting" This reverts commit b46c67ea974e94fc4956df973cdd9b6974d3d4a9. * Revert "refactor: the getSystemRoles() to propery use filters" This reverts commit fb400bd1141363f06c9b942157f2a6ba5d2f6dc5. * Revert "Move access mgnt into the gateway service" This reverts commit a9c6fa8e44ded436f8545f6d8dd9ffe55adeda9a. * Fix failing Role feature's resolver tests https://github.com/opencrvs/opencrvs-core/issues/7698 * Stop sys admins from de-activating themselves The sys admin will no longer see the feature for their own accounts, it will only available on other users, this should stop them from eccidentally deactivating their accounts. https://github.com/opencrvs/opencrvs-core/issues/7691 * Minor tisy up --------- Co-authored-by: euanmillar --- CHANGELOG.md | 1 + .../src/views/SysAdmin/Team/user/UserList.tsx | 13 ++-- .../client/src/views/SysAdmin/Team/utils.ts | 5 ++ .../client/src/views/UserAudit/UserAudit.tsx | 11 +++- .../src/features/role/root-resolvers.test.ts | 31 ++++++++- .../src/features/role/root-resolvers.ts | 16 ++++- packages/gateway/src/features/role/utils.ts | 65 +++++++++++++++++++ .../src/features/user/root-resolvers.ts | 12 +++- .../gateway/src/features/user/utils/index.ts | 18 +++++ 9 files changed, 160 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b2d038c8a8..d77877e1af1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Bug fixes - Maximum upload file size limit is now based on the size of the uploaded files after compression and not before. [#7840](https://github.com/opencrvs/opencrvs-core/issues/7840) +- Stops local sys admins creating national level users. [#7698](https://github.com/opencrvs/opencrvs-core/issues/7698) ### New features diff --git a/packages/client/src/views/SysAdmin/Team/user/UserList.tsx b/packages/client/src/views/SysAdmin/Team/user/UserList.tsx index 34a7659c9ab..e48985f71b1 100644 --- a/packages/client/src/views/SysAdmin/Team/user/UserList.tsx +++ b/packages/client/src/views/SysAdmin/Team/user/UserList.tsx @@ -39,7 +39,8 @@ import { SysAdminContentWrapper } from '@client/views/SysAdmin/SysAdminContentWr import { getAddressName, getUserRoleIntlKey, - UserStatus + UserStatus, + canDeactivateUser } from '@client/views/SysAdmin/Team/utils' import { LinkButton } from '@opencrvs/components/lib/buttons' import { Button } from '@opencrvs/components/lib/Button' @@ -396,7 +397,7 @@ function UserListComponent(props: IProps) { ) const getMenuItems = useCallback( - function getMenuItems(user: User) { + function getMenuItems(user: User, userDetails: UserDetails | null) { const menuItems = [ { label: intl.formatMessage(messages.editUserDetailsTitle), @@ -432,7 +433,11 @@ function UserListComponent(props: IProps) { }) } - if (user.status === 'active') { + if ( + userDetails && + user.status === 'active' && + canDeactivateUser(user.id, userDetails) + ) { menuItems.push({ label: intl.formatMessage(messages.deactivate), handler: () => toggleUserActivationModal(user) @@ -530,7 +535,7 @@ function UserListComponent(props: IProps) { toggleButton={ } - menuItems={getMenuItems(user)} + menuItems={getMenuItems(user, userDetails)} /> )} diff --git a/packages/client/src/views/SysAdmin/Team/utils.ts b/packages/client/src/views/SysAdmin/Team/utils.ts index 19f647bb920..c02e5ac0556 100644 --- a/packages/client/src/views/SysAdmin/Team/utils.ts +++ b/packages/client/src/views/SysAdmin/Team/utils.ts @@ -13,6 +13,7 @@ import { IntlShape, MessageDescriptor } from 'react-intl' import { messages } from '@client/i18n/messages/views/userSetup' import { SystemRoleType } from '@client/utils/gateway' import { ILocation, IOfflineData } from '@client/offline/reducer' +import { UserDetails } from '@client/utils/userUtils' export enum UserStatus { ACTIVE, @@ -112,3 +113,7 @@ export function getUserSystemRole( export const getUserRoleIntlKey = (_roleId: string) => { return `role.${_roleId}` } + +export const canDeactivateUser = (id: string, userDetails: UserDetails) => { + return id !== userDetails.id ? true : false +} diff --git a/packages/client/src/views/UserAudit/UserAudit.tsx b/packages/client/src/views/UserAudit/UserAudit.tsx index 680ed4cec5d..fcf6fb32d8c 100644 --- a/packages/client/src/views/UserAudit/UserAudit.tsx +++ b/packages/client/src/views/UserAudit/UserAudit.tsx @@ -23,7 +23,10 @@ import { AvatarSmall } from '@client/components/Avatar' import styled from 'styled-components' import { ToggleMenu } from '@opencrvs/components/lib/ToggleMenu' import { Button } from '@opencrvs/components/lib/Button' -import { getUserRoleIntlKey } from '@client/views/SysAdmin//Team/utils' +import { + getUserRoleIntlKey, + canDeactivateUser +} from '@client/views/SysAdmin/Team/utils' import { EMPTY_STRING, LANG_EN } from '@client/utils/constants' import { Loader } from '@opencrvs/components/lib/Loader' import { messages as userSetupMessages } from '@client/i18n/messages/views/userSetup' @@ -246,7 +249,11 @@ export const UserAudit = () => { ) } - if (status === 'active') { + if ( + status === 'active' && + userDetails && + canDeactivateUser(userId, userDetails) + ) { menuItems.push({ label: intl.formatMessage(sysMessages.deactivate), handler: () => toggleUserActivationModal() diff --git a/packages/gateway/src/features/role/root-resolvers.test.ts b/packages/gateway/src/features/role/root-resolvers.test.ts index fec0be47d08..c4de70bd8e7 100644 --- a/packages/gateway/src/features/role/root-resolvers.test.ts +++ b/packages/gateway/src/features/role/root-resolvers.test.ts @@ -202,17 +202,42 @@ describe('Role root resolvers', () => { } ] it('returns full role list', async () => { + const sysAdminToken = jwt.sign( + { scope: ['natlsysadmin'] }, + readFileSync('./test/cert.key'), + { + subject: 'ba7022f0ff4822', + algorithm: 'RS256', + issuer: 'opencrvs:auth-service', + audience: 'opencrvs:gateway-user' + } + ) + const authHeaderSysAdmin = { + Authorization: `Bearer ${sysAdminToken}` + } fetch.mockResponseOnce(JSON.stringify(dummyRoleList)) - const response = await resolvers.Query!.getSystemRoles( {}, {}, - { headers: undefined } + { headers: authHeaderSysAdmin } ) expect(response).toEqual(dummyRoleList) }) it('returns filtered role list', async () => { + const sysAdminToken = jwt.sign( + { scope: ['sysadmin'] }, + readFileSync('./test/cert.key'), + { + subject: 'ba7022f0ff4822', + algorithm: 'RS256', + issuer: 'opencrvs:auth-service', + audience: 'opencrvs:gateway-user' + } + ) + const authHeaderSysAdmin = { + Authorization: `Bearer ${sysAdminToken}` + } fetch.mockResponseOnce(JSON.stringify([dummyRoleList[2]])) const response = await resolvers.Query!.getSystemRoles( @@ -225,7 +250,7 @@ describe('Role root resolvers', () => { type: 'Mayor', active: true }, - { headers: undefined } + { headers: authHeaderSysAdmin } ) expect(response).toEqual([dummyRoleList[2]]) }) diff --git a/packages/gateway/src/features/role/root-resolvers.ts b/packages/gateway/src/features/role/root-resolvers.ts index efe34779f16..b4d3152d690 100644 --- a/packages/gateway/src/features/role/root-resolvers.ts +++ b/packages/gateway/src/features/role/root-resolvers.ts @@ -12,8 +12,13 @@ import { GQLResolver } from '@gateway/graphql/schema' import fetch from '@gateway/fetch' import { USER_MANAGEMENT_URL } from '@gateway/constants' import { IRoleSearchPayload } from '@gateway/features/role/type-resolvers' -import { transformMongoComparisonObject } from '@gateway/features/role/utils' +import { + getAccessibleRolesForScope, + SystemRole, + transformMongoComparisonObject +} from '@gateway/features/role/utils' import { hasScope } from '@gateway/features/user/utils' +import { getTokenPayload } from '@opencrvs/commons/authentication' export const resolvers: GQLResolver = { Query: { @@ -51,6 +56,7 @@ export const resolvers: GQLResolver = { if (active !== null) { payload = { ...payload, active } } + const res = await fetch(`${USER_MANAGEMENT_URL}getSystemRoles`, { method: 'POST', body: JSON.stringify(payload), @@ -59,7 +65,13 @@ export const resolvers: GQLResolver = { ...authHeader } }) - return await res.json() + + const { scope } = getTokenPayload(authHeader.Authorization.split(' ')[1]) + const accessibleSysAdminRoles = getAccessibleRolesForScope(scope) + const allSysAdminRoles = (await res.json()) as SystemRole[] + return allSysAdminRoles.filter((sysAdminRole) => + accessibleSysAdminRoles?.includes(sysAdminRole.value) + ) } }, Mutation: { diff --git a/packages/gateway/src/features/role/utils.ts b/packages/gateway/src/features/role/utils.ts index 7a9448f3af3..e3cec0da12c 100644 --- a/packages/gateway/src/features/role/utils.ts +++ b/packages/gateway/src/features/role/utils.ts @@ -8,6 +8,58 @@ * * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ + +import { Scope } from '@opencrvs/commons/authentication' + +export const SYSTEM_ROLE_KEYS = [ + 'FIELD_AGENT', + 'LOCAL_REGISTRAR', + 'LOCAL_SYSTEM_ADMIN', + 'NATIONAL_REGISTRAR', + 'NATIONAL_SYSTEM_ADMIN', + 'PERFORMANCE_MANAGEMENT', + 'REGISTRATION_AGENT' +] as const + +// Derive the type from SYSTEM_ROLE_KEYS +type SystemRoleKeyType = (typeof SYSTEM_ROLE_KEYS)[number] + +export const SysAdminAccessMap: Partial< + Record +> = { + LOCAL_SYSTEM_ADMIN: [ + 'FIELD_AGENT', + 'LOCAL_REGISTRAR', + 'LOCAL_SYSTEM_ADMIN', + 'PERFORMANCE_MANAGEMENT', + 'REGISTRATION_AGENT' + ], + NATIONAL_SYSTEM_ADMIN: [ + 'FIELD_AGENT', + 'LOCAL_REGISTRAR', + 'LOCAL_SYSTEM_ADMIN', + 'NATIONAL_REGISTRAR', + 'NATIONAL_SYSTEM_ADMIN', + 'PERFORMANCE_MANAGEMENT', + 'REGISTRATION_AGENT' + ] +} + +type UserRole = { + labels: Label[] +} + +type Label = { + lang: string + label: string +} + +export type SystemRole = { + value: SystemRoleKeyType + roles: UserRole[] + active: boolean + creationDate: number +} export interface IComparisonObject { eq?: string gt?: string @@ -46,3 +98,16 @@ export function transformMongoComparisonObject( {} ) } + +export function getAccessibleRolesForScope(scope: Scope[]) { + let roleFilter: keyof typeof SysAdminAccessMap + if (scope.includes('natlsysadmin')) { + roleFilter = 'NATIONAL_SYSTEM_ADMIN' + } else if (scope.includes('sysadmin')) { + roleFilter = 'LOCAL_SYSTEM_ADMIN' + } else { + throw Error('Create user is only allowed for sysadmin/natlsysadmin') + } + + return SysAdminAccessMap[roleFilter] +} diff --git a/packages/gateway/src/features/user/root-resolvers.ts b/packages/gateway/src/features/user/root-resolvers.ts index 08497c6b00a..1b38d9eadf0 100644 --- a/packages/gateway/src/features/user/root-resolvers.ts +++ b/packages/gateway/src/features/user/root-resolvers.ts @@ -20,7 +20,8 @@ import { hasScope, inScope, isTokenOwner, - getUserId + getUserId, + canAssignRole } from '@gateway/features/user/utils' import { GQLHumanNameInput, @@ -37,6 +38,7 @@ import { validateAttachments } from '@gateway/utils/validators' import { postMetrics } from '@gateway/features/metrics/service' import { uploadBase64ToMinio } from '@gateway/features/documents/service' import { rateLimitedResolver } from '@gateway/rate-limit' +import { getTokenPayload } from '@opencrvs/commons/authentication' export const resolvers: GQLResolver = { Query: { @@ -272,6 +274,14 @@ export const resolvers: GQLResolver = { ) } + const { scope: loggedInUserScope } = getTokenPayload( + authHeader.Authorization.split(' ')[1] + ) + + if (!canAssignRole(loggedInUserScope, user)) { + throw Error('Create user is only allowed for sysadmin/natlsysadmin') + } + try { if (user.signature) { await validateAttachments([user.signature]) diff --git a/packages/gateway/src/features/user/utils/index.ts b/packages/gateway/src/features/user/utils/index.ts index 9618fe3c29a..bbe9e1c70ca 100644 --- a/packages/gateway/src/features/user/utils/index.ts +++ b/packages/gateway/src/features/user/utils/index.ts @@ -17,6 +17,8 @@ import { import * as decode from 'jwt-decode' import fetch from '@gateway/fetch' import { Scope } from '@opencrvs/commons/authentication' +import { GQLUserInput } from '@gateway/graphql/schema' +import { SysAdminAccessMap } from '@gateway/features/role/utils' export interface ITokenPayload { sub: string @@ -49,6 +51,22 @@ export async function getUser( return await res.json() } +export function canAssignRole( + loggedInUserScope: Scope[], + userToSave: GQLUserInput +) { + let roleFilter: keyof typeof SysAdminAccessMap + if (loggedInUserScope.includes('natlsysadmin')) { + roleFilter = 'NATIONAL_SYSTEM_ADMIN' + } else if (loggedInUserScope.includes('sysadmin')) { + roleFilter = 'LOCAL_SYSTEM_ADMIN' + } else { + throw Error('Create user is only allowed for sysadmin/natlsysadmin') + } + + return SysAdminAccessMap[roleFilter]?.includes(userToSave.systemRole) +} + export async function getSystem( body: { [key: string]: string | undefined }, authHeader: IAuthHeader From 4685dd1af0820d4acb1d4479f1c19aeaa6c5042c Mon Sep 17 00:00:00 2001 From: euanmillar Date: Tue, 19 Nov 2024 22:08:18 +0000 Subject: [PATCH 07/14] Fix conflict in yarn lock --- .../src/features/uploadDocument/handler.ts | 37 +++++++++++++++---- yarn.lock | 6 +-- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/packages/documents/src/features/uploadDocument/handler.ts b/packages/documents/src/features/uploadDocument/handler.ts index fc860f6021f..a0028fb0689 100644 --- a/packages/documents/src/features/uploadDocument/handler.ts +++ b/packages/documents/src/features/uploadDocument/handler.ts @@ -31,15 +31,36 @@ export async function documentUploadHandler( const payload = request.payload as IDocumentPayload const ref = uuid() try { - const base64String = payload.fileData.split(',')[1] - const base64Decoded = Buffer.from(base64String, 'base64') - const fileType = (await fromBuffer(base64Decoded)) as IFileInfo - const generateFileName = `${ref}.${fileType.ext}` + // payload is an image + const regex = /^data:image/ + let generateFileName + if (regex.test(payload.fileData)) { + const base64String = payload.fileData.split(',')[1] + const base64Decoded = Buffer.from(base64String, 'base64') + const fileType = (await fromBuffer(base64Decoded)) as IFileInfo + generateFileName = `${ref}.${fileType.ext}` - await minioClient.putObject(MINIO_BUCKET, generateFileName, base64Decoded, { - 'content-type': fileType.mime, - ...payload.metaData - }) + await minioClient.putObject( + MINIO_BUCKET, + generateFileName, + base64Decoded, + { + 'content-type': fileType.mime, + ...payload.metaData + } + ) + } else { + // payload is an svg cert + const generateFileName = `${ref}.CERTIFICATE` + await minioClient.putObject( + MINIO_BUCKET, + generateFileName, + payload.fileData, + { + 'content-type': 'image/svg+xml' + } + ) + } return h .response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) diff --git a/yarn.lock b/yarn.lock index 071f9c97bc7..8b1aea1ca9e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23937,9 +23937,9 @@ typescript@4.9.5, typescript@^4.5: integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== "typescript@>=3 < 6": - version "5.4.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.4.5.tgz#42ccef2c571fdbd0f6718b1d1f5e6e5ef006f611" - integrity sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ== + version "5.6.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.6.3.tgz#5f3449e31c9d94febb17de03cc081dd56d81db5b" + integrity sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw== typical@^2.6.1: version "2.6.1" From 5830cf3cdb7c9bf0d4d92f338091601a0f06819d Mon Sep 17 00:00:00 2001 From: euanmillar Date: Wed, 20 Nov 2024 11:52:15 +0000 Subject: [PATCH 08/14] Refactor approach to sending files from workflow to documents --- .../src/features/uploadDocument/handler.ts | 41 +++---------- packages/gateway/src/utils/documents.ts | 31 ---------- packages/workflow/src/documents.ts | 59 +++++++------------ .../src/records/handler/correction/request.ts | 6 +- 4 files changed, 34 insertions(+), 103 deletions(-) delete mode 100644 packages/gateway/src/utils/documents.ts diff --git a/packages/documents/src/features/uploadDocument/handler.ts b/packages/documents/src/features/uploadDocument/handler.ts index a0028fb0689..cce0f134b72 100644 --- a/packages/documents/src/features/uploadDocument/handler.ts +++ b/packages/documents/src/features/uploadDocument/handler.ts @@ -31,40 +31,17 @@ export async function documentUploadHandler( const payload = request.payload as IDocumentPayload const ref = uuid() try { - // payload is an image - const regex = /^data:image/ - let generateFileName - if (regex.test(payload.fileData)) { - const base64String = payload.fileData.split(',')[1] - const base64Decoded = Buffer.from(base64String, 'base64') - const fileType = (await fromBuffer(base64Decoded)) as IFileInfo - generateFileName = `${ref}.${fileType.ext}` + const base64String = payload.fileData.split(',')[1] + const base64Decoded = Buffer.from(base64String, 'base64') + const fileType = (await fromBuffer(base64Decoded)) as IFileInfo + const generateFileName = `${ref}.${fileType.ext}` - await minioClient.putObject( - MINIO_BUCKET, - generateFileName, - base64Decoded, - { - 'content-type': fileType.mime, - ...payload.metaData - } - ) - } else { - // payload is an svg cert - const generateFileName = `${ref}.CERTIFICATE` - await minioClient.putObject( - MINIO_BUCKET, - generateFileName, - payload.fileData, - { - 'content-type': 'image/svg+xml' - } - ) - } + await minioClient.putObject(MINIO_BUCKET, generateFileName, base64Decoded, { + 'content-type': fileType.mime, + ...payload.metaData + }) - return h - .response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) - .code(200) + return h.response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) } catch (error) { return Promise.reject(new Error(`request failed: ${error.message}`)) } diff --git a/packages/gateway/src/utils/documents.ts b/packages/gateway/src/utils/documents.ts deleted file mode 100644 index 7a08a3ed2fa..00000000000 --- a/packages/gateway/src/utils/documents.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. - * - * OpenCRVS is also distributed under the terms of the Civil Registration - * & Healthcare Disclaimer located at http://opencrvs.org/license. - * - * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. - */ -import fetch from '@gateway/fetch' -import { DOCUMENTS_URL } from '@gateway/constants' -import { internal } from '@hapi/boom' -import { logger, IAuthHeader } from '@opencrvs/commons' - -export async function uploadSvg(fileData: string, authHeader: IAuthHeader) { - const url = new URL('upload-svg', DOCUMENTS_URL).toString() - const res = await fetch(url, { - method: 'POST', - headers: { - ...authHeader, - 'Content-Type': 'image/svg+xml' - }, - body: Buffer.from(fileData) - }) - if (!res.ok) { - logger.error(await res.json()) - throw internal() - } - return (await res.json()).refUrl -} diff --git a/packages/workflow/src/documents.ts b/packages/workflow/src/documents.ts index 671c4b444c2..87c1157905b 100644 --- a/packages/workflow/src/documents.ts +++ b/packages/workflow/src/documents.ts @@ -18,36 +18,23 @@ import { } from '@opencrvs/commons/types' import { CertifyInput, IssueInput } from './records/validations' -const fetchDocuments = async ( - suffix: string, +export async function uploadFileToMinio( + fileData: string, authHeader: IAuthHeader, - method = 'GET', - body: string | undefined = undefined -): Promise => { - const result = await fetch(`${DOCUMENTS_URL}${suffix}`, { - method, + svg?: boolean +): Promise { + const suffix = svg ? '/upload-svg' : '/upload' + const request = { + method: 'POST', headers: { ...authHeader, - 'Content-Type': 'application/json' + 'Content-Type': svg ? 'image/svg+xml' : 'application/json' }, - body - }) + body: svg ? fileData : JSON.stringify({ fileData: fileData }) + } + const result = await fetch(`${DOCUMENTS_URL}${suffix}`, request) const res = await result.json() - return res -} - -export async function uploadBase64ToMinio( - fileData: string, - authHeader: IAuthHeader -): Promise { - const docUploadResponse = await fetchDocuments( - '/upload', - authHeader, - 'POST', - JSON.stringify({ fileData: fileData }) - ) - - return docUploadResponse.refUrl + return res.refUrl } export async function uploadCertificateAttachmentsToDocumentsStore< @@ -58,13 +45,14 @@ export async function uploadCertificateAttachmentsToDocumentsStore< certificateDetails.collector.affidavit ) { for (const affidavit of certificateDetails.collector.affidavit) { - affidavit.data = await uploadBase64ToMinio(affidavit.data, authHeader) + affidavit.data = await uploadFileToMinio(affidavit.data, authHeader, true) } } if ('data' in certificateDetails) { - certificateDetails.data = await uploadBase64ToMinio( + certificateDetails.data = await uploadFileToMinio( certificateDetails.data, - authHeader + authHeader, + true ) } return certificateDetails @@ -86,7 +74,7 @@ function uploadOrNormaliseSignatureData( authHeader: IAuthHeader ) { if (isBase64FileString(signature)) { - return uploadBase64ToMinio(signature, authHeader) + return uploadFileToMinio(signature, authHeader) } if (isPresignedUrl(signature)) { @@ -138,7 +126,7 @@ export async function uploadBase64AttachmentsToDocumentsStore( if (record.registration?.attachments) { for (const attachment of record.registration.attachments) { if (attachment.data && isBase64FileString(attachment.data)) { - const fileUri = await uploadBase64ToMinio(attachment.data, authHeader) + const fileUri = await uploadFileToMinio(attachment.data, authHeader) attachment.data = fileUri } } @@ -151,10 +139,7 @@ export async function uploadBase64AttachmentsToDocumentsStore( if (certificate.collector.affidavit) { for (const affidavit of certificate.collector.affidavit) { if (affidavit.data && isBase64FileString(affidavit.data)) { - const fileUri = await uploadBase64ToMinio( - affidavit.data, - authHeader - ) + const fileUri = await uploadFileToMinio(affidavit.data, authHeader) affidavit.data = fileUri } } @@ -162,7 +147,7 @@ export async function uploadBase64AttachmentsToDocumentsStore( if (certificate.collector.photo) { for (const photo of certificate.collector.photo) { if (photo.data && isBase64FileString(photo.data)) { - const fileUri = await uploadBase64ToMinio(photo.data, authHeader) + const fileUri = await uploadFileToMinio(photo.data, authHeader) photo.data = fileUri } } @@ -172,13 +157,13 @@ export async function uploadBase64AttachmentsToDocumentsStore( if (record.registration?.correction?.attachments) { for (const attachment of record.registration.correction.attachments) { if (attachment.data && isBase64FileString(attachment.data)) { - const fileUri = await uploadBase64ToMinio(attachment.data, authHeader) + const fileUri = await uploadFileToMinio(attachment.data, authHeader) attachment.data = fileUri } } } if (record.registration?.correction?.payment?.attachmentData) { - const fileUri = await uploadBase64ToMinio( + const fileUri = await uploadFileToMinio( record.registration.correction.payment.attachmentData, authHeader ) diff --git a/packages/workflow/src/records/handler/correction/request.ts b/packages/workflow/src/records/handler/correction/request.ts index a645336add2..af6e8438857 100644 --- a/packages/workflow/src/records/handler/correction/request.ts +++ b/packages/workflow/src/records/handler/correction/request.ts @@ -12,7 +12,7 @@ import { conflict } from '@hapi/boom' import { getAuthHeader } from '@opencrvs/commons/http' import { CorrectionRequestedRecord } from '@opencrvs/commons/types' -import { uploadBase64ToMinio } from '@workflow/documents' +import { uploadFileToMinio } from '@workflow/documents' import { getLoggedInPractitionerResource, getPractitionerOfficeId @@ -51,7 +51,7 @@ export const requestCorrectionRoute = createRoute({ const paymentAttachmentUrl = correctionDetails.payment?.attachmentData && - (await uploadBase64ToMinio( + (await uploadFileToMinio( correctionDetails.payment.attachmentData, getAuthHeader(request) )) @@ -59,7 +59,7 @@ export const requestCorrectionRoute = createRoute({ const proofOfLegalCorrectionAttachments = await Promise.all( correctionDetails.attachments.map(async (attachment) => ({ type: attachment.type, - url: await uploadBase64ToMinio(attachment.data, getAuthHeader(request)) + url: await uploadFileToMinio(attachment.data, getAuthHeader(request)) })) ) From c3af7ed3256dbaf6eac4b0808a329657c73c7cff Mon Sep 17 00:00:00 2001 From: euanmillar Date: Wed, 20 Nov 2024 11:59:16 +0000 Subject: [PATCH 09/14] Fixed test --- packages/workflow/src/records/handler/certify.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/workflow/src/records/handler/certify.test.ts b/packages/workflow/src/records/handler/certify.test.ts index 56a9af5e170..0b77f5614b5 100644 --- a/packages/workflow/src/records/handler/certify.test.ts +++ b/packages/workflow/src/records/handler/certify.test.ts @@ -57,7 +57,7 @@ describe('Certify record endpoint', () => { // Upload certificate to minio mswServer.use( - rest.post('http://localhost:9050/upload', (_, res, ctx) => { + rest.post('http://localhost:9050/upload-svg', (_, res, ctx) => { return res( ctx.json({ refUrl: '/ocrvs/6e964d7a-25d0-4524-bdc2-b1f29d1e816c' }) ) @@ -153,7 +153,7 @@ describe('Certify record endpoint', () => { // Upload certificate to minio mswServer.use( - rest.post('http://localhost:9050/upload', (_, res, ctx) => { + rest.post('http://localhost:9050/upload-svg', (_, res, ctx) => { return res( ctx.json({ refUrl: '/ocrvs/6e964d7a-25d0-4524-bdc2-b1f29d1e816c' }) ) From b8205e8b1cda01b44866b8c2c951140d247ed86b Mon Sep 17 00:00:00 2001 From: euanmillar Date: Wed, 20 Nov 2024 12:02:00 +0000 Subject: [PATCH 10/14] Revert mistaken removal of response code --- packages/documents/src/features/uploadDocument/handler.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/documents/src/features/uploadDocument/handler.ts b/packages/documents/src/features/uploadDocument/handler.ts index cce0f134b72..fc860f6021f 100644 --- a/packages/documents/src/features/uploadDocument/handler.ts +++ b/packages/documents/src/features/uploadDocument/handler.ts @@ -41,7 +41,9 @@ export async function documentUploadHandler( ...payload.metaData }) - return h.response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) + return h + .response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) + .code(200) } catch (error) { return Promise.reject(new Error(`request failed: ${error.message}`)) } From 6e174168085830d3c52f675ac0af075c690a0e9a Mon Sep 17 00:00:00 2001 From: euanmillar Date: Wed, 4 Dec 2024 10:40:55 +0000 Subject: [PATCH 11/14] renamed method --- packages/workflow/src/documents.ts | 34 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/workflow/src/documents.ts b/packages/workflow/src/documents.ts index 87c1157905b..e31854b449a 100644 --- a/packages/workflow/src/documents.ts +++ b/packages/workflow/src/documents.ts @@ -20,17 +20,34 @@ import { CertifyInput, IssueInput } from './records/validations' export async function uploadFileToMinio( fileData: string, - authHeader: IAuthHeader, - svg?: boolean + authHeader: IAuthHeader +): Promise { + const suffix = '/upload' + const request = { + method: 'POST', + headers: { + ...authHeader, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ fileData: fileData }) + } + const result = await fetch(`${DOCUMENTS_URL}${suffix}`, request) + const res = await result.json() + return res.refUrl +} + +export async function uploadSVGToMinio( + fileData: string, + authHeader: IAuthHeader ): Promise { - const suffix = svg ? '/upload-svg' : '/upload' + const suffix = '/upload-svg' const request = { method: 'POST', headers: { ...authHeader, - 'Content-Type': svg ? 'image/svg+xml' : 'application/json' + 'Content-Type': 'image/svg+xml' }, - body: svg ? fileData : JSON.stringify({ fileData: fileData }) + body: fileData } const result = await fetch(`${DOCUMENTS_URL}${suffix}`, request) const res = await result.json() @@ -45,14 +62,13 @@ export async function uploadCertificateAttachmentsToDocumentsStore< certificateDetails.collector.affidavit ) { for (const affidavit of certificateDetails.collector.affidavit) { - affidavit.data = await uploadFileToMinio(affidavit.data, authHeader, true) + affidavit.data = await uploadSVGToMinio(affidavit.data, authHeader) } } if ('data' in certificateDetails) { - certificateDetails.data = await uploadFileToMinio( + certificateDetails.data = await uploadSVGToMinio( certificateDetails.data, - authHeader, - true + authHeader ) } return certificateDetails From a44ee03a9d0012fa85450233d52f675ba827721a Mon Sep 17 00:00:00 2001 From: euanmillar Date: Fri, 6 Dec 2024 15:01:34 +0000 Subject: [PATCH 12/14] Affidavt is not an SVG --- packages/workflow/src/documents.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/workflow/src/documents.ts b/packages/workflow/src/documents.ts index e31854b449a..3af2de96b64 100644 --- a/packages/workflow/src/documents.ts +++ b/packages/workflow/src/documents.ts @@ -62,7 +62,7 @@ export async function uploadCertificateAttachmentsToDocumentsStore< certificateDetails.collector.affidavit ) { for (const affidavit of certificateDetails.collector.affidavit) { - affidavit.data = await uploadSVGToMinio(affidavit.data, authHeader) + affidavit.data = await uploadFileToMinio(affidavit.data, authHeader) } } if ('data' in certificateDetails) { From 2c17549fafd4b9c6bae74b144bafbe17f9b91ed7 Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Mon, 9 Dec 2024 11:56:02 +0200 Subject: [PATCH 13/14] Apply suggestions from code review --- packages/workflow/src/documents.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/workflow/src/documents.ts b/packages/workflow/src/documents.ts index 3af2de96b64..debe9bc2105 100644 --- a/packages/workflow/src/documents.ts +++ b/packages/workflow/src/documents.ts @@ -36,7 +36,7 @@ export async function uploadFileToMinio( return res.refUrl } -export async function uploadSVGToMinio( +async function uploadSVGToMinio( fileData: string, authHeader: IAuthHeader ): Promise { From 7ecbae2bb4e53056c6284febb3d5677aae14d41f Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Tue, 10 Dec 2024 15:12:44 +0200 Subject: [PATCH 14/14] wrap up version 1.6.1 --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d77877e1af1..761b8d3e774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 1.6.1 (TBD) +## 1.6.1 ### Bug fixes @@ -42,6 +42,7 @@ ] } ``` + - `hasChildLocation` query has been removed from gateway. We have created the query `isLeafLevelLocation` instead which is more descriptive on its intended use. ### New features @@ -121,7 +122,7 @@ - **Check your Metabase map file.** For Metabase configuration, we renamed `farajaland-map.geojson` to `map.geojson` to not tie implementations into example country naming conventions. - **Feature flags** In order to make application config settings more readable, we re-organised `src/api/application/application-config-default.ts` with a clear feature flag block like so. These are then used across the front and back end of the application to control configurable functionality. New feature flags DEATH_REGISTRATION allow you to optionally run off death registration if your country doesnt want to run its first pilot including death and PRINT_DECLARATION (see New Features) have been added. `FEATURES: { - DEATH_REGISTRATION: true, + DEATH_REGISTRATION: true, MARRIAGE_REGISTRATION: false, ... } ` @@ -267,6 +268,7 @@ In the next OpenCRVS release v1.5.0, there will be two significant changes both - **New handlebars serving the location ids of the admin level locations** Apart from the new handlebars, a couple more improvements were introduced: + - stricter type for locations in client - **"location"** handlebar helper can now resolve offices & facilities - restrict the properties exposed through **"location"** handlebar helper