From 71b057242ec24b7eb00efa51c895b7deb56b1015 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Mon, 16 Oct 2023 17:30:21 +0200 Subject: [PATCH 01/13] Add min ad max columns to features entity --- .../1645026803969-AddTokenIdColumnToLocks.ts | 1 + .../import/features-amounts-upload.service.ts | 20 +++++++++++-- .../src/modules/projects/projects.service.ts | 30 +++++++++++++------ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/api/apps/api/src/migrations/api/1645026803969-AddTokenIdColumnToLocks.ts b/api/apps/api/src/migrations/api/1645026803969-AddTokenIdColumnToLocks.ts index 9c9699885b..926997d4af 100644 --- a/api/apps/api/src/migrations/api/1645026803969-AddTokenIdColumnToLocks.ts +++ b/api/apps/api/src/migrations/api/1645026803969-AddTokenIdColumnToLocks.ts @@ -1,3 +1,4 @@ + import { MigrationInterface, QueryRunner } from 'typeorm'; export class AddTokenIdColumnToLocks1645026803969 diff --git a/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts b/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts index 2435a3ed7d..3b4a0e5b13 100644 --- a/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts +++ b/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts @@ -1,4 +1,10 @@ -import { BadRequestException, Injectable, Logger } from '@nestjs/common'; +import { + BadRequestException, + forwardRef, + Inject, + Injectable, + Logger, +} from '@nestjs/common'; import { DbConnections } from '@marxan-api/ormconfig.connections'; import { DataSource, EntityManager, QueryRunner } from 'typeorm'; import { InjectDataSource, InjectEntityManager } from '@nestjs/typeorm'; @@ -6,6 +12,7 @@ import { featureAmountCsvParser } from '@marxan-api/modules/geo-features/import/ import { FeatureAmountCSVDto } from '@marxan-api/modules/geo-features/dto/feature-amount-csv.dto'; import { FeatureAmountUploadRegistry } from '@marxan-api/modules/geo-features/import/features-amounts-upload-registry.api.entity'; import { + GeoFeaturesService, importedFeatureNameAlreadyExist, unknownPuidsInFeatureAmountCsvUpload, } from '@marxan-api/modules/geo-features/geo-features.service'; @@ -19,6 +26,7 @@ import { CHUNK_SIZE_FOR_BATCH_APIDB_OPERATIONS } from '@marxan-api/utils/chunk-s import { UploadedFeatureAmount } from '@marxan-api/modules/geo-features/import/features-amounts-data.api.entity'; import { Project } from '@marxan-api/modules/projects/project.api.entity'; import { ProjectSourcesEnum } from '@marxan/projects'; +import { ScenariosService } from '@marxan-api/modules/scenarios/scenarios.service'; @Injectable() export class FeatureAmountUploadService { @@ -33,6 +41,8 @@ export class FeatureAmountUploadService { @InjectEntityManager(DbConnections.default) private readonly apiEntityManager: EntityManager, private readonly events: FeatureImportEventsService, + @Inject(forwardRef(() => GeoFeaturesService)) + private readonly geoFeaturesService: GeoFeaturesService, ) {} async uploadFeatureFromCsv(data: { @@ -107,6 +117,12 @@ export class FeatureAmountUploadService { apiQueryRunner.manager, ); + this.logger.log(`Saving min and max amounts for new features...`); + + await this.geoFeaturesService.saveAmountRangeForFeatures( + newFeaturesFromCsvUpload.map((feature) => feature.id), + ); + this.logger.log(`Csv file upload process finished successfully`); // Committing transaction @@ -211,7 +227,7 @@ export class FeatureAmountUploadService { queryRunner: QueryRunner, uploadId: string, projectId: string, - ) { + ): Promise { const newFeaturesToCreate = ( await queryRunner.manager .createQueryBuilder() diff --git a/api/apps/api/src/modules/projects/projects.service.ts b/api/apps/api/src/modules/projects/projects.service.ts index ddbae825df..015b7eab71 100644 --- a/api/apps/api/src/modules/projects/projects.service.ts +++ b/api/apps/api/src/modules/projects/projects.service.ts @@ -178,16 +178,28 @@ export class ProjectsService { return project; } - return right( - await this.geoCrud.findAllPaginated(fetchSpec, { - ...appInfo, - params: { - ...appInfo.params, - projectId: project.right.id, - bbox: project.right.bbox, - }, + const result = await this.geoCrud.findAllPaginated(fetchSpec, { + ...appInfo, + params: { + ...appInfo.params, + projectId: project.right.id, + bbox: project.right.bbox, + }, + }); + + const resultWithMappedAmountRange = { + data: result.data.map((feature) => { + if (feature?.amountMax || feature?.amountMin) { + return { + ...feature, + amountRange: { min: feature.amountMin, max: feature.amountMax }, + }; + } }), - ); + metadata: result.metadata, + }; + + return right(resultWithMappedAmountRange); } async findAll(fetchSpec: FetchSpecification, info: ProjectsServiceRequest) { From 2bf773a44a2412859ed451f59c34736d8913b3b2 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Mon, 16 Oct 2023 17:31:21 +0200 Subject: [PATCH 02/13] Add service to find and save min and max values from puvspr_calculations for a features --- .../geo-features/geo-feature.api.entity.ts | 14 ++++++++++++ .../geo-features/geo-features.controller.ts | 7 +++--- .../geo-features/geo-features.service.ts | 22 ++++++++++++++++++- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/api/apps/api/src/modules/geo-features/geo-feature.api.entity.ts b/api/apps/api/src/modules/geo-features/geo-feature.api.entity.ts index 37e5578a22..a8b1049571 100644 --- a/api/apps/api/src/modules/geo-features/geo-feature.api.entity.ts +++ b/api/apps/api/src/modules/geo-features/geo-feature.api.entity.ts @@ -27,6 +27,11 @@ export interface GeoFeatureProperty { distinctValues: Array; } +export type FeatureAmountRange = { + min: number | undefined; + max: number | undefined; +}; + @Entity('features') export class GeoFeature extends BaseEntity { @ApiProperty() @@ -95,6 +100,12 @@ export class GeoFeature extends BaseEntity { @Column('boolean', { name: 'is_custom' }) isCustom?: boolean; + @Column('float8', { name: 'amount_min' }) + amountMin?: number; + + @Column('float8', { name: 'amount_max' }) + amountMax?: number; + @Column('boolean', { name: 'is_legacy' }) isLegacy!: boolean; @@ -109,6 +120,9 @@ export class GeoFeature extends BaseEntity { @ApiPropertyOptional() scenarioUsageCount?: number; + + @ApiPropertyOptional() + amountRange?: FeatureAmountRange; } export class JSONAPIGeoFeaturesData { diff --git a/api/apps/api/src/modules/geo-features/geo-features.controller.ts b/api/apps/api/src/modules/geo-features/geo-features.controller.ts index e02eaf91a2..ad3f896d34 100644 --- a/api/apps/api/src/modules/geo-features/geo-features.controller.ts +++ b/api/apps/api/src/modules/geo-features/geo-features.controller.ts @@ -56,7 +56,6 @@ import { mapAclDomainToHttpError } from '@marxan-api/utils/acl.utils'; ) export class GeoFeaturesController { constructor( - public readonly service: GeoFeaturesService, private readonly geoFeatureService: GeoFeaturesService, public readonly geoFeaturesTagService: GeoFeatureTagsService, private readonly proxyService: ProxyService, @@ -75,8 +74,10 @@ export class GeoFeaturesController { async findAll( @ProcessFetchSpecification() fetchSpecification: FetchSpecification, ): Promise { - const results = await this.service.findAllPaginated(fetchSpecification); - return this.service.serialize(results.data, results.metadata); + const results = await this.geoFeatureService.findAllPaginated( + fetchSpecification, + ); + return this.geoFeatureService.serialize(results.data, results.metadata); } @ImplementsAcl() diff --git a/api/apps/api/src/modules/geo-features/geo-features.service.ts b/api/apps/api/src/modules/geo-features/geo-features.service.ts index 25c8e375c7..316a366dd4 100644 --- a/api/apps/api/src/modules/geo-features/geo-features.service.ts +++ b/api/apps/api/src/modules/geo-features/geo-features.service.ts @@ -41,7 +41,6 @@ import { ScenarioFeaturesService } from '@marxan-api/modules/scenarios-features' import { GeoFeatureTag } from '@marxan-api/modules/geo-feature-tags/geo-feature-tag.api.entity'; import { GeoFeatureTagsService } from '@marxan-api/modules/geo-feature-tags/geo-feature-tags.service'; import { FeatureAmountUploadService } from '@marxan-api/modules/geo-features/import/features-amounts-upload.service'; -import { isLeft } from 'fp-ts/Either'; import { isNil } from 'lodash'; const geoFeatureFilterKeyNames = [ @@ -138,6 +137,7 @@ export class GeoFeaturesService extends AppBaseService< 'isCustom', 'tag', 'scenarioUsageCount', + 'amountRange', ], keyForAttribute: 'camelCase', }; @@ -834,4 +834,24 @@ export class GeoFeaturesService extends AppBaseService< scenarioUsageCount: usage ? Number(usage.count) : 0, } as GeoFeature; } + + async saveAmountRangeForFeatures(featureIds: string[]) { + for (const featureId of featureIds) { + const minAndMaxAmount = await this.geoEntityManager + .createQueryBuilder() + .select('MIN(amount)', 'min') + .addSelect('MAX(amount)', 'max') + .from('puvspr_calculations', 'puvspr') + .where('puvspr.feature_id = :featureId', { featureId }) + .getOneOrFail(); + + await this.geoFeaturesRepository.update( + { id: featureId }, + { + amountMin: minAndMaxAmount.min, + amountMax: minAndMaxAmount.max, + }, + ); + } + } } From 9f2dcc1e89d902c07ee37eb9f942eefed0efa060 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Mon, 16 Oct 2023 17:45:56 +0200 Subject: [PATCH 03/13] Update custom project features piece exporter to include min and max amount values --- .../project-custom-features.piece-exporter.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/project-custom-features.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/project-custom-features.piece-exporter.ts index a51bd975eb..0009028b58 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/project-custom-features.piece-exporter.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/project-custom-features.piece-exporter.ts @@ -32,6 +32,8 @@ type ProjectCustomFeaturesSelectResult = { list_property_keys: string[]; is_legacy: boolean; tag: string; + amount_min: number | null; + amount_max: number | null; }; type FeaturesDataSelectResult = { @@ -81,6 +83,8 @@ export class ProjectCustomFeaturesPieceExporter 'f.creation_status', 'f.list_property_keys', 'f.is_legacy', + 'f.min_amount', + 'f.max_amount', 'pft.tag', ]) .from('features', 'f') From 53cf6ff0b189d15f7d555477bc58a1830ca4086e Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Tue, 17 Oct 2023 10:17:20 +0200 Subject: [PATCH 04/13] Add migration file for min and max amounts columns to features --- ...673344-AddMinMaxAmountColumnsToFeatures.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 api/apps/api/src/migrations/api/1697210673344-AddMinMaxAmountColumnsToFeatures.ts diff --git a/api/apps/api/src/migrations/api/1697210673344-AddMinMaxAmountColumnsToFeatures.ts b/api/apps/api/src/migrations/api/1697210673344-AddMinMaxAmountColumnsToFeatures.ts new file mode 100644 index 0000000000..93626d8911 --- /dev/null +++ b/api/apps/api/src/migrations/api/1697210673344-AddMinMaxAmountColumnsToFeatures.ts @@ -0,0 +1,21 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class AddMinMaxAmountColumnsToFeatures1697210673344 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE features + ADD COLUMN amount_min float8, + ADD COLUMN amount_max float8; + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE features + DROP COLUMN amount_min float8, + DROP COLUMN amount_max float8; + `); + } + +} From 826cd93da5a81a209740824ef4d127a514b3d76a Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Tue, 17 Oct 2023 16:32:18 +0200 Subject: [PATCH 05/13] Update min and max amounts mapping for features --- .../modules/geo-features/geo-feature.api.entity.ts | 4 ++-- .../api/src/modules/projects/projects.service.ts | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api/apps/api/src/modules/geo-features/geo-feature.api.entity.ts b/api/apps/api/src/modules/geo-features/geo-feature.api.entity.ts index a8b1049571..6b2a9327e2 100644 --- a/api/apps/api/src/modules/geo-features/geo-feature.api.entity.ts +++ b/api/apps/api/src/modules/geo-features/geo-feature.api.entity.ts @@ -28,8 +28,8 @@ export interface GeoFeatureProperty { } export type FeatureAmountRange = { - min: number | undefined; - max: number | undefined; + min: number | null; + max: number | null; }; @Entity('features') diff --git a/api/apps/api/src/modules/projects/projects.service.ts b/api/apps/api/src/modules/projects/projects.service.ts index 015b7eab71..6d30c49bf5 100644 --- a/api/apps/api/src/modules/projects/projects.service.ts +++ b/api/apps/api/src/modules/projects/projects.service.ts @@ -189,12 +189,13 @@ export class ProjectsService { const resultWithMappedAmountRange = { data: result.data.map((feature) => { - if (feature?.amountMax || feature?.amountMin) { - return { - ...feature, - amountRange: { min: feature.amountMin, max: feature.amountMax }, - }; - } + return { + ...feature, + amountRange: { + min: feature?.amountMin ?? null, + max: feature?.amountMax ?? null, + }, + }; }), metadata: result.metadata, }; From 391dd81d6ab3cb2e63bc80318a4ad35c1385f001 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Tue, 17 Oct 2023 17:12:56 +0200 Subject: [PATCH 06/13] Fix min and max amounts in feature piece exporter --- .../project-custom-features.piece-exporter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/project-custom-features.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/project-custom-features.piece-exporter.ts index 0009028b58..a565406a86 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/project-custom-features.piece-exporter.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/project-custom-features.piece-exporter.ts @@ -83,8 +83,8 @@ export class ProjectCustomFeaturesPieceExporter 'f.creation_status', 'f.list_property_keys', 'f.is_legacy', - 'f.min_amount', - 'f.max_amount', + 'f.amount_min', + 'f.amount_max', 'pft.tag', ]) .from('features', 'f') From e741b91dda7b03a27d39e67526d3c7498bb7af6a Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Wed, 18 Oct 2023 09:46:45 +0200 Subject: [PATCH 07/13] Update upload feature tests fixtures --- api/apps/api/test/upload-feature/upload-feature.fixtures.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/apps/api/test/upload-feature/upload-feature.fixtures.ts b/api/apps/api/test/upload-feature/upload-feature.fixtures.ts index 1719574abc..c108a67a9d 100644 --- a/api/apps/api/test/upload-feature/upload-feature.fixtures.ts +++ b/api/apps/api/test/upload-feature/upload-feature.fixtures.ts @@ -271,6 +271,8 @@ export const getFixtures = async () => { featureClassName: name, description, alias: null, + amountMax: null, + amountMin: null, propertyName: null, intersection: null, creationStatus: `done`, From 6e0084bfb4df2ba9a9d67394ec5f9806f8e1f5f7 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Wed, 18 Oct 2023 09:57:23 +0200 Subject: [PATCH 08/13] Update geo-features endpoint test --- api/apps/api/test/geo-features.e2e-spec.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/apps/api/test/geo-features.e2e-spec.ts b/api/apps/api/test/geo-features.e2e-spec.ts index 591305bcdd..265a60a725 100644 --- a/api/apps/api/test/geo-features.e2e-spec.ts +++ b/api/apps/api/test/geo-features.e2e-spec.ts @@ -9,7 +9,7 @@ import { import { bootstrapApplication } from './utils/api-application'; import { GivenUserIsLoggedIn } from './steps/given-user-is-logged-in'; -import { createWorld } from './project/projects-world'; +import { createWorld } from './projects/projects-world'; import { Repository } from 'typeorm'; import { ScenarioFeaturesData } from '@marxan/features'; import { v4 } from 'uuid'; @@ -65,6 +65,10 @@ describe('GeoFeaturesModule (e2e)', () => { const geoFeaturesForProject: GeoFeature[] = response.body.data; expect(geoFeaturesForProject.length).toBeGreaterThan(0); expect(response.body.data[0].type).toBe(geoFeatureResource.name.plural); + expect(response.body.data[0].attributes.amountRange).toEqual({ + min: null, + max: null, + }); }); test('should include correct scenarioUsageCounts for the given project', async () => { From f2919fdd468a26e1191445b1c80de9517408776c Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Wed, 18 Oct 2023 10:10:29 +0200 Subject: [PATCH 09/13] Update geo-features service to retrieve feature amounts range from puvspr --- .../api/src/modules/geo-features/geo-features.service.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/apps/api/src/modules/geo-features/geo-features.service.ts b/api/apps/api/src/modules/geo-features/geo-features.service.ts index 316a366dd4..09f8db3209 100644 --- a/api/apps/api/src/modules/geo-features/geo-features.service.ts +++ b/api/apps/api/src/modules/geo-features/geo-features.service.ts @@ -843,13 +843,13 @@ export class GeoFeaturesService extends AppBaseService< .addSelect('MAX(amount)', 'max') .from('puvspr_calculations', 'puvspr') .where('puvspr.feature_id = :featureId', { featureId }) - .getOneOrFail(); + .getOne(); await this.geoFeaturesRepository.update( { id: featureId }, { - amountMin: minAndMaxAmount.min, - amountMax: minAndMaxAmount.max, + amountMin: minAndMaxAmount?.min ?? null, + amountMax: minAndMaxAmount?.max ?? null, }, ); } From fdcaeef076e3a8dabf712ce6aebf10ae39ce09ee Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Thu, 19 Oct 2023 11:30:44 +0200 Subject: [PATCH 10/13] Add index to (geoDB)puvspr_calculations.feature_id --- ...392-AddIndexToFeatureIdOfPuvsprCalculations.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 api/apps/geoprocessing/src/migrations/geoprocessing/1697707458392-AddIndexToFeatureIdOfPuvsprCalculations.ts diff --git a/api/apps/geoprocessing/src/migrations/geoprocessing/1697707458392-AddIndexToFeatureIdOfPuvsprCalculations.ts b/api/apps/geoprocessing/src/migrations/geoprocessing/1697707458392-AddIndexToFeatureIdOfPuvsprCalculations.ts new file mode 100644 index 0000000000..8ff9f32241 --- /dev/null +++ b/api/apps/geoprocessing/src/migrations/geoprocessing/1697707458392-AddIndexToFeatureIdOfPuvsprCalculations.ts @@ -0,0 +1,15 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddIndexToFeatureIdOfPuvsprCalculations1697707458392 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX puvspr_calculations_feature_id__idx ON "puvspr_calculations" ("feature_id") `, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX puvspr_calculations_feature_id__idx`); + } +} From d00eba0b1f73446ccf4baae2505706e5b9bb4e29 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Thu, 19 Oct 2023 12:06:44 +0200 Subject: [PATCH 11/13] Update GeoFeaturesService.saveAmountRangeForFeatures() to improve performance --- .../geo-features/geo-features.service.ts | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/api/apps/api/src/modules/geo-features/geo-features.service.ts b/api/apps/api/src/modules/geo-features/geo-features.service.ts index 09f8db3209..a8d8a7a848 100644 --- a/api/apps/api/src/modules/geo-features/geo-features.service.ts +++ b/api/apps/api/src/modules/geo-features/geo-features.service.ts @@ -835,23 +835,20 @@ export class GeoFeaturesService extends AppBaseService< } as GeoFeature; } + // @TODO: update tests once saving amounts in puvspr_calculations is consolidates + async saveAmountRangeForFeatures(featureIds: string[]) { - for (const featureId of featureIds) { - const minAndMaxAmount = await this.geoEntityManager - .createQueryBuilder() - .select('MIN(amount)', 'min') - .addSelect('MAX(amount)', 'max') - .from('puvspr_calculations', 'puvspr') - .where('puvspr.feature_id = :featureId', { featureId }) - .getOne(); - - await this.geoFeaturesRepository.update( - { id: featureId }, - { - amountMin: minAndMaxAmount?.min ?? null, - amountMax: minAndMaxAmount?.max ?? null, - }, - ); - } + this.logger.log(`Saving min and max amounts for new features...`); + + const minAndMaxAmountsForFeatures = await this.geoEntityManager + .createQueryBuilder() + .select('feature_id', 'id') + .select('MIN(amount)', 'amountMin') + .addSelect('MAX(amount)', 'amountMax') + .from('puvspr_calculations', 'puvspr') + .where('puvspr.feature_id IN (:...featureIds)', { featureIds }) + .getMany(); + + await this.geoFeaturesRepository.save(minAndMaxAmountsForFeatures); } } From 7172b6a5febabd48a2d8a0d64538a71d50c2891b Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Thu, 19 Oct 2023 12:56:56 +0200 Subject: [PATCH 12/13] Fix GeoFeaturesService.saveAmountRangeForFeatures() query --- .../api/src/modules/geo-features/geo-features.service.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/apps/api/src/modules/geo-features/geo-features.service.ts b/api/apps/api/src/modules/geo-features/geo-features.service.ts index a8d8a7a848..22b7bef340 100644 --- a/api/apps/api/src/modules/geo-features/geo-features.service.ts +++ b/api/apps/api/src/modules/geo-features/geo-features.service.ts @@ -843,12 +843,15 @@ export class GeoFeaturesService extends AppBaseService< const minAndMaxAmountsForFeatures = await this.geoEntityManager .createQueryBuilder() .select('feature_id', 'id') - .select('MIN(amount)', 'amountMin') + .addSelect('MIN(amount)', 'amountMin') .addSelect('MAX(amount)', 'amountMax') .from('puvspr_calculations', 'puvspr') .where('puvspr.feature_id IN (:...featureIds)', { featureIds }) - .getMany(); + .groupBy('puvspr.feature_id') + .getRawMany(); - await this.geoFeaturesRepository.save(minAndMaxAmountsForFeatures); + await this.geoFeaturesRepository.upsert(minAndMaxAmountsForFeatures, [ + 'id', + ]); } } From 4268a885002edd789ffde622faca1eca881b9503 Mon Sep 17 00:00:00 2001 From: yulia-bel Date: Thu, 19 Oct 2023 17:53:52 +0200 Subject: [PATCH 13/13] Refactor feature csv upload service --- .../geo-features/geo-features.service.ts | 20 ++++++++++++++++--- .../import/features-amounts-upload.service.ts | 12 +++++++---- .../import-files/feature_amount_upload.csv | 2 +- .../upload-feature/upload-feature.fixtures.ts | 7 ++++++- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/api/apps/api/src/modules/geo-features/geo-features.service.ts b/api/apps/api/src/modules/geo-features/geo-features.service.ts index 22b7bef340..aaa2eddfae 100644 --- a/api/apps/api/src/modules/geo-features/geo-features.service.ts +++ b/api/apps/api/src/modules/geo-features/geo-features.service.ts @@ -850,8 +850,22 @@ export class GeoFeaturesService extends AppBaseService< .groupBy('puvspr.feature_id') .getRawMany(); - await this.geoFeaturesRepository.upsert(minAndMaxAmountsForFeatures, [ - 'id', - ]); + const minMaxSqlValueStringForFeatures = minAndMaxAmountsForFeatures + .map( + (feature) => + `(uuid('${feature.id}'), ${feature.amountMin}, ${feature.amountMax})`, + ) + .join(', '); + + const query = ` + update features set + amount_min = minmax.min, + amount_max = minmax.max + from ( + values + ${minMaxSqlValueStringForFeatures} + ) as minmax(feature_id, min, max) + where features.id = minmax.feature_id;`; + await this.geoFeaturesRepository.query(query); } } diff --git a/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts b/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts index 3b4a0e5b13..75f31629c9 100644 --- a/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts +++ b/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts @@ -119,15 +119,15 @@ export class FeatureAmountUploadService { this.logger.log(`Saving min and max amounts for new features...`); - await this.geoFeaturesService.saveAmountRangeForFeatures( - newFeaturesFromCsvUpload.map((feature) => feature.id), - ); - this.logger.log(`Csv file upload process finished successfully`); // Committing transaction await apiQueryRunner.commitTransaction(); await geoQueryRunner.commitTransaction(); + + await this.geoFeaturesService.saveAmountRangeForFeatures( + newFeaturesFromCsvUpload.map((feature) => feature.id), + ); } catch (err) { await this.events.failEvent(err); await apiQueryRunner.rollbackTransaction(); @@ -334,6 +334,10 @@ export class FeatureAmountUploadService { `, parameters, ); + await geoQueryRunner.manager.query( + ` INSERT INTO puvspr_calculations (project_id, feature_id, amount, project_pu_id) select $1, $2, amount, project_pu_id from features_data where feature_id = $2`, + [projectId, newFeature.id], + ); this.logger.log( `Chunk with index ${amountIndex} saved to (geoDB).features_data`, ); diff --git a/api/apps/api/test/upload-feature/import-files/feature_amount_upload.csv b/api/apps/api/test/upload-feature/import-files/feature_amount_upload.csv index 5b8aed641d..dc2d861d5c 100644 --- a/api/apps/api/test/upload-feature/import-files/feature_amount_upload.csv +++ b/api/apps/api/test/upload-feature/import-files/feature_amount_upload.csv @@ -1,4 +1,4 @@ puid,feat_1d666bd,feat_28135ef 1,4.245387225,0 2,4.245387225,0 -3,4.245387225,0 \ No newline at end of file +3,3.245387225,0 diff --git a/api/apps/api/test/upload-feature/upload-feature.fixtures.ts b/api/apps/api/test/upload-feature/upload-feature.fixtures.ts index c108a67a9d..00fbb2cb28 100644 --- a/api/apps/api/test/upload-feature/upload-feature.fixtures.ts +++ b/api/apps/api/test/upload-feature/upload-feature.fixtures.ts @@ -307,6 +307,8 @@ export const getFixtures = async () => { expect(newFeaturesAdded[0].projectId).toBe(projectId); expect(newFeaturesAdded[0].isLegacy).toBe(true); expect(newFeaturesAdded[1].isLegacy).toBe(true); + expect(newFeaturesAdded[0].amountMin).toEqual(3.245387225); + expect(newFeaturesAdded[0].amountMax).toEqual(4.245387225); }, ThenNewFeaturesAmountsAreCreated: async () => { @@ -322,6 +324,9 @@ export const getFixtures = async () => { }); const newFeature1Amounts = await featuresAmounsGeoDbRepository.find({ where: { featureId: newFeatures1?.id }, + order: { + amount: 'DESC', + }, }); const newFeature2Amounts = await featuresAmounsGeoDbRepository.find({ where: { featureId: newFeatures2?.id }, @@ -331,7 +336,7 @@ export const getFixtures = async () => { expect(newFeature2Amounts).toHaveLength(3); expect(newFeature1Amounts[0].amount).toBe(4.245387225); expect(newFeature1Amounts[1].amount).toBe(4.245387225); - expect(newFeature1Amounts[2].amount).toBe(4.245387225); + expect(newFeature1Amounts[2].amount).toBe(3.245387225); expect(newFeature2Amounts[0].amount).toBe(0); expect(newFeature2Amounts[1].amount).toBe(0);