[IMPORT] Migration script for transition from IMPORT V2 to IMPORT V3

backend/geonature/core/gn_synthese/imports/synthese_import_mixin.py

@@ -65,7 +65,8 @@ class SyntheseImportMixin(ImportMixin):
     @staticmethod
     def statistics_labels() -> typing.List[ImportStatisticsLabels]:
         return [
-            {"key": "taxa_count", "value": "Nombre de taxons importés"},
+            {"key": "observation_count", "value": "Nombre d'observations importées"},
+            {"key": "taxa_count", "value": "Nombre de taxons"},
         ]
 
     @staticmethod
@@ -356,12 +357,17 @@ def import_data_to_destination(imprt: TImports) -> None:
             select=select_stmt,
         )
         db.session.execute(insert_stmt)
+
+        # TODO: Improve this
         imprt.statistics = {
+            "observation_count": (
+                db.session.query(func.count(Synthese.cd_nom)).filter_by(source=source).scalar()
+            ),
             "taxa_count": (
                 db.session.query(func.count(distinct(Synthese.cd_nom)))
                 .filter_by(source=source)
                 .scalar()
-            )
+            ),
         }
 
     @staticmethod

backend/geonature/core/imports/config_schema.py

@@ -28,8 +28,8 @@
         "filter": False,
     },
     {
-        "prop": "import_count",
-        "name": "Nb de donnees",
+        "prop": "statistics_rows",
+        "name": "Lignes importées",
         "max_width": 100,
         "show": True,
         "filter": False,

backend/geonature/core/imports/models.py

@@ -290,6 +290,8 @@ class TImports(InstancePermissionMixin, db.Model):
     date_end_import = db.Column(db.DateTime, nullable=True)
     source_count = db.Column(db.Integer, nullable=True)
     erroneous_rows = deferred(db.Column(ARRAY(db.Integer), nullable=True))
+    # TODO: integrate this in statistics
+    # statistics: common and destination (line count, etc. etc. dedans)
     import_count = db.Column(db.Integer, nullable=True)
     statistics = db.Column(
         MutableDict.as_mutable(JSON), nullable=False, server_default="'{}'::jsonb"

backend/geonature/migrations/versions/imports/2b0b3bd0248c_multidest.py

@@ -6,6 +6,8 @@
 
 """
 
+import warnings
+
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.schema import Table, MetaData
@@ -212,7 +214,37 @@ def upgrade():
     # Remove from bib_fields columns moved to cor_entity_field
     for column_name in ["desc_field", "id_theme", "order_field", "comment"]:
         op.drop_column(schema="gn_imports", table_name="bib_fields", column_name=column_name)
+
     ### Permissions
+
+    #### Enlever scope dans C IMPORT
+    op.execute(
+        """
+        WITH CTE AS (
+            SELECT
+                m.id_module, o.id_object, a.id_action
+            FROM
+                gn_commons.t_modules m,
+                gn_permissions.t_objects o,
+                gn_permissions.bib_actions a
+            WHERE
+                m.module_code = 'IMPORT'
+                AND
+                o.code_object = 'ALL'
+                AND
+                a.code_action = 'C'
+        )
+        UPDATE gn_permissions.t_permissions_available pa
+        SET scope_filter = NULL
+        FROM CTE
+        WHERE
+            pa.id_module = CTE.id_module
+            AND
+            pa.id_object = CTE.id_object
+            AND
+            pa.id_action = CTE.id_action
+        """
+    )
     op.execute(
         """
         INSERT INTO
@@ -231,6 +263,7 @@ def upgrade():
             a.code_action = 'C'
         """
     )
+
     op.execute(
         """
         INSERT INTO
@@ -251,6 +284,7 @@ def upgrade():
             new_module.module_code = 'SYNTHESE' AND new_object.code_object = 'ALL';
         """
     )
+
     # TODO constraint entity_field.entity.id_destination == entity_field.field.id_destination
     ### Remove synthese specific 'id_source' column
     op.drop_column(schema="gn_imports", table_name="t_imports", column_name="id_source_synthese")
@@ -326,8 +360,77 @@ def upgrade():
         """
     )
 
+    ID_MODULE_IMPORT = (
+        op.get_bind()
+        .execute(
+            """
+            SELECT id_module FROM gn_commons.t_modules WHERE module_code = 'IMPORT';
+            """
+        )
+        .first()[0]
+    )
+    ID_MODULE_SYNTHESE = (
+        op.get_bind()
+        .execute(
+            """
+            SELECT id_module FROM gn_commons.t_modules WHERE module_code = 'SYNTHESE';
+            """
+        )
+        .first()[0]
+    )
+
+    ## JDD IMPORT -> JDD Synthese
+    # update row with module = import to module=synthese in cor_module_dataset, only if the dataset is not already associated with a synthese
+    op.execute(
+        f"""
+        UPDATE gn_commons.cor_module_dataset
+        SET id_module = {ID_MODULE_SYNTHESE}
+        WHERE id_module = {ID_MODULE_IMPORT};
+        """
+    )
+
+    # delete remaining rows with association between datasets and the import module
+    op.execute(
+        f"""
+        DELETE FROM gn_commons.cor_module_dataset
+        WHERE id_module = {ID_MODULE_IMPORT};
+        """
+    )
+
 
 def downgrade():
+
+    ## C IMPORT -> C Synthese
+    warnings.warn("!!!!! Created synthese permissions created previously will remain !!!!)")
+    ## JDD Synthese -> JDD Import
+    warnings.warn(
+        "Re-add association between datasets and the import module (!!!!! association between synthese and dataset created previously will remain! !!!!)"
+    )
+
+    ID_MODULE_IMPORT = (
+        op.get_bind()
+        .execute(
+            """
+        SELECT id_module FROM gn_commons.t_modules WHERE module_code = 'IMPORT';
+        """
+        )
+        .first()[0]
+    )
+    ID_MODULE_SYNTHESE = (
+        op.get_bind()
+        .execute(
+            """
+        SELECT id_module FROM gn_commons.t_modules WHERE module_code = 'SYNTHESE';
+        """
+        )
+        .first()[0]
+    )
+    op.execute(
+        f"""INSERT INTO gn_commons.cor_module_dataset (id_module, id_dataset) 
+        SELECT {ID_MODULE_IMPORT}, id_dataset FROM gn_commons.cor_module_dataset WHERE id_module = {ID_MODULE_SYNTHESE};
+        """
+    )
+
     meta = MetaData(bind=op.get_bind())
     # Remove new error types
     error_type = Table("bib_errors_types", meta, autoload=True, schema="gn_imports")

frontend/src/app/modules/imports/components/import_list/import-list.component.html

@@ -90,6 +90,9 @@ <h5 class="card-title mb-0">Liste des imports</h5>
                     {{ row[col.prop] }}
                   </a>
                 </ng-container>
+                <ng-container *ngSwitchCase="'statistics_rows'">
+                  {{ formattedRowCount(row) }}
+                </ng-container>
                 <ng-container *ngSwitchDefault>
                   {{ row[col.prop] }}
                 </ng-container>

frontend/src/app/modules/imports/components/import_list/import-list.component.ts

@@ -12,6 +12,7 @@ import { ImportProcessService } from '../import_process/import-process.service';
 import { Import } from '../../models/import.model';
 import { ConfigService } from '@geonature/services/config.service';
 import { CsvExportService } from '../../services/csv-export.service';
+import { formatRowCount } from '../../utils/format-row-count';
 
 @Component({
   styleUrls: ['import-list.component.scss'],
@@ -165,6 +166,10 @@ export class ImportListComponent implements OnInit {
     });
   }
 
+  formattedRowCount(row: Import): string {
+    return formatRowCount(row);
+  }
+
   openDeleteModal(row: Import, modalDelete) {
     this.deleteOne = row;
     this._ds.setDestination(row.destination.code);
@@ -207,7 +212,7 @@ export class ImportListComponent implements OnInit {
   getStatisticsTooltip(row) {
     const statistics = this._getStatistics(row);
     return Object.keys(statistics)
-      .map((statkey) => this.getStatisticsLabel(row, statkey) + ' : ' + statistics[statkey])
+      .map((statkey) => this.getStatisticsLabel(row, statkey) + ': ' + statistics[statkey])
       .join('\n');
   }
 

frontend/src/app/modules/imports/components/import_report/import_report.component.html

@@ -80,14 +80,10 @@ <h3>
               {{ importData?.format_source_file }}
             </p>
             <p>
-              <b>Nombre de lignes :</b>
-              {{ importData?.source_count || 0 }}
+              <b>Nombre de lignes importées :</b>
+              {{ formattedRowCount(importData) }}
             </p>
             <div *ngIf="importStatus === 'TERMINE'">
-              <p>
-                <b>Nombre d’entités importées :</b>
-                {{ importData?.import_count || 0 }}
-              </p>
               <div *ngFor="let item of importData?.destination.statistics_labels">
                 <ng-container *ngIf="importData?.statistics[item.key] !== null">
                   <p>
@@ -409,7 +405,7 @@ <h6>{{ importWarnings.length }} alerte(s)</h6>
               color="accent"
               (click)="_csvExport.onCSV(importData?.id_import)"
             >
-              Exporter vos {{ nbTotalErrors }} observations invalides
+              Exporter vos {{ nbTotalErrors }} lignes invalides
             </button>
           </div>
         </div>

frontend/src/app/modules/imports/components/import_report/import_report.component.ts

@@ -23,6 +23,7 @@ import { FieldMappingValues } from '../../models/mapping.model';
 
 import { HttpClient } from '@angular/common/http';
 import { finalize } from 'rxjs/operators';
+import { formatRowCount } from '../../utils/format-row-count';
 
 interface CorrespondancesField {
   source: string;
@@ -277,4 +278,8 @@ export class ImportReportComponent implements OnInit {
     });
     return mappedFields;
   }
+
+  formattedRowCount(row: Import): string {
+    return formatRowCount(row);
+  }
 }

frontend/src/app/modules/imports/utils/format-row-count.ts

@@ -0,0 +1,5 @@
+import { Import } from '../models/import.model';
+
+export function formatRowCount(imprt: Import): string {
+  return imprt && imprt.source_count ? `${imprt.import_count ?? 0} / ${imprt.source_count}` : '';
+}

frontend/src/styles.scss

@@ -53,10 +53,6 @@ mat-sidenav-content {
   height: 40px !important;
 }
 
-.mat-tooltip {
-  white-space: pre-line;
-}
-
 .app-content {
   //background-color: $solitude;
   //height: calc(100% - 5vh);
@@ -578,3 +574,14 @@ ngb-modal-backdrop {
 .mat-mdc-button-touch-target {
   display: none;
 }
+
+.cdk-overlay-pane {
+  &.mat-mdc-tooltip-panel {
+    .mat-mdc-tooltip {
+      div {
+        max-width: unset;
+      }
+      white-space: preserve nowrap;
+    }
+  }
+}