migration: postgres (#470)

* start migration * test * add varchar * delete export * delete dead package.json * dead code * link postgres to var env * update voie service * update numero service * add findCentroid * stash * change to uuid * refacto voie service * avance on numero.service * finish numeros and voie * fix DTO * migration toponyme service * base locales progress * finish service baseLocale * correct import voie * import toponyme * import many numero * progress * finish voie service * migration export csv * migration module habilitation * finish without stat * change geom srid postgis * finish stats modules * correct find distinct parcelles * change numeros * change entities api property * object id * correct * c * add banId * correct server tiles * correct extend * replace uuid to objectId * correct insert/update positions * correct import many * add import positions * create/update voie * create/update toponyme * optional function in entities * correct batch numeros toponyme * correct create update numero * correct delete numero * soft delete toponyme * correct soft delete voie * correct route find all bal deleted * correct trash voie * delete batch * correct batch * correct import many toponyme * start test * correct test bases_locales * correct test toponyme * correct test voie * correct test numeros * correct test habilitation * correct test publication * add test stats * correct test ci * correct build node 22 * correct await * correct connection and migration typeorm * start cron * add cache to cron * correct test * finish cron * finish test cron * test cron * correct build cron * correct cache string redis * index and rank * add migration to module * test email to * correct rank * correct mail * correct return bug jules * delete schemas * add migration in code * delete bal_id to numero entity * correct test * correct lint * order numeros * correct batch numero -> toponyme null * correct order csv liste voies * correct bug date updatedAt and sync.currentUpdated * correct test * correct bases_locales updateOne * correct test * refacto migration initialization * update README * ci: correct coverage soundcloud * correct alert souncloud * correct alert souncloud part 2 * correct object id mongo length in postgres * delete mongoose * email case insencitive * change migration * change index * correct mail balId * add space in numero and suffixe in numeroComplet
BaseAdresseNationale · Sep 10, 2024 · 925ddfb · 925ddfb
1 parent 5d8c3cb
commit 925ddfb
Show file tree

Hide file tree

Showing 122 changed files with 5,022 additions and 6,006 deletions.
diff --git a/.env.sample b/.env.sample
@@ -1,15 +1,19 @@
-MONGODB_URL=mongodb://localhost:27017
-MONGODB_DBNAME=api-bal
+
+POSTGRES_URL=
+REDIS_URL=
+
 PORT=5000
+
 SMTP_HOST=
 SMTP_PORT=587
 SMTP_USER=
 SMTP_PASS=
 SMTP_SECURE=
 SMTP_FROM=
 SMTP_BCC=
-SHOW_EMAILS=YES
+
 EDITOR_URL_PATTERN=https://mes-adresses.data.gouv.fr/bal/<id>/<token>
+
 API_URL=http://localhost:5000
 API_DEPOT_URL=https://plateforme-bal.adresse.data.gouv.fr/api-depot
 API_DEPOT_CLIENT_ID=

diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml
@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x, 18.x, 20.x, 22.x]
+        node-version: [18.x, 20.x, 22.x]
 
     steps:
       - uses: actions/checkout@v3
@@ -50,8 +50,8 @@ jobs:
           fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
       - name: Install dependencies
         run: yarn
-      - name: Test and coverage
-        run: yarn jest --coverage
+      # - name: Test and coverage
+      #   run: yarn jest --coverage
       - name: SonarCloud Scan
         uses: SonarSource/sonarcloud-github-action@master
         env:

diff --git a/Procfile b/Procfile
@@ -0,0 +1 @@
+clock: yarn start:cron
diff --git a/README.md b/README.md
@@ -11,9 +11,18 @@ https://adresse-data-gouv-fr.gitbook.io/bal/mes-adresses
 
 ## Pré-requis
 
-- [Node.js](https://nodejs.org) 16+
+- [Node.js](https://nodejs.org) 18+
 - [yarn](https://www.yarnpkg.com)
-- [MongoDB](https://www.mongodb.com) 4+
+- [PostgresSQL](https://www.postgresql.org/)
+- [Redis](https://redis.io/fr/)
+
+### Postgres
+
+Le base de donnée postgres doit avoir l'extension postgis d'installé
+
+```
+CREATE EXTENSION postgis
+```
 
 ## Utilisation
 
@@ -47,7 +56,6 @@ Lancer le cron de développement :
 $ yarn dev:cron
 ```
 
-
 ### Production
 
 Créer une version de production :
@@ -98,17 +106,17 @@ Cette application utilise des variables d'environnement pour sa configuration.
 Elles peuvent être définies classiquement ou en créant un fichier `.env` sur la base du modèle `.env.sample`.
 
 | Nom de la variable        | Description                                                                 |
-| --------------------------| --------------------------------------------------------------------------- |
-| `MONGODB_URL`             | Paramètre de connexion à MongoDB                                            |
-| `MONGODB_DBNAME`          | Nom de la base de données à utiliser                                        |
+| ------------------------- | --------------------------------------------------------------------------- |
+| `POSTGRES_URL`            | Url de connection a la db postgres                                          |
+| `REDIS_URL`               | Url de connection a la db redis                                             |
 | `PORT`                    | Port à utiliser pour l'API                                                  |
-| `SHOW_EMAILS`             | Indique si les courriels doivent être affichés dans les logs (`YES`)        |
 | `API_URL`                 | URL de base de l’API                                                        |
 | `API_DEPOT_URL`           | URL de l'api-depot                                                          |
 | `API_DEPOT_CLIENT_ID`     | Id du client de l'api-depot                                                 |
 | `API_DEPOT_CLIENT_SECRET` | Token du client de l'api-depot                                              |
 | `EDITOR_URL_PATTERN`      | Pattern permettant de construire l'URL vers l'édition d'une BAL             |
-|---|---|
+| `BAN_API_URL`             | URL de ban-plateform                                                        |
+| ---                       | ---                                                                         |
 | `SMTP_HOST`               | Nom d'hôte du serveur SMTP                                                  |
 | `SMTP_PORT`               | Port du serveur SMTP                                                        |
 | `SMTP_USER`               | Nom d'utilisateur pour se connecter au serveur SMTP                         |

diff --git a/apps/api/src/api.module.ts b/apps/api/src/api.module.ts
@@ -1,31 +1,41 @@
 import { ConfigModule, ConfigService } from '@nestjs/config';
 import { Module } from '@nestjs/common';
-import { MongooseModule } from '@nestjs/mongoose';
+import { TypeOrmModule } from '@nestjs/typeorm';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import { join } from 'path';
 
+import { Initialization1725371358514 } from 'migrations/1725371358514-initialization';
+import { BaseLocale } from '@/shared/entities/base_locale.entity';
+import { Voie } from '@/shared/entities/voie.entity';
+import { Numero } from '@/shared/entities/numero.entity';
+import { Toponyme } from '@/shared/entities/toponyme.entity';
+import { Position } from '@/shared/entities/position.entity';
+
 import { NumeroModule } from './modules/numeros/numero.module';
 import { BaseLocaleModule } from './modules/base_locale/base_locale.module';
 import { VoieModule } from './modules/voie/voie.module';
 import { ToponymeModule } from './modules/toponyme/toponyme.module';
 import { StatsModule } from './modules/stats/stats.module';
-import { AdminModule } from './modules/admin/admin.module';
-import { CronModule } from 'apps/cron/src/cron.module';
 import { MailerModule } from '@nestjs-modules/mailer';
 import { MailerParams } from '@/shared/params/mailer.params';
 
 @Module({
   imports: [
+    ConfigModule.forRoot(),
     ServeStaticModule.forRoot({
       rootPath: join(__dirname, '../../../'),
       renderPath: 'public/',
     }),
-    ConfigModule.forRoot(),
-    MongooseModule.forRootAsync({
+    TypeOrmModule.forRootAsync({
       imports: [ConfigModule],
       useFactory: async (config: ConfigService) => ({
-        uri: config.get('MONGODB_URL'),
-        dbName: config.get('MONGODB_DBNAME'),
+        type: 'postgres',
+        url: config.get('POSTGRES_URL'),
+        keepConnectionAlive: true,
+        schema: 'public',
+        migrationsRun: true,
+        migrations: [Initialization1725371358514],
+        entities: [BaseLocale, Voie, Numero, Toponyme, Position],
       }),
       inject: [ConfigService],
     }),
@@ -35,9 +45,6 @@ import { MailerParams } from '@/shared/params/mailer.params';
     VoieModule,
     ToponymeModule,
     StatsModule,
-    AdminModule,
-    // We run the cron module in the api module when deployed on Scalingo
-    ...(process.env.RUN_CRON_IN_API === 'true' ? [CronModule] : []),
   ],
   controllers: [],
   providers: [],

diff --git a/apps/api/src/lib/types/request.type.ts b/apps/api/src/lib/types/request.type.ts
@@ -1,9 +1,9 @@
 import { Request } from 'express';
 
-import { Numero } from '@/shared/schemas/numero/numero.schema';
-import { Voie } from '@/shared/schemas/voie/voie.schema';
-import { Toponyme } from '@/shared/schemas/toponyme/toponyme.schema';
-import { BaseLocale } from '@/shared/schemas/base_locale/base_locale.schema';
+import { Numero } from '@/shared/entities/numero.entity';
+import { Voie } from '@/shared/entities/voie.entity';
+import { Toponyme } from '@/shared/entities/toponyme.entity';
+import { BaseLocale } from '@/shared/entities/base_locale.entity';
 
 export interface CustomRequest extends Request {
   token?: string;

diff --git a/apps/api/src/lib/types/validator.types.ts b/apps/api/src/lib/types/validator.types.ts
@@ -0,0 +1,72 @@
+export enum LevelEnum {
+  INFO = 'I',
+  WARNING = 'W',
+  ERROR = 'E',
+}
+
+export interface Field {
+  name: string;
+  schemaName: string;
+}
+
+export interface FieldNotFound {
+  schemaVersion: string;
+  schemaName: string;
+  level: LevelEnum;
+}
+
+export interface Row {
+  rawValues: any;
+  parsedValues: any;
+  additionalValues: any;
+  localizedValues: any;
+  errors: any[];
+  line: number;
+  isValid: boolean;
+}
+
+export interface FieldFileValidation {
+  value: string;
+  isValid: boolean;
+}
+
+export interface FileValidation {
+  encoding: FieldFileValidation;
+  delimiter: FieldFileValidation;
+  linebreak: FieldFileValidation;
+}
+export interface ProfileValidation {
+  code: string;
+  name: string;
+  isValid: boolean;
+}
+
+export interface ProfileError {
+  code: string;
+  level: LevelEnum;
+}
+
+export interface ParseError {
+  type: string;
+  code: string;
+  message: string;
+  row: number;
+}
+
+export interface ValidationBal {
+  encoding: string;
+  linebreak: string;
+  delimiter: string;
+  originalFields: string[];
+  parseOk: boolean;
+  parseErrors: ParseError[];
+  fields: Field[];
+  notFoundFields: FieldNotFound[];
+  rows: Row[];
+  fileValidation: FileValidation;
+  profilesValidation: Record<string, ProfileValidation>;
+  globalErrors: string[];
+  rowsErrors: string[];
+  uniqueErrors: string[];
+  profilErrors: ProfileError[];
+}