chore(API Entreprise): Mise à jour des appels pour fonctionner sur la v3 de l'api

config/settings/base.py

@@ -507,7 +507,7 @@
 # API Entreprise.
 # https://dashboard.entreprise.api.gouv.fr/login (login is done through auth.api.gouv.fr)
 # https://doc.entreprise.api.gouv.fr/
-API_ENTREPRISE_BASE_URL = "https://entreprise.api.gouv.fr/v2"
+API_ENTREPRISE_BASE_URL = env.str("API_ENTREPRISE_BASE_URL", "https://entreprise.api.gouv.fr/v3")
 API_ENTREPRISE_CONTEXT = "emplois.inclusion.beta.gouv.fr"
 API_ENTREPRISE_RECIPIENT = env.str("API_ENTREPRISE_RECIPIENT", "")
 API_ENTREPRISE_TOKEN = env.str("API_ENTREPRISE_TOKEN", "")

env.default.sh

@@ -29,3 +29,7 @@ export OPENAI_API_BASE=""
 export OPENAI_API_KEY=""
 export OPENAI_MODEL=""
 
+# API Entreprise / see https://entreprise.api.gouv.fr/developpeurs#kit-de-mise-en-production
+export API_ENTREPRISE_RECIPIENT=""
+export API_ENTREPRISE_BASE_URL="https://staging.entreprise.api.gouv.fr/v3"
+export API_ENTREPRISE_TOKEN=""

env.docker_default.local

@@ -52,12 +52,10 @@ OPENAI_API_BASE=
 OPENAI_API_BASE=
 OPENAI_MODEL=
 
-# ELASTICSEARCH
-ELASTICSEARCH_HOST=
-ELASTICSEARCH_USERNAME=
-ELASTICSEARCH_PASSWORD=
-ELASTICSEARCH_INDEX_SIAES=
-ELASTICSEARCH_MIN_SCORE=0.9
-
 # DATACUBE API
 DATACUBE_API_TOKEN=
+
+# API Entreprise / see https://entreprise.api.gouv.fr/developpeurs#kit-de-mise-en-production
+API_ENTREPRISE_RECIPIENT=
+API_ENTREPRISE_BASE_URL="https://staging.entreprise.api.gouv.fr/v3"
+API_ENTREPRISE_TOKEN=

lemarche/siaes/management/commands/update_api_entreprise_fields.py

@@ -1,10 +1,18 @@
 import time
+from datetime import datetime
 
 from django.db.models import Q
+from django.utils import timezone
+from sentry_sdk.crons import monitor
 
 from lemarche.siaes.models import Siae
 from lemarche.utils.apis import api_slack
-from lemarche.utils.apis.api_entreprise import siae_update_entreprise, siae_update_etablissement, siae_update_exercice
+from lemarche.utils.apis.api_entreprise import (
+    API_ENTREPRISE_REASON,
+    entreprise_get_or_error,
+    etablissement_get_or_error,
+    exercice_get_or_error,
+)
 from lemarche.utils.commands import BaseCommand
 
 
@@ -33,6 +41,7 @@ def add_arguments(self, parser):
         parser.add_argument("--siret", type=str, default=None, help="Lancer sur un Siret spécifique")
         parser.add_argument("--limit", type=int, default=None, help="Limiter le nombre de structures à processer")
 
+    @monitor(monitor_slug="update-api-entreprise-fields")
     def handle(self, *args, **options):
         self.stdout_info("-" * 80)
         self.stdout_info("Populating API Entreprise fields...")
@@ -52,124 +61,83 @@ def handle(self, *args, **options):
         if options["limit"]:
             siae_queryset = siae_queryset[: options["limit"]]
 
-        # self.stdout_info(f"Found {siae_queryset.count()} Siae")
+        self._update_siae_api_entreprise_fields(siae_queryset, options["scope"])
 
-        if options["scope"] in ("all", "entreprise"):
-            self.update_api_entreprise_entreprise_fields(siae_queryset)
+    def _update_siae_api_entreprise_fields(self, siae_queryset, scope):
+        results = {
+            "entreprise": {"success": 0, "error": 0},
+            "etablissement": {"success": 0, "error": 0},
+            "exercice": {"success": 0, "error": 0},
+        }
 
-        if options["scope"] in ("all", "etablissement"):
-            self.update_api_entreprise_etablissement_fields(siae_queryset)
-
-        if options["scope"] in ("all", "exercice"):
-            self.update_api_entreprise_exercice_fields(siae_queryset)
-
-    # API Entreprise: entreprises
-    def update_api_entreprise_entreprise_fields(self, siae_queryset):
         progress = 0
-        results = {"success": 0, "error": 0}
-        siae_queryset_entreprise = siae_queryset.filter(api_entreprise_entreprise_last_sync_date=None)
-        self.stdout_info("-" * 80)
-        self.stdout_info(f"Populating 'entreprise' for {siae_queryset_entreprise.count()} Siae...")
-
-        for siae in siae_queryset_entreprise:
-            try:
-                progress += 1
-                if (progress % 50) == 0:
-                    self.stdout_info(f"{progress}...")
-                # self.stdout_info("-" * 80)
-                # self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}")
-                response, message = siae_update_entreprise(siae)
-                if response:
-                    results["success"] += 1
+        for siae in siae_queryset:
+            progress += 1
+            if (progress % 50) == 0:
+                self.stdout_info(f"{progress}...")
+
+            if not siae.siret:
+                self.stdout_error(f"SIAE {siae.id} without SIRET")
+                continue
+
+            update_data = dict()
+            if scope in ("all", "entreprise") and siae.api_entreprise_entreprise_last_sync_date is None:
+                entreprise, error = entreprise_get_or_error(siae.siret[:9], reason=API_ENTREPRISE_REASON)
+                if error:
+                    results["entreprise"]["error"] += 1
+                    self.stdout_error(str(error))
                 else:
-                    self.stdout_error(str(message))
-                    results["error"] += 1
-                # small delay to avoid going above the API limitation
-                # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints"
-                time.sleep(0.5)
-            except Exception as e:
-                self.stdout_error(str(e))
-                api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: entreprises")
-
-        msg_success = [
-            "----- Synchronisation API Entreprise (/entreprises) -----",
-            f"Done! Processed {siae_queryset_entreprise.count()} siae",
-            f"success count: {results['success']}/{siae_queryset_entreprise.count()}",
-            f"error count: {results['error']}/{siae_queryset_entreprise.count()} (voir les logs)",
-        ]
-        self.stdout_messages_success(msg_success)
-        api_slack.send_message_to_channel("\n".join(msg_success))
-
-    # API Entreprise: etablissements
-    def update_api_entreprise_etablissement_fields(self, siae_queryset):
-        progress = 0
-        results = {"success": 0, "error": 0}
-        siae_queryset_etablissement = siae_queryset.filter(api_entreprise_etablissement_last_sync_date=None)
-        self.stdout_info("-" * 80)
-        self.stdout_info(f"Populating 'etablissement' for {siae_queryset_etablissement.count()} Siae...")
-
-        for siae in siae_queryset_etablissement:
-            try:
-                progress += 1
-                if (progress % 50) == 0:
-                    self.stdout_info(f"{progress}...")
-                # self.stdout_info("-" * 80)
-                # self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}")
-                response, message = siae_update_etablissement(siae)
-                if response:
-                    results["success"] += 1
+                    results["entreprise"]["success"] += 1
+                    update_data["api_entreprise_forme_juridique"] = entreprise.forme_juridique
+                    update_data["api_entreprise_forme_juridique_code"] = entreprise.forme_juridique_code
+                    update_data["api_entreprise_entreprise_last_sync_date"] = timezone.now()
+
+            if scope in ("all", "etablissement") and siae.api_entreprise_etablissement_last_sync_date is None:
+                etablissement, error = etablissement_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON)
+                if error:
+                    results["etablissement"]["error"] += 1
+                    self.stdout_error(str(error))
+                else:
+                    results["etablissement"]["success"] += 1
+                    update_data["api_entreprise_employees"] = (
+                        etablissement.employees
+                        if (etablissement.employees != "Unités non employeuses")
+                        else "Non renseigné"
+                    )
+                    update_data["api_entreprise_employees_year_reference"] = etablissement.employees_date_reference
+                    update_data["api_entreprise_date_constitution"] = etablissement.date_constitution
+                    update_data["api_entreprise_etablissement_last_sync_date"] = timezone.now()
+
+            if scope in ("all", "exercice") and siae.api_entreprise_exercice_last_sync_date is None:
+                exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON)
+                if error:
+                    results["exercice"]["error"] += 1
+                    self.stdout_error(str(error))
                 else:
-                    self.stdout_error(str(message))
-                    results["error"] += 1
-                # small delay to avoid going above the API limitation
-                # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints"
-                time.sleep(0.5)
-            except Exception as e:
-                self.stdout_error(str(e))
-                api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: etablissements")
+                    results["exercice"]["success"] += 1
+                    update_data["api_entreprise_ca"] = exercice.chiffre_affaires
+                    update_data["api_entreprise_ca_date_fin_exercice"] = datetime.strptime(
+                        exercice.date_fin_exercice, "%Y-%m-%d"
+                    ).date()
+                    update_data["api_entreprise_exercice_last_sync_date"] = timezone.now()
 
-        msg_success = [
-            "----- Synchronisation API Entreprise (/etablissements) -----",
-            f"Done! Processed {siae_queryset_etablissement.count()} siae",
-            f"success count: {results['success']}/{siae_queryset_etablissement.count()}",
-            f"error count: {results['error']}/{siae_queryset_etablissement.count()} (voir les logs)",
-        ]
-        self.stdout_messages_success(msg_success)
-        api_slack.send_message_to_channel("\n".join(msg_success))
+            Siae.objects.filter(id=siae.id).update(**update_data)
 
-    # API Entreprise: exercices
-    def update_api_entreprise_exercice_fields(self, siae_queryset):
-        progress = 0
-        results = {"success": 0, "error": 0}
-        siae_queryset_exercice = siae_queryset.filter(api_entreprise_exercice_last_sync_date=None)
-        self.stdout_info("-" * 80)
-        self.stdout_info(f"Populating 'exercice' for {siae_queryset_exercice.count()} Siae...")
-
-        for siae in siae_queryset_exercice:
-            try:
-                progress += 1
-                if (progress % 50) == 0:
-                    self.stdout_info(f"{progress}...")
-                # self.stdout_info("-" * 80)
-                # self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}")
-                response, message = siae_update_exercice(siae)
-                if response:
-                    results["success"] += 1
-                else:
-                    self.stdout_error(str(message))
-                    results["error"] += 1
-                # small delay to avoid going above the API limitation
-                # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints"
-                time.sleep(0.5)
-            except Exception as e:
-                self.stdout_error(str(e))
-                api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: exercices")
+            # small delay to avoid going above the API limitation, one loop generates 3 requests
+            # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints"
+            time.sleep(1)
 
         msg_success = [
-            "----- Synchronisation API Entreprise (/exercices) -----",
-            f"Done! Processed {siae_queryset_exercice.count()} siae",
-            f"success count: {results['success']}/{siae_queryset_exercice.count()}",
-            f"error count: {results['error']}/{siae_queryset_exercice.count()} (voir les logs)",
+            "----- Synchronisation API Entreprise -----",
+            f"Done! Processed {siae_queryset.count()} siae",
+            "----- Success -----",
+            f"entreprise: {results['entreprise']['success']}/{siae_queryset.count()}",
+            f"etablissement: {results['etablissement']['success']}/{siae_queryset.count()}",
+            f"exercice: {results['exercice']['success']}/{siae_queryset.count()}",
+            "----- Error ----- (voir les logs)",
+            f"entreprise: {results['entreprise']['error']}/{siae_queryset.count()}",
+            f"etablissement: {results['etablissement']['error']}/{siae_queryset.count()}",
+            f"exercice: {results['exercice']['error']}/{siae_queryset.count()}",
         ]
         self.stdout_messages_success(msg_success)
         api_slack.send_message_to_channel("\n".join(msg_success))

lemarche/siaes/tests/test_commands.py

@@ -1,11 +1,14 @@
-import factory
+import datetime
+import json
 import logging
 import os
 from unittest.mock import patch
 
+import factory
 from django.core.management import call_command
 from django.db.models import signals
 from django.test import TransactionTestCase
+from django.utils import timezone
 
 from lemarche.perimeters.factories import PerimeterFactory
 from lemarche.perimeters.models import Perimeter
@@ -14,6 +17,11 @@
 from lemarche.siaes.factories import SiaeActivityFactory, SiaeFactory
 from lemarche.siaes.models import Siae, SiaeActivity
 from lemarche.users.factories import UserFactory
+from lemarche.utils.mocks.api_entreprise import (
+    MOCK_ENTREPRISE_API_DATA,
+    MOCK_ETABLISSEMENT_API_DATA,
+    MOCK_EXERCICES_API_DATA,
+)
 
 
 class SyncWithEmploisInclusionCommandTest(TransactionTestCase):
@@ -524,3 +532,55 @@ def test_update_count_fields_with_id(self):
         siae_not_updated.refresh_from_db()
         self.assertEqual(siae_not_updated.user_count, 0)
         self.assertEqual(siae_not_updated.sector_count, 0)
+
+
+class SiaeUpdateApiEntrepriseFieldsCommandTest(TransactionTestCase):
+    @patch("requests.get")
+    def test_siae_update_entreprise(self, mock_api):
+        mock_response = mock_api.return_value
+        mock_response.json.return_value = json.loads(MOCK_ENTREPRISE_API_DATA)
+        mock_response.status_code = 200
+
+        siae = SiaeFactory(siret="13002526500013")
+
+        call_command("update_api_entreprise_fields", scope="entreprise")
+
+        # Assert the updates
+        siae.refresh_from_db()
+        self.assertEqual(siae.api_entreprise_forme_juridique, "Service central d'un ministère")
+        self.assertEqual(siae.api_entreprise_forme_juridique_code, "7120")
+        self.assertLess((timezone.now() - siae.api_entreprise_entreprise_last_sync_date).total_seconds(), 60)
+
+    @patch("requests.get")
+    def test_siae_update_etablissement(self, mock_api):
+        mock_response = mock_api.return_value
+        mock_response.json.return_value = json.loads(MOCK_ETABLISSEMENT_API_DATA)
+        mock_response.status_code = 200
+
+        siae = SiaeFactory(siret="30613890001294")
+
+        call_command("update_api_entreprise_fields", scope="etablissement")
+
+        # Assert the updates
+        siae.refresh_from_db()
+        self.assertEqual(siae.siret, "30613890001294")
+        self.assertEqual(siae.api_entreprise_employees, "2 000 à 4 999 salariés")
+        self.assertEqual(siae.api_entreprise_employees_year_reference, "2016")
+        self.assertEqual(siae.api_entreprise_date_constitution, datetime.date(2021, 10, 13))
+        self.assertLess((timezone.now() - siae.api_entreprise_etablissement_last_sync_date).total_seconds(), 60)
+
+    @patch("requests.get")
+    def test_siae_update_exercice(self, mock_api):
+        mock_response = mock_api.return_value
+        mock_response.json.return_value = json.loads(MOCK_EXERCICES_API_DATA)
+        mock_response.status_code = 200
+
+        siae = SiaeFactory(siret="30613890001294")
+
+        call_command("update_api_entreprise_fields", scope="exercice")
+
+        # Assert the updates
+        siae.refresh_from_db()
+        self.assertEqual(siae.api_entreprise_ca, 900001)
+        self.assertEqual(siae.api_entreprise_ca_date_fin_exercice, datetime.date(2015, 12, 1))
+        self.assertLess((timezone.now() - siae.api_entreprise_exercice_last_sync_date).total_seconds(), 60)