Refactor de l'indexation de planete PHP

Author: Mopolo

.github/workflows/ci.yml

@@ -32,7 +32,7 @@ jobs:
         run: composer install --no-scripts
 
       - name: Tests - Unit
-        run: ./bin/phpunit
+        run: ./bin/phpunit --testsuite unit
 
   lint:
     name: "Linter"
@@ -128,3 +128,27 @@ jobs:
         with:
           name: Functional tests - deprecated log - report
           path: var/logs/test.deprecations_grouped.log
+
+  integration:
+    name: "Integration tests"
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache Docker images.
+        uses: ScribeMD/docker-cache@0.5.0
+        with:
+          key: |
+            docker-${{ runner.os }}-${{ hashFiles(
+              'compose.yml',
+              'docker/dockerfiles/apachephp/Dockerfile',
+              'docker/dockerfiles/mysql/Dockerfile',
+              'docker/dockerfiles/mysqltest/Dockerfile'
+            ) }}
+
+      - name: Delete symfony cache
+        run: rm -rf var/cache/test
+
+      - name: Tests - Integration
+        run: make test-integration-ci

Makefile

@@ -65,9 +65,13 @@ console:
 
 ### (Dans Docker) Tests unitaires
 test:
-	./bin/phpunit
+	./bin/phpunit --testsuite unit
 	./bin/php-cs-fixer fix --dry-run -vv
 
+### (Dans Docker) Tests d'intégration
+test-integration:
+	./bin/phpunit --testsuite integration
+
 ### (Dans Docker) Tests fonctionnels
 behat:
 	./bin/behat
@@ -89,6 +93,13 @@ test-functional: data config htdocs/uploads tmp
 	make var/logs/test.deprecations_grouped.log
 	CURRENT_UID=$(CURRENT_UID) $(DOCKER_COMPOSE_BIN) stop dbtest apachephptest mailcatcher
 
+### Tests d'intégration avec start/stop des images docker
+test-integration-ci:
+	CURRENT_UID=$(CURRENT_UID) $(DOCKER_COMPOSE_BIN) stop dbtest apachephptest
+	CURRENT_UID=$(CURRENT_UID) $(DOCKER_COMPOSE_BIN) up -d dbtest apachephptest
+	CURRENT_UID=$(CURRENT_UID) $(DOCKER_COMPOSE_BIN) run --no-deps --rm -u localUser apachephp ./bin/phpunit --testsuite integration
+	CURRENT_UID=$(CURRENT_UID) $(DOCKER_COMPOSE_BIN) stop dbtest apachephptest
+
 ### Analyse PHPStan
 phpstan:
 	docker run -v $(shell pwd):/app --rm ghcr.io/phpstan/phpstan

app/config/services.yml

@@ -31,6 +31,9 @@ services:
         autowire: true
         autoconfigure: true
 
+    AppBundle\Clock\NativeClock: ~
+    Psr\Clock\ClockInterface: '@AppBundle\Clock\NativeClock'
+
     AppBundle\Command\UpdateMailchimpMembersCommand:
         autoconfigure: true
         autowire: true
@@ -716,10 +719,6 @@ services:
     PlanetePHP\FeedArticleRepository:
         autowire: true
 
-    PlanetePHP\FeedCrawler:
-        autowire: true
-        public: true
-
     AppBundle\Github\GithubClient:
         arguments:
             $githubClient: '@http_client.github'
@@ -728,3 +727,15 @@ services:
         arguments:
             $httpClient: '@http_client.meetup'
             $antennesCollection: '@AppBundle\Antennes\AntennesCollection'
+
+    PlanetePHP\SymfonyFeedClient:
+        arguments:
+            $httpClient: '@http_client'
+
+    PlanetePHP\FeedCrawler:
+        arguments:
+            $clock: '@Psr\Clock\ClockInterface'
+            $httpClient: '@PlanetePHP\SymfonyFeedClient'
+            $feedRepository: '@PlanetePHP\FeedRepository'
+            $feedArticleRepository: '@PlanetePHP\FeedArticleRepository'
+            $logger: '@logger'

clevercloud/cron.json

@@ -1,5 +1,5 @@
 [
-  "0 * * * * $ROOT/clevercloud/scripts/deduplicate_cron.sh /usr/bin/php $ROOT/htdocs/robots/planete/explorateur.php",
+  "0 * * * * $ROOT/clevercloud/scripts/deduplicate_cron.sh /usr/bin/php $ROOT/bin/console --env=prod indexing:planete",
   "0 5 * * * $ROOT/clevercloud/scripts/deduplicate_cron.sh /usr/bin/php $ROOT/bin/console --env=prod subscription:reminder",
   "0 23 * * * $ROOT/clevercloud/scripts/deduplicate_cron.sh /usr/bin/php $ROOT/bin/console --env=prod mailchimp:update-members",
   "0 22 * * * $ROOT/clevercloud/scripts/deduplicate_cron.sh /usr/bin/php $ROOT/bin/console --env=prod indexing:meetups --run-scraping",

composer.json

@@ -21,11 +21,11 @@
     "ekino/newrelic-bundle": "^2.4",
     "erusev/parsedown": "^1.6",
     "excelwebzone/recaptcha-bundle": "^1.5",
-    "ezyang/htmlpurifier": "^4.10",
     "friendsofpear/pear_exception": "0.0.*",
     "j7mbo/twitter-api-php": "^1.0",
     "jms/serializer-bundle": "5.*",
     "knpuniversity/oauth2-client-bundle": "^1.4",
+    "laminas/laminas-feed": "^2.18",
     "league/iso3166": "^4.0",
     "league/oauth2-github": "^0.2.1",
     "nojimage/twitter-text-php": "1.1.*",
@@ -34,6 +34,7 @@
     "phpmailer/phpmailer": "^6.9",
     "phpoffice/phpspreadsheet": "^1.6",
     "presta/sitemap-bundle": "3.3.0",
+    "psr/clock": "^1.0",
     "robmorgan/phinx": "^0.14.0",
     "sabre/vobject": "^4.1",
     "setasign/fpdf": "^1.8",
@@ -99,7 +100,9 @@
   "autoload-dev": {
     "psr-4": {
       "Afup\\Site\\Tests\\": "tests/unit/Afup/",
-      "AppBundle\\Tests\\": "tests/unit/AppBundle/"
+      "Afup\\Tests\\Support\\": "tests/support/",
+      "AppBundle\\Tests\\": "tests/unit/AppBundle/",
+      "PlanetePHP\\IntegrationTests\\": "tests/integration/PlanetePHP/"
     }
   },
   "license": "MIT",