Docker debian11

.github/workflows/dockers_builder.yml

@@ -8,16 +8,11 @@ on:
       - '*'
 
 env:
-
-  front_debian8_components: 'tyr-web instances-configurator'
-  backend_debian8_components: 'kraken tyr-beat tyr-worker mock-kraken eitri'
-
   SBX_ECR_REGISTRY_BACKEND: "110444322584.dkr.ecr.eu-west-1.amazonaws.com"
   SBX_ECR_REGISTRY_FRONT: "051314639660.dkr.ecr.eu-west-1.amazonaws.com"
   PRD_ECR_REGISTRY: "162230498103.dkr.ecr.eu-west-1.amazonaws.com"
 
 jobs:
-
   common_variables:
     name: Common variables
     runs-on: [self-hosted, corefront, sandbox]
@@ -55,84 +50,12 @@ jobs:
           echo '{"text":":warning: Github Actions: workflow dockers_builder common_variables failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}}
 
 
-  debian11_images:
-    runs-on: [self-hosted, corefront, sandbox]
-    name: Build debian11 images
-    needs: common_variables
-    steps:
-      - name: force chown to avoid errors
-        run: sudo chown -R $USER:$USER .
-
-      - name: Git config
-        run: git config --global --add safe.directory /__w/navitia/navitia
-
-      - name: Generate github private access token
-        id: ci-core-app-token
-        uses: getsentry/action-github-app-token@v2.0.0
-        with:
-          app_id: ${{ secrets.CI_CORE_APP_ID }}
-          private_key: ${{ secrets.CI_CORE_APP_PEM }}
-
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-          token: ${{ steps.ci-core-app-token.outputs.token }}
-
-      - name: Login to Amazon ECR
-        uses: aws-actions/amazon-ecr-login@v1
-        env:
-          AWS_REGION: eu-west-1
-        with:
-          # 162230498103 : shared
-          # 051314639660 : corefront sbx
-          registries: "162230498103,051314639660"
-
-      - name: Create builder docker
-        run: |
-          docker build -f docker/debian11/Dockerfile-builder -t navitia/builder_debian11 .
-
-      - name: Build navitia
-        run: |
-          docker run -v `pwd`:/navitia/navitia/  navitia/builder_debian11
-
-      - name: Create navitia images
-        run: |
-          echo "*********  Building Jormungandr ***************"
-          docker build -t navitia/jormungandr_debian11 -f  docker/debian11/Dockerfile-jormungandr .
-
-      - name: Push jormun dev image on SBX ECR
-        if: github.ref == 'refs/heads/dev'
-        run: |
-          jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_dev
-          docker tag navitia/jormungandr_debian11 ${jormun_tag}
-          docker push ${jormun_tag}
-
-      - name: Push jormun release image on SBX ECR
-        if: startsWith(github.ref, 'refs/tags/')
-        run: |
-          jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${{ needs.common_variables.outputs.RELEASE_TAG }}
-          docker tag navitia/jormungandr_debian11 ${jormun_tag}
-          docker push ${jormun_tag}
-
-
-      - name: failure notification
-        if: failure()
-        run: |
-          echo '{"text":":warning: Github Actions: workflow dockers_builder debian11_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}}
-
-      - name: clean up workspace
-        if: ${{ always() }}
-        run: |
-          # some files are created by a docker container
-          sudo chown -R $USER:$USER .
-          rm -rf ./*
-          rm -rf ./.??*
-
-  debian8_back_images:
+  debian11_back_images:
     runs-on: [self-hosted, kraken, sandbox]
-    name: Build debian8 back images
+    name: Build debian11 back images
     needs: common_variables
+    env:
+      backend_debian11_components: 'kraken tyr-beat tyr-worker mock-kraken eitri'
     steps:
       - name: force chown to avoid errors
         run: sudo chown -R $USER:$USER .
@@ -158,7 +81,7 @@ jobs:
         with:
           repo: 'hove-io/cosmogony2cities'
           version: 'tags/v0.1.1'
-          file: 'cosmogony2cities-debian8.deb'
+          file: 'cosmogony2cities-debian11.deb'
           target: 'cosmogony2cities_.deb'
           token: ${{ steps.ci-core-app-token.outputs.token }}
 
@@ -167,8 +90,8 @@ jobs:
         with:
           repo: 'hove-io/mimirsbrunn'
           version: 'tags/v3.1.0'
-          file: 'mimirsbrunn7_jessie-3.1.0.deb'
-          target: 'mimirsbrunn7_jessie-.deb'
+          file: 'mimirsbrunn7_bullseye-3.1.0.deb'
+          target: 'mimirsbrunn7_bullseye-.deb'
           token: ${{ steps.ci-core-app-token.outputs.token }}
 
       - name: Download mimir-config package
@@ -187,28 +110,27 @@ jobs:
         with:
           # 162230498103 : shared
           # 110444322584 : kraken sbx
+          # 051314639660: corefront sbx
           registries: "162230498103,110444322584,051314639660"
 
-      - name: Create master docker
+      - name: Create builder docker image
         run: |
-          docker build -f docker/debian8/Dockerfile-master -t navitia/master .
-          docker build -f docker/debian8/Dockerfile-builder -t navitia/builder .
+          docker build -f docker/debian11/Dockerfile-builder-kraken -t navitia/builder_debian11 .
 
       - name: Build packages in master docker
-        #  Will build navitia-*.deb packages in current folder
-        run: docker run -v `pwd`:/build/navitia/  navitia/builder
+        run: docker run -v `pwd`:/navitia/navitia/  navitia/builder_debian11
 
       - name: Create navitia images
         run: |
-          for component in ${{env.backend_debian8_components}}; do
+          for component in ${{env.backend_debian11_components}}; do
               echo "*********  Building $component ***************"
-              docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f  docker/debian8/Dockerfile-${component} .
+              docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f  docker/debian11/Dockerfile-${component} .
           done
 
       - name: Push dev images on SBX ECR
         if: github.ref == 'refs/heads/dev'
         run: |
-          for component in ${{env.backend_debian8_components}}; do
+          for component in ${{env.backend_debian11_components}}; do
             component_tag=${SBX_ECR_REGISTRY_BACKEND}/${component}:dev
             docker tag navitia/$component ${component_tag}
             docker push ${component_tag}
@@ -232,14 +154,11 @@ jobs:
           docker tag navitia/tyr-worker ${tyr_worker_tag}
           docker push ${tyr_worker_tag}
 
-
-
-
       - name: failure notification
         if: failure()
         run: |
           sudo apt update && sudo apt install -y httpie
-          echo '{"text":":warning: Github Actions: workflow dockers_builder debian8_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}}
+          echo '{"text":":warning: Github Actions: workflow dockers_builder debian11_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}}
 
       - name: clean up workspace
         if: ${{ always() }}
@@ -249,9 +168,9 @@ jobs:
           rm -rf ./*
           rm -rf ./.??*
 
-  debian8_front_images:
+  debian11_front_images:
     runs-on: [self-hosted, corefront, sandbox]
-    name: Build debian8 front images
+    name: Build debian11 front images
     needs: common_variables
     steps:
       - name: force chown to avoid errors
@@ -273,42 +192,63 @@ jobs:
           submodules: recursive
           token: ${{ steps.ci-core-app-token.outputs.token }}
 
-
       - name: Login to Amazon ECR
         uses: aws-actions/amazon-ecr-login@v1
         env:
           AWS_REGION: eu-west-1
         with:
           # 162230498103 : shared
           # 051314639660 : corefront sbx
-          registries: "162230498103,110444322584,051314639660"
+          registries: "162230498103,051314639660"
 
-      - name: Create master docker
+      - name: Create builder docker
         run: |
-          docker build -f docker/debian8/Dockerfile-master -t navitia/master .
-          docker build -f docker/debian8/Dockerfile-builder -t navitia/builder .
+          docker build -f docker/debian11/Dockerfile-builder -t navitia/builder .
 
-      - name: Build packages in master docker
-        #  Will build navitia-*.deb packages in current folder
-        run: docker run -v `pwd`:/build/navitia/  navitia/builder
+      - name: Build navitia
+        run: |
+          docker run -v `pwd`:/navitia/navitia/  navitia/builder
 
-      - name: Create navitia images
+      - name: Create docker image Jormungandr
         run: |
-          for component in ${{env.front_debian8_components}}; do
-              echo "*********  Building $component ***************"
-              docker build -t navitia/$component -f  docker/debian8/Dockerfile-${component} .
-          done
+          echo "*********  Building Jormungandr ***************"
+          docker build -t navitia/jormungandr_debian11 -f  docker/debian11/Dockerfile-jormungandr .
 
-      - name: Push dev images on SBX ECR
+      - name: Create images for Tyr-web
+        run: docker build -t navitia/tyr-web -f  docker/debian11/Dockerfile-tyr-web .
+
+      - name: Create images for Instance configurator
+        run: docker build -t navitia/instances-configurator -f  docker/debian11/Dockerfile-instances-configurator .
+
+      - name: Push Jormungandr dev image on SBX ECR
         if: github.ref == 'refs/heads/dev'
         run: |
-          for component in ${{env.front_debian8_components}}; do
-              component_tag=${SBX_ECR_REGISTRY_FRONT}/${component}:dev
-              docker tag navitia/$component ${component_tag}
-              docker push ${component_tag}
-          done
+          jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_dev
+          docker tag navitia/jormungandr_debian11 ${jormun_tag}
+          docker push ${jormun_tag}
 
-      - name: Push release images on PRD ECR
+      - name: Push Jormungandr release image on SBX ECR
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${{ needs.common_variables.outputs.RELEASE_TAG }}
+          docker tag navitia/jormungandr_debian11 ${jormun_tag}
+          docker push ${jormun_tag}
+
+      - name: Push Tyr-Web dev image on SBX ECR
+        if: github.ref == 'refs/heads/dev'
+        run: |
+          component_tag=${SBX_ECR_REGISTRY_FRONT}/tyr-web:dev
+          docker tag navitia/tyr-web ${component_tag}
+          docker push ${component_tag}
+
+      - name: Push Instances configurator dev image on SBX ECR
+        if: github.ref == 'refs/heads/dev'
+        run: |
+          component_tag=${SBX_ECR_REGISTRY_FRONT}/instances-configurator:dev
+          docker tag navitia/instances-configurator ${component_tag}
+          docker push ${component_tag}
+
+      - name: Push Try-web release image on PRD ECR
         if: startsWith(github.ref, 'refs/tags/')
         run: |
           # Tyr-web
@@ -319,8 +259,7 @@ jobs:
       - name: failure notification
         if: failure()
         run: |
-          sudo apt update && sudo apt install -y httpie
-          echo '{"text":":warning: Github Actions: workflow dockers_builder debian8_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}}
+          echo '{"text":":warning: Github Actions: workflow dockers_builder debian11_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}}
 
       - name: clean up workspace
         if: ${{ always() }}
@@ -330,12 +269,10 @@ jobs:
           rm -rf ./*
           rm -rf ./.??*
 
-
-
   publish_aws:
     runs-on: [self-hosted, corefront, sandbox]
     name: Aws Dispatch (Dev)
-    needs: [debian8_front_images, debian8_back_images, debian11_images, common_variables]
+    needs: [debian11_front_images, debian11_back_images, common_variables]
     steps:
 
       - name: Generate token for aws images

.github/workflows/workflow.yml

@@ -119,7 +119,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-            os: [{docker_image: mutable-debian8_dev , python_version: python3.6}, {docker_image: mutable-debian11_dev, python_version: python3.9}]
+            os: [{docker_image: mutable-debian11_dev, python_version: python3.9}]
 
     container:
         image: 162230498103.dkr.ecr.eu-west-1.amazonaws.com/${{ matrix.os.docker_image }}:latest
@@ -159,16 +159,6 @@ jobs:
         key: ${{matrix.os.docker_image}}-ci
         max-size: 2000M
         save: ${{ github.event_name == 'push' }}
-    - name: Ensure CMake version is recent enough
-      if: ${{ matrix.os.docker_image == 'mutable-debian8_dev'}}
-      run: |
-        apt remove --yes cmake
-        mkdir local_bin
-        cd local_bin
-        wget https://cmake.org/files/v3.18/cmake-3.18.6-Linux-x86_64.tar.gz
-        tar xf cmake-3.18.6-Linux-x86_64.tar.gz
-        echo "`pwd`/cmake-3.18.6-Linux-x86_64/bin" >> $GITHUB_PATH
-        cd ..
     - name: configure for Release
       run: mkdir build && cd ./build && cmake -DSTRIP_SYMBOLS=ON ../source
     - name: run

docker/build_kraken_deb11.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+cd /navitia/navitia/
+mkdir -p docker_build && cd docker_build
+cmake -DCMAKE_BUILD_TYPE=Release ../source
+
+# kraken
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) protobuf_files
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) kraken
+
+# mocks for integration tests
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) departure_board_test
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) main_routing_test
+
+# some binaries for tyr-worker
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) cities
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) ed2nav
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) fare2ed
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) fusio2ed
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) geopal2ed
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) gtfs2ed
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) osm2ed
+make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) poi2ed
+
+# Build libkeepalive https://libkeepalive.sourceforge.net/
+# libkeepalive is a library to keep tcp connection alive. The reason of doing so is that aws LoadBalancer, to which
+# kraken is connected, check the aliveness continuously at regular interval of non-customisable 350s
+# (https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html). If nothing happens during
+# this interval, the connection will be dropped. It takes Kraken, by default, 7200s(/proc/sys/net/ipv4/tcp_keepalive_time)
+# to be aware of the disconnection. By using libkeepalive, we can override tcp_keepalive_time,
+# which is not customisable via docker-container.
+(mkdir -p build_libkeepalive && cp -rf ../docker/libkeepalive-0.3/. build_libkeepalive/ && cd build_libkeepalive/ &&   make )

docker/debian11/Dockerfile-builder-kraken

@@ -0,0 +1,10 @@
+FROM 162230498103.dkr.ecr.eu-west-1.amazonaws.com/mutable-debian11_dev:latest
+
+RUN apt-get update && apt-get install -y git wget
+
+RUN git config --global --add safe.directory /navitia/navitia
+
+COPY docker/build_kraken_deb11.sh /build_navitia.sh
+RUN chmod +x /build_navitia.sh
+
+ENTRYPOINT ["bash", "/build_navitia.sh"]

docker/debian11/Dockerfile-eitri

@@ -0,0 +1,27 @@
+FROM debian:bullseye-slim
+
+COPY source/ /navitia/source/
+
+RUN apt-get update --fix-missing \
+    && apt-get install -y --force-yes libpq5 python3.9 python3-pip git zip jq curl \
+    libboost-all-dev \
+    libprotobuf-dev \
+    liblog4cplus-2.0.5 \
+    libzmq3-dev \
+    libpqxx-dev \
+    libgoogle-perftools4 \
+    libosmpbf-dev \
+    libproj-dev \
+    && pip3 install --no-cache-dir -U -r /navitia/source/eitri/requirements.txt\
+    && apt-get autoremove -y
+
+# binaries from navitia
+COPY ./docker_build/ed/ed2nav /usr/bin/ed2nav
+COPY ./docker_build/ed/fare2ed /usr/bin/fare2ed
+COPY ./docker_build/ed/fusio2ed /usr/bin/fusio2ed
+COPY ./docker_build/ed/geopal2ed /usr/bin/geopal2ed
+COPY ./docker_build/ed/gtfs2ed /usr/bin/gtfs2ed
+COPY ./docker_build/ed/osm2ed /usr/bin/osm2ed
+COPY ./docker_build/ed/poi2ed /usr/bin/poi2ed
+
+CMD [""]

docker/debian11/Dockerfile-instances-configurator

@@ -0,0 +1,17 @@
+FROM debian:bullseye-slim
+
+
+COPY ./source/sql/alembic /usr/share/navitia/ed/alembic
+COPY ./source/sql/requirements.txt /tmp/requirements.txt
+COPY ./source/cities /usr/share/navitia/cities
+COPY ./docker/templates/* /templates/
+
+COPY docker/instances_configuration.sh /
+RUN chmod +x /instances_configuration.sh
+
+RUN apt-get update --fix-missing \
+    && apt-get install -y --force-yes libpq5 python3.9-dev python3-pip postgresql-client gettext-base \
+    && pip3 install --no-cache-dir -U -r /tmp/requirements.txt \
+    && apt-get autoremove -y
+
+ENTRYPOINT ["/bin/bash","/instances_configuration.sh"]

docker/debian11/Dockerfile-kraken

@@ -0,0 +1,51 @@
+FROM debian:bullseye-slim
+
+WORKDIR /srv/kraken
+
+COPY ./docker/ca-certificates/*.crt /usr/local/share/ca-certificates/
+COPY ./source/monitor /srv/monitor_kraken
+
+RUN apt-get update --fix-missing \
+    && apt-get install -y ca-certificates \
+    && update-ca-certificates \
+    && apt-get install -y libgoogle-perftools4 \
+    python3.9 \
+    python3-pip \
+    libboost-all-dev \
+    libprotobuf-dev \
+    liblog4cplus-2.0.5 \
+    libzmq3-dev \
+    libpqxx-dev \
+    netcat \
+    && (cd /srv/monitor_kraken && pip3 install --no-cache-dir -r requirements.txt && python3 setup.py install) \
+    && apt-get purge -y \
+    && apt-get purge -y python3-pip \
+    && apt-get autoremove -y
+
+RUN ln -s /usr/local/lib/python*/dist-packages/monitor_kraken/app.py monitor_kraken
+
+COPY docker/run_kraken_deb11.sh /srv/kraken/run_kraken.sh
+
+RUN chmod +x /srv/kraken/run_kraken.sh
+
+COPY ./docker_build/kraken/kraken /usr/bin/kraken
+RUN chmod +x /usr/bin/kraken
+
+RUN mkdir /libkeepalive
+COPY ./docker_build/build_libkeepalive/libkeepalive.so /libkeepalive/.
+
+# User timeout = 300 + 30 * 6 = 480s
+ENV KEEPIDLE=300
+ENV KEEPCNT=6
+ENV KEEPINTVL=30
+# if you are willing to use settings from os, set KEEPALIVE to off
+# ENV KEEPALIVE=Off
+
+ENV KRAKEN_GENERAL_zmq_socket=tcp://0.0.0.0:30000
+ENV KRAKEN_GENERAL_log_level=INFO
+HEALTHCHECK CMD nc -z localhost 30000 || exit 1
+EXPOSE 30000
+EXPOSE 5000
+
+
+ENTRYPOINT ["bash", "/srv/kraken/run_kraken.sh"]

docker/debian11/Dockerfile-mock-kraken

@@ -0,0 +1,24 @@
+FROM debian:bullseye-slim
+
+WORKDIR /srv/kraken
+
+RUN apt-get update --fix-missing \
+    && apt-get install -y libgoogle-perftools4 \
+    libboost-all-dev \
+    libprotobuf-dev \
+    liblog4cplus-2.0.5 \
+    libzmq3-dev \
+    libpqxx-dev \
+    netcat \
+    && apt-get purge -y \
+    && apt-get autoremove -y
+
+COPY ./docker_build/tests/mock-kraken/departure_board_test /srv/kraken/departure_board_test
+RUN chmod +x /srv/kraken/departure_board_test
+
+COPY ./docker_build/tests/mock-kraken/main_routing_test /srv/kraken/main_routing_test
+RUN chmod +x /srv/kraken/main_routing_test
+
+EXPOSE 30000
+ENV KRAKEN_GENERAL_zmq_socket=tcp://*:30000
+ENV KRAKEN_GENERAL_log_level=INFO

docker/debian11/Dockerfile-tyr-beat

@@ -5,22 +5,21 @@ WORKDIR /usr/share/tyr/
 COPY ./source/navitiacommon ./navitiacommon
 COPY ./source/tyr ./tyr
 COPY ./docker/ca-certificates/*.crt /usr/local/share/ca-certificates/
-COPY ./docker/run_tyr_beat.sh ./run.sh
+COPY ./docker/run_tyr_beat.sh /run.sh
 COPY ./source/sql/alembic /usr/share/navitia/ed/alembic
 COPY ./source/sql/requirements.txt /tmp/requirements.txt
 COPY ./source/cities /usr/share/navitia/cities
 COPY ./docker/templates/* /templates/
 
+RUN chmod +x /run.sh
+
 RUN apt-get update --fix-missing \
-    && apt-get install -y libpq5 python3.9-dev python3-pip git ca-certificates  postgresql-client gettext-base jq \
+    && apt-get install -y --force-yes libgeos-dev libpq5 python3.9-dev python3-pip git ca-certificates postgresql-client gettext-base jq \
     && update-ca-certificates \
     && (cd navitiacommon && python3 setup.py install) \
     && (cd tyr && python3 setup.py install && pip3 install --no-cache-dir -U -r requirements.txt)\
     && pip3 install --no-cache-dir -U -r /tmp/requirements.txt \
-    && rm -rf navitiacommon tyr \
-    && apt-get purge -y \
-        python3-pip \
-        git \
+    && apt-get purge -y git \
     && apt-get autoremove -y
 
-ENTRYPOINT ["bash", "/usr/share/tyr/run.sh"]
+ENTRYPOINT ["bash", "/run.sh"]

docker/debian11/Dockerfile-tyr-web

@@ -7,14 +7,14 @@ COPY ./source/tyr ./tyr
 COPY ./docker/ca-certificates/*.crt /usr/local/share/ca-certificates/
 COPY ./docker/run_tyr_web.sh /usr/src/app/run.sh
 
+RUN apt-get update --force-yes --fix-missing || exit 0
 
-RUN apt-get update --fix-missing \
-    && apt-get install -y curl libpq5 python3.9-dev python3-pip git ca-certificates libgeos-c1v5 postgresql-client \
+RUN apt-get install -y --force-yes curl libpq5 python3.9-dev python3-pip git ca-certificates libgeos-c1v5 postgresql-client \
     && update-ca-certificates \
     && (cd navitiacommon && python3 setup.py install) \
     && (cd tyr && python3 setup.py install && pip3 install --no-cache-dir -U -r requirements.txt)\
-    && pip3 install --no-cache-dir uwsgi==2.0.21 \
-    && rm -rf navitiacommon tyr \
+    && pip3 install --no-cache-dir uwsgi==2.0.22 \
+    && pip3 install --no-cache-dir -r /usr/share/tyr/requirements.txt \
     && chmod +x /usr/src/app/run.sh \
     && ln -sf /usr/share/tyr/migrations migrations \
     && ln -s /usr/bin/python3.9 /usr/bin/python \
@@ -24,9 +24,6 @@ RUN apt-get update --fix-missing \
         git \
     && apt-get autoremove -y
 
-
- # Python 'requests' package handle its own CA certificate list
- # Let's force it to use the OS's list
- ENV REQUESTS_CA_BUNDLE /etc/ssl/certs
+ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs
 
 ENTRYPOINT ["bash", "/usr/src/app/run.sh" ]

docker/debian11/Dockerfile-tyr-worker

@@ -0,0 +1,66 @@
+# Compilation of rust binaries from tartare-tools
+FROM rust:1.82-bullseye AS tartare-tools
+
+ARG TARTARE_TOOLS_VERSION="v0.47.1"
+ARG GITHUB_TOKEN
+RUN git config --global url."https://x-access-token:${GITHUB_TOKEN}@github.com/hove-io/".insteadOf "ssh://git@github.com/hove-io/"
+RUN git clone -b ${TARTARE_TOOLS_VERSION} --depth 1 https://x-access-token:${GITHUB_TOKEN}@github.com/hove-io/tartare-tools
+
+RUN cd tartare-tools \
+    && cargo  build --release -p enrich-ntfs-with-addresses \
+    && cargo  build --release -p split-trip-geometries \
+    && cp target/release/enrich-ntfs-with-addresses /usr/bin/ \
+    && cp target/release/split-trip-geometries /usr/bin/ \
+    && cd .. \
+    && rm -rf tartare-tools
+
+FROM debian:bullseye-slim
+
+WORKDIR /usr/src/app/
+
+COPY ./source/navitiacommon ./navitiacommon
+COPY ./source/tyr ./tyr
+COPY ./docker/run_tyr_worker.sh /usr/src/app/run.sh
+RUN chmod +x /usr/src/app/run.sh
+
+COPY mimirsbrunn7_bullseye-*.deb  cosmogony2cities_*.deb  mimirsbrunn-config-*.deb ./
+
+RUN apt-get update --fix-missing \
+    && apt-get install -y --force-yes unzip curl libgeos-dev libpq5 python3.9 python3-pip git ca-certificates postgresql-client gettext-base jq \
+    libboost-all-dev \
+    libprotobuf-dev \
+    liblog4cplus-2.0.5 \
+    libzmq3-dev \
+    libpqxx-dev \
+    libgoogle-perftools4 \
+    libosmpbf-dev \
+    libproj-dev \
+    ./mimirsbrunn7_bullseye-*.deb ./cosmogony2cities_*.deb ./mimirsbrunn-config-*.deb \
+    && update-ca-certificates \
+    && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
+    && unzip awscliv2.zip && ./aws/install \
+    && (cd navitiacommon && python3 setup.py install) \
+    && (cd tyr && python3 setup.py install && pip3 install --no-cache-dir -U -r requirements.txt)\
+    && rm -rf ./mimirsbrunn7_bullseye-*.deb ./cosmogony2cities*.deb ./mimirsbrunn-config-*.deb \
+    && apt-get purge -y git python3-pip \
+    && apt-get autoremove -y
+
+# binaries from tartare-tools
+COPY --from=tartare-tools /usr/bin/enrich-ntfs-with-addresses /usr/bin/
+COPY --from=tartare-tools /usr/bin/split-trip-geometries /usr/bin/
+
+# binaries from navitia
+COPY ./docker_build/cities/cities /usr/bin/cities
+COPY ./docker_build/ed/ed2nav /usr/bin/ed2nav
+COPY ./docker_build/ed/fare2ed /usr/bin/fare2ed
+COPY ./docker_build/ed/fusio2ed /usr/bin/fusio2ed
+COPY ./docker_build/ed/geopal2ed /usr/bin/geopal2ed
+COPY ./docker_build/ed/gtfs2ed /usr/bin/gtfs2ed
+COPY ./docker_build/ed/osm2ed /usr/bin/osm2ed
+COPY ./docker_build/ed/poi2ed /usr/bin/poi2ed
+
+# TODO change the user to remove this ugly C_FORCE_ROOT
+ENV C_FORCE_ROOT=1
+ENV TYR_WORKER_N_PROC=1
+
+ENTRYPOINT ["bash", "/usr/src/app/run.sh"]

docker/run_kraken_deb11.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python3 monitor_kraken & LD_PRELOAD=/libkeepalive/libkeepalive.so /usr/bin/kraken

docker/run_tyr_beat.sh

@@ -63,4 +63,4 @@ while read var ; do
 done < <(env)
 
 
-exec celery beat -A tyr.tasks
+exec celery -A tyr.tasks beat

docker/run_tyr_worker.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-celery worker -A tyr.tasks -O fair -c $TYR_WORKER_N_PROC
+celery -A tyr.tasks worker -O fair -c $TYR_WORKER_N_PROC

source/kraken/CMakeLists.txt

@@ -36,7 +36,7 @@ target_link_libraries(fill_disruption_from_database
 
 add_executable(kraken kraken_zmq.cpp)
 target_link_libraries(kraken workers ${NAVITIA_ALLOCATOR} ${Boost_THREAD_LIBRARY})
-add_dependencies(kraken protobuf_files)
+add_dependencies(kraken protobuf_files prometheus-cpp::pull)
 
 install(TARGETS kraken DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
 

source/monitor/monitor_kraken/app.py

@@ -29,14 +29,18 @@
 
 from flask import Flask, request
 import json
-from ConfigParser import ConfigParser
 import zmq
 import os
 
 from monitor_kraken import request_pb2
 from monitor_kraken import response_pb2
 from monitor_kraken import type_pb2
 
+try:
+    import ConfigParser
+except ImportError:
+    import configparser as ConfigParser  # type: ignore
+
 app = Flask(__name__)
 # config used in monitor() for several kraken
 app.config.from_object('monitor_kraken.default_settings')
@@ -217,4 +221,4 @@ def request_kraken_zmq_status(zmq_socket, zmq_timeout_in_ms):
 
 
 if __name__ == '__main__':
-    app.run(host=os.environ.get("MONITOR_HOST", "127.0.0.1"))
+    app.run(host=os.environ.get("MONITOR_HOST", "0.0.0.0"))

source/tyr/tyr/binarisation.py

@@ -184,7 +184,7 @@ def wrapper(*args, **kwargs):
             logging.debug('args: %s -- kwargs: %s', args, kwargs)
             job = models.Job.query.get(job_id)
             logger = get_instance_logger(job.instance, task_id=job_id)
-            task = args[func.func_code.co_varnames.index('self')]
+            task = args[func.__code__.co_varnames.index('self')]
             try:
                 lock = redis.lock('tyr.lock|' + job.instance.name, timeout=self.timeout)
                 locked = lock.acquire(blocking=False)

source/tyr/tyr/helper.py

@@ -337,7 +337,7 @@ def get_named_arg(arg_name, func, args, kwargs):
     if kwargs and arg_name in kwargs:
         return kwargs[arg_name]
     else:
-        idx = func.func_code.co_varnames.index(arg_name)
+        idx = func.__code__.co_varnames.index(arg_name)
         if args and idx < len(args):
             return args[idx]
         else: