Merge branch 'main' into feature/trunk-ci-debug

Author: tcarmet

.devcontainer/devcontainer.json

@@ -63,7 +63,7 @@
     }
   },
   "remoteUser": "vscode",
-  "postCreateCommand": "poetry install --no-interaction --no-ansi --no-root --group=docs --group=dev",
+  "postCreateCommand": "poetry install --no-interaction --no-ansi --no-root --with docs,dev",
   "postAttachCommand": {},
   "portsAttributes": {
     "6379": {

.github/workflows/build.yaml

@@ -19,5 +19,5 @@ jobs:
     uses: scality/workflows/.github/workflows/docker-build.yaml@v2
     with:
       name: runner-manager
-      namespace: scality
+      namespace: ${{ github.repository_owner }}
       tag: ${{ github.ref_name == 'main' && 'latest' || github.sha }}

.github/workflows/test-deployment.yaml

@@ -4,17 +4,19 @@ name: test-deployment
 on: pull_request
 
 permissions:
-  contents: read
+  contents: write
+  packages: write
 
 jobs:
   docker-build:
     permissions:
-      contents: read
+      contents: write
       packages: write
     uses: scality/workflows/.github/workflows/docker-build.yaml@v2
     with:
       name: runner-manager
-      namespace: scality
+      namespace: ${{ github.repository_owner }}
+
   test-deployment:
     needs: docker-build
     runs-on: ubuntu-latest
@@ -23,7 +25,7 @@ jobs:
       - uses: helm/kind-action@v1
       - name: Edit the runner-manager image
         working-directory: manifests
-        run: kustomize edit set image ghcr.io/scality/runner-manager:${{ github.sha }}
+        run: kustomize edit set image ghcr.io/${{ github.repository_owner }}/runner-manager:${{ github.sha }}
       - name: Deploy runner-manager
         run: kubectl apply -k manifests
       - name: Check if deployment is ready

codecov.yml

@@ -11,3 +11,13 @@ coverage:
 codecov:
   notify:
     wait_for_ci: true
+
+flag_management:
+  default_rules:
+    carryforward: true
+    statuses:
+      - type: project
+        target: auto
+        threshold: 1%
+      - type: patch
+        target: 80%

mkdocs.yml

@@ -52,11 +52,15 @@ markdown_extensions:
   - pymdownx.highlight:
       anchor_linenums: true
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
   - pymdownx.inlinehilite
   - pymdownx.snippets
-  - pymdownx.superfences
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
   - tables
   - attr_list
   - md_in_html

poetry.lock

@@ -13,15 +13,15 @@ python = "^3.11"
 fastapi = "^0.110.0"
 uvicorn = { extras = ["standard"], version = "^0.29.0" }
 pydantic = ">=1.10.2,<2.0"
-redis-om = "^0.2.1"
+redis-om = "^0.2.2"
 rq = "^1.16.1"
 pyyaml = "^6.0.1"
 redis = "^4.6.0"
 docker = "^7.0.0"
 google-cloud-compute = "^1.18.0"
 boto3 = "^1.28.85"
 botocore = "^1.31.85"
-boto3-stubs = { extras = ["ec2"], version = "^1.34.70" }
+boto3-stubs = { extras = ["ec2"], version = "^1.34.75" }
 githubkit = { extras = ["auth-app"], version = "^0.11.3" }
 rq-scheduler = "^0.13.1"
 
@@ -30,20 +30,20 @@ rq-scheduler = "^0.13.1"
 optional = true
 
 [tool.poetry.group.docs.dependencies]
-mkdocs-material = "^9.5.15"
+mkdocs-material = "^9.5.16"
 mkdocs = "^1.5.3"
 
 [tool.poetry.group.dev.dependencies]
 connexion = "^3.0.5"
 pytest = "^8.1.1"
 pytest-cov = "^5.0.0"
-hypothesis = "^6.99.13"
-pyright = "^1.1.355"
+hypothesis = "^6.100.0"
+pyright = "^1.1.356"
 black = "^24.3.0"
 isort = "^5.13.2"
-ruff = "^0.3.4"
+ruff = "^0.3.5"
 yamllint = "^1.35.1"
-types-redis = "^4.6.0.20240218"
+types-redis = "^4.6.0.20240311"
 
 
 [build-system]

pyproject.toml

@@ -7,6 +7,7 @@
 from mypy_boto3_ec2.type_defs import (
     BlockDeviceMappingTypeDef,
     EbsBlockDeviceTypeDef,
+    IamInstanceProfileTypeDef,
     TagSpecificationTypeDef,
     TagTypeDef,
 )
@@ -133,6 +134,7 @@ class AWSConfig(BackendConfig):
         "BlockDeviceMappings": Sequence[BlockDeviceMappingTypeDef],
         "MaxCount": int,
         "MinCount": int,
+        "IamInstanceProfile": IamInstanceProfileTypeDef,
     },
 )
 
@@ -150,6 +152,7 @@ class AWSInstanceConfig(InstanceConfig):
     tags: Dict[str, str] = {}
     volume_type: VolumeTypeType = "gp3"
     disk_size_gb: int = 20
+    iam_instance_profile_arn: str = ""
 
     def configure_instance(self, runner: Runner) -> AwsInstance:
         """Configure instance."""
@@ -178,8 +181,15 @@ def configure_instance(self, runner: Runner) -> AwsInstance:
             TagSpecificationTypeDef(
                 ResourceType="instance",
                 Tags=tags,
-            )
+            ),
+            TagSpecificationTypeDef(
+                ResourceType="volume",
+                Tags=tags,
+            ),
         ]
+        iam_instance_profile: IamInstanceProfileTypeDef = IamInstanceProfileTypeDef(
+            Arn=self.iam_instance_profile_arn
+        )
         return AwsInstance(
             ImageId=self.image,
             InstanceType=self.instance_type,
@@ -190,4 +200,5 @@ def configure_instance(self, runner: Runner) -> AwsInstance:
             MaxCount=self.max_count,
             MinCount=self.min_count,
             BlockDeviceMappings=block_device_mappings,
+            IamInstanceProfile=iam_instance_profile,
         )

runner_manager/models/backend.py

@@ -83,7 +83,8 @@ def find_from_webhook(cls, webhook: WorkflowJobEvent) -> "Runner | None":
         Returns:
             Runner: A runner object
         """
-
+        if webhook.workflow_job.runner_id is None:
+            return None
         try:
             runner: Runner | None = cls.find(
                 cls.id == webhook.workflow_job.runner_id

runner_manager/models/runner.py

@@ -271,7 +271,7 @@ def save(
         return super().save(pipeline=pipeline)
 
     @classmethod
-    def find_from_webhook(cls, webhook: WorkflowJobEvent) -> "RunnerGroup":
+    def find_from_webhook(cls, webhook: WorkflowJobEvent) -> "RunnerGroup | None":
         """Find the runner group from a webhook instance.
 
         Args:
@@ -280,6 +280,8 @@ def find_from_webhook(cls, webhook: WorkflowJobEvent) -> "RunnerGroup":
         Returns:
             RunnerGroup: Runner group instance.
         """
+        if webhook.workflow_job.runner_group_id is None:
+            return None
         try:
             group: RunnerGroup | None = cls.find(
                 (cls.id == webhook.workflow_job.runner_group_id)

runner_manager/models/runner_group.py

@@ -46,16 +46,23 @@ def aws_runner(runner: Runner, aws_group: RunnerGroup) -> Runner:
 def test_aws_instance_config(runner: Runner):
     AWSConfig()
     instance_config = AWSInstanceConfig(
-        tags={"test": "test"}, subnet_id="i-0f9b0a3b7b3b3b3b3"
+        tags={"test": "test"},
+        subnet_id="i-0f9b0a3b7b3b3b3b3",
+        iam_instance_profile_arn="test",
     )
     instance: AwsInstance = instance_config.configure_instance(runner)
     assert instance["ImageId"] == instance_config.image
     assert instance["SubnetId"] == instance_config.subnet_id
+    assert (
+        instance["IamInstanceProfile"]["Arn"]
+        == instance_config.iam_instance_profile_arn
+    )
     assert runner.name in instance["UserData"]
     tags = instance["TagSpecifications"][0]["Tags"]
     assert TagTypeDef(Key="test", Value="test") in tags
     assert TagTypeDef(Key="Name", Value=runner.name) in tags
     assert runner.encoded_jit_config in instance["UserData"]
+    assert instance["TagSpecifications"][1]["ResourceType"] == "volume"
 
 
 @mark.skipif(not os.getenv("AWS_ACCESS_KEY_ID"), reason="AWS credentials not found")

tests/unit/backend/test_aws.py

@@ -52,6 +52,8 @@ def test_find_from_webhook(runner: Runner, webhook: WorkflowJobCompleted):
     assert Runner.find_from_webhook(webhook) == runner
     runner.delete(runner.pk)
     assert Runner.find_from_webhook(webhook) is None
+    webhook.workflow_job.runner_id = None
+    assert Runner.find_from_webhook(webhook) is None
 
 
 def test_update_from_github(runner: Runner, github: GitHub):

tests/unit/models/test_runner.py

@@ -94,6 +94,9 @@ def test_find_from_webhook(runner_group: RunnerGroup, webhook: WorkflowJobComple
     assert RunnerGroup.find_from_webhook(webhook) == runner_group
     runner_group.delete(runner_group.pk)
     assert RunnerGroup.find_from_webhook(webhook) is None
+    webhook.workflow_job.runner_group_name = None
+    webhook.workflow_job.runner_group_id = None
+    assert RunnerGroup.find_from_webhook(webhook) is None
 
 
 def test_runner_group_delete_method(runner_group: RunnerGroup, github: GitHub):