feat: add support for repo-filter parameter for planning operations (#…

…276)

* feat: add support for --repo-filter parameter for plan and apply operations

* fix: add repo-filter parameter where needed

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Added
 
+- Added support for parameter `--repo-filter` for `plan` and `apply` operations. ([#275](https://github.com/eclipse-csi/otterdog/issues/275))
 - Added support for tags for deployment policies in `environments`. ([#268](https://github.com/eclipse-csi/otterdog/issues/268))
 - Added support for `custom properties`. ([#256](https://github.com/eclipse-csi/otterdog/issues/256))
 - Added validation for setting `forked_repository` of a repository to match the expected format `<owner>/<repo>`.

otterdog/cli.py

@@ -332,6 +332,12 @@ def import_command(organizations: list[str], force, no_web_ui):
     default=False,
     help="skip settings retrieved via web ui",
 )
+@click.option(
+    "--repo-filter",
+    show_default=True,
+    default="*",
+    help="a valid shell pattern to match repository names to be included",
+)
 @click.option(
     "--update-webhooks",
     is_flag=True,
@@ -349,10 +355,10 @@ def import_command(organizations: list[str], force, no_web_ui):
 @click.option(
     "--update-filter",
     show_default=True,
-    default=".*",
+    default="*",
     help="a valid shell pattern to match webhook urls / secret names to be included for update",
 )
-def plan(organizations: list[str], no_web_ui, update_webhooks, update_secrets, update_filter):
+def plan(organizations: list[str], no_web_ui, repo_filter, update_webhooks, update_secrets, update_filter):
     """
     Show changes that would be applied by otterdog based on the current configuration
     compared to the current live configuration at GitHub.
@@ -361,6 +367,7 @@ def plan(organizations: list[str], no_web_ui, update_webhooks, update_secrets, u
         organizations,
         PlanOperation(
             no_web_ui=no_web_ui,
+            repo_filter=repo_filter,
             update_webhooks=update_webhooks,
             update_secrets=update_secrets,
             update_filter=update_filter,
@@ -376,6 +383,12 @@ def plan(organizations: list[str], no_web_ui, update_webhooks, update_secrets, u
     default="-BASE",
     help="suffix to append to the configuration for comparison",
 )
+@click.option(
+    "--repo-filter",
+    show_default=True,
+    default="*",
+    help="a valid shell pattern to match repository names to be included",
+)
 @click.option(
     "--update-webhooks",
     is_flag=True,
@@ -393,18 +406,22 @@ def plan(organizations: list[str], no_web_ui, update_webhooks, update_secrets, u
 @click.option(
     "--update-filter",
     show_default=True,
-    default=".*",
+    default="*",
     help="a valid shell pattern to match webhook urls / secret names to be included for update",
 )
-def local_plan(organizations: list[str], suffix, update_webhooks, update_secrets, update_filter):
+def local_plan(organizations: list[str], suffix, repo_filter, update_webhooks, update_secrets, update_filter):
     """
     Show changes that would be applied by otterdog based on the current configuration
     compared to another local configuration.
     """
     _execute_operation(
         organizations,
         LocalPlanOperation(
-            suffix=suffix, update_webhooks=update_webhooks, update_secrets=update_secrets, update_filter=update_filter
+            suffix=suffix,
+            repo_filter=repo_filter,
+            update_webhooks=update_webhooks,
+            update_secrets=update_secrets,
+            update_filter=update_filter,
         ),
     )
 
@@ -426,6 +443,12 @@ def local_plan(organizations: list[str], suffix, update_webhooks, update_secrets
     default=False,
     help="skip settings retrieved via web ui",
 )
+@click.option(
+    "--repo-filter",
+    show_default=True,
+    default="*",
+    help="a valid shell pattern to match repository names to be included",
+)
 @click.option(
     "--update-webhooks",
     is_flag=True,
@@ -443,7 +466,7 @@ def local_plan(organizations: list[str], suffix, update_webhooks, update_secrets
 @click.option(
     "--update-filter",
     show_default=True,
-    default=".*",
+    default="*",
     help="a valid shell pattern to match webhook urls / secret names to be included for update",
 )
 @click.option(
@@ -458,6 +481,7 @@ def apply(
     organizations: list[str],
     force,
     no_web_ui,
+    repo_filter,
     update_webhooks,
     update_secrets,
     update_filter,
@@ -471,6 +495,7 @@ def apply(
         ApplyOperation(
             force_processing=force,
             no_web_ui=no_web_ui,
+            repo_filter=repo_filter,
             update_webhooks=update_webhooks,
             update_secrets=update_secrets,
             update_filter=update_filter,
@@ -503,6 +528,12 @@ def apply(
     default=False,
     help="skip settings retrieved via web ui",
 )
+@click.option(
+    "--repo-filter",
+    show_default=True,
+    default="*",
+    help="a valid shell pattern to match repository names to be included",
+)
 @click.option(
     "--update-webhooks",
     is_flag=True,
@@ -520,7 +551,7 @@ def apply(
 @click.option(
     "--update-filter",
     show_default=True,
-    default=".*",
+    default="*",
     help="a valid shell pattern to match webhook urls / secret names to be included for update",
 )
 @click.option(
@@ -535,6 +566,7 @@ def local_apply(
     organizations: list[str],
     force,
     no_web_ui,
+    repo_filter,
     update_webhooks,
     update_secrets,
     update_filter,
@@ -550,6 +582,7 @@ def local_apply(
             suffix=suffix,
             force_processing=force,
             no_web_ui=no_web_ui,
+            repo_filter=repo_filter,
             update_webhooks=update_webhooks,
             update_secrets=update_secrets,
             update_filter=update_filter,

otterdog/models/__init__.py

@@ -152,6 +152,7 @@ class PatchContext:
 @dataclasses.dataclass
 class LivePatchContext:
     org_id: str
+    repo_filter: str
     update_webhooks: bool
     update_secrets: bool
     update_filter: str
@@ -389,7 +390,7 @@ def is_key_valid_for_diff_computation(self, key: str) -> bool:
     def include_field_for_patch_computation(self, field: dataclasses.Field) -> bool:
         return self.include_field_for_diff_computation(field)
 
-    def include_for_live_patch(self) -> bool:
+    def include_for_live_patch(self, context: LivePatchContext) -> bool:
         """
         Indicates if this ModelObject should be considered when generating a live patch.
 
@@ -561,15 +562,15 @@ def generate_live_patch_of_list(
                     cls.generate_live_patch(None, current_object, parent_object, context, handler)
                 continue
 
-            if expected_object.include_for_live_patch():
+            if expected_object.include_for_live_patch(context):
                 cls.generate_live_patch(expected_object, current_object, parent_object, context, handler)
 
             for k in expected_object.get_all_key_values():
                 expected_objects_by_all_keys.pop(k)
             expected_objects_by_key.pop(expected_object.get_key_value())
 
         for _, expected_object in expected_objects_by_key.items():
-            if expected_object.include_for_live_patch():
+            if expected_object.include_for_live_patch(context):
                 cls.generate_live_patch(expected_object, None, parent_object, context, handler)
 
     @classmethod

otterdog/models/repository.py

@@ -525,6 +525,11 @@ def include_field_for_patch_computation(self, field: dataclasses.Field) -> bool:
         # archived repos, the properties might be used when the repo gets unarchived.
         return True
 
+    def include_for_live_patch(self, context: LivePatchContext) -> bool:
+        import fnmatch
+
+        return fnmatch.fnmatch(self.name, context.repo_filter)
+
     def get_model_objects(self) -> Iterator[tuple[ModelObject, ModelObject]]:
         for webhook in self.webhooks:
             yield webhook, self

otterdog/models/secret.py

@@ -84,7 +84,7 @@ def is_key_valid_for_diff_computation(self, key: str) -> bool:
     def include_field_for_patch_computation(self, field: dataclasses.Field) -> bool:
         return True
 
-    def include_for_live_patch(self) -> bool:
+    def include_for_live_patch(self, context: LivePatchContext) -> bool:
         return not self.has_dummy_secret()
 
     @classmethod

otterdog/models/webhook.py

@@ -73,7 +73,7 @@ def is_key_valid_for_diff_computation(self, key: str) -> bool:
     def include_field_for_patch_computation(self, field: dataclasses.Field) -> bool:
         return True
 
-    def include_for_live_patch(self) -> bool:
+    def include_for_live_patch(self, context: LivePatchContext) -> bool:
         return not self.has_dummy_secret()
 
     def validate(self, context: ValidationContext, parent_object: Any) -> None:

otterdog/operations/apply.py

@@ -22,14 +22,15 @@ def __init__(
         self,
         force_processing: bool,
         no_web_ui: bool,
+        repo_filter: str,
         update_webhooks: bool,
         update_secrets: bool,
         update_filter: str,
         delete_resources: bool,
         resolve_secrets: bool = True,
         include_resources_with_secrets: bool = True,
     ):
-        super().__init__(no_web_ui, update_webhooks, update_secrets, update_filter)
+        super().__init__(no_web_ui, repo_filter, update_webhooks, update_secrets, update_filter)
         self._force_processing = force_processing
         self._delete_resources = delete_resources
         self._resolve_secrets = resolve_secrets

otterdog/operations/diff_operation.py

@@ -42,10 +42,18 @@ def __call__(self, org_id: str, diff_status: DiffStatus, patches: list[LivePatch
 
 
 class DiffOperation(Operation):
-    def __init__(self, no_web_ui: bool, update_webhooks: bool, update_secrets: bool, update_filter: str):
+    def __init__(
+        self,
+        no_web_ui: bool,
+        repo_filter: str,
+        update_webhooks: bool,
+        update_secrets: bool,
+        update_filter: str,
+    ):
         super().__init__()
 
         self.no_web_ui = no_web_ui
+        self.repo_filter = repo_filter
         self.update_webhooks = update_webhooks
         self.update_secrets = update_secrets
         self.update_filter = update_filter
@@ -199,7 +207,12 @@ def handle(patch: LivePatch) -> None:
                     )
 
         context = LivePatchContext(
-            github_id, self.update_webhooks, self.update_secrets, self.update_filter, expected_org.settings
+            github_id,
+            self.repo_filter,
+            self.update_webhooks,
+            self.update_secrets,
+            self.update_filter,
+            expected_org.settings,
         )
         expected_org.generate_live_patch(current_org, context, handle)
 

otterdog/operations/local_apply.py

@@ -20,6 +20,7 @@ def __init__(
         suffix: str,
         force_processing: bool,
         no_web_ui: bool,
+        repo_filter: str,
         update_webhooks: bool,
         update_secrets: bool,
         update_filter: str,
@@ -30,6 +31,7 @@ def __init__(
         super().__init__(
             force_processing=force_processing,
             no_web_ui=no_web_ui,
+            repo_filter=repo_filter,
             update_webhooks=update_webhooks,
             update_secrets=update_secrets,
             update_filter=update_filter,

otterdog/operations/local_plan.py

@@ -17,9 +17,20 @@
 
 
 class LocalPlanOperation(PlanOperation):
-    def __init__(self, suffix: str, update_webhooks: bool, update_secrets: bool, update_filter: str) -> None:
+    def __init__(
+        self,
+        suffix: str,
+        repo_filter: str,
+        update_webhooks: bool,
+        update_secrets: bool,
+        update_filter: str,
+    ) -> None:
         super().__init__(
-            no_web_ui=False, update_webhooks=update_webhooks, update_secrets=update_secrets, update_filter=update_filter
+            no_web_ui=False,
+            repo_filter=repo_filter,
+            update_webhooks=update_webhooks,
+            update_secrets=update_secrets,
+            update_filter=update_filter,
         )
 
         self.suffix = suffix

otterdog/operations/open_pull_request.py

@@ -99,7 +99,7 @@ async def execute(self, org_config: OrganizationConfig) -> int:
                     self.printer.level_up()
 
                     try:
-                        operation = LocalPlanOperation("-BASE", False, False, "")
+                        operation = LocalPlanOperation("-BASE", "*", False, False, "")
                         operation.init(self.config, self.printer)
                         valid_config = await operation.generate_diff(org_config)
                     finally:

otterdog/operations/plan.py

@@ -21,8 +21,15 @@ class PlanOperation(DiffOperation):
     Calculates and displays changes that would be applied to an organization based on the current configuration.
     """
 
-    def __init__(self, no_web_ui: bool, update_webhooks: bool, update_secrets: bool, update_filter: str):
-        super().__init__(no_web_ui, update_webhooks, update_secrets, update_filter)
+    def __init__(
+        self,
+        no_web_ui: bool,
+        repo_filter: str,
+        update_webhooks: bool,
+        update_secrets: bool,
+        update_filter: str,
+    ):
+        super().__init__(no_web_ui, repo_filter, update_webhooks, update_secrets, update_filter)
 
     def init(self, config: OtterdogConfig, printer: IndentingPrinter) -> None:
         super().init(config, printer)

otterdog/operations/push_config.py

@@ -178,7 +178,7 @@ async def _display_diff(self, org_config: OrganizationConfig, provider: GitHubPr
         self.printer.level_up()
 
         try:
-            operation = LocalPlanOperation("-BASE", False, False, "")
+            operation = LocalPlanOperation("-BASE", "*", False, False, "")
             operation.init(self.config, self.printer)
             valid_config = await operation.generate_diff(org_config)
         finally:

otterdog/webapp/tasks/apply_changes.py

@@ -189,6 +189,7 @@ async def _execute(self) -> ApplyResult:
             operation = LocalApplyOperation(
                 force_processing=True,
                 no_web_ui=True,
+                repo_filter="*",
                 update_webhooks=False,
                 update_secrets=False,
                 update_filter="",

otterdog/webapp/tasks/check_sync.py

@@ -153,7 +153,7 @@ async def _execute(self) -> bool:
 
             output = StringIO()
             printer = IndentingPrinter(output, log_level=LogLevel.ERROR)
-            operation = PlanOperation(True, False, False, "")
+            operation = PlanOperation(True, "*", False, False, "")
             # set concurrency to 20 to avoid hitting secondary rate limits with installation tokens
             operation.concurrency = 20
 

otterdog/webapp/tasks/validate_pull_request.py

@@ -145,7 +145,7 @@ async def _execute(self) -> ValidationResult:
             else:
                 output = StringIO()
                 printer = IndentingPrinter(output, log_level=self.log_level)
-                operation = LocalPlanOperation("-BASE", False, False, "")
+                operation = LocalPlanOperation("-BASE", "*", False, False, "")
 
                 def callback(org_id: str, diff_status: DiffStatus, patches: list[LivePatch]):
                     validation_result.requires_secrets = any(list(map(lambda x: x.requires_secrets(), patches)))