Release 0.4.8 (#1465)

# Description Please describe the change you have made. ## Checklist - [ ] Prefixed the PR title with the Jira issue number on the form `[CDF-12345]`. - [ ] Tests added/updated. - [ ] Run Demo Job Locally. - [ ] Documentation updated. - [ ] Changelogs updated in [CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md). - [ ] Template changelogs updated in [CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md). [CDF-12345]: https://cognitedata.atlassian.net/browse/CDF-12345?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ
cognitedata · Feb 14, 2025 · e5f1a15 · e5f1a15
2 parents 6832762 + f5f4b6e
commit e5f1a15
Show file tree

Hide file tree

Showing 29 changed files with 142 additions and 31 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -12,7 +12,7 @@ repos:
           - --fixable=E,W,F,I,T,RUF,TID,UP
           - --target-version=py39
       - id: ruff-format
-    rev: v0.9.5
+    rev: v0.9.6
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.44.0

diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md
@@ -15,6 +15,18 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.4.8] - 2025-02-14
+
+### Fixed
+
+- The Toolkit no longer tries to update a data modeling container that does not have the `usedFor` field set.
+- The Toolkit no longer hides `409` errors when failing to create a resource behind a low severity warning.
+
+## Improved
+
+- If the `cdf deploy` command fails to deploy multiple `Group`s due to `Failed to buffer the request body` error, the Toolkit
+  now retries to create the `Group`s one by one.
+
 ## [0.4.7] - 2025-02-13
 
 ### Added

diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md
@@ -15,6 +15,13 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.4.8] - 2025-02-14
+
+### Fixed
+
+- The annotation containers in the `models/cdf_cdm_extension_full` and `models/cdf_process_industry_extension_full`
+  modules have now correctly set the `usedFor` to `edge`. This caused a 409 error when deploying these modules.
+
 ## [0.4.7] - 2025-02-13
 
 No changes to templates.

diff --git a/cdf.toml b/cdf.toml
@@ -17,4 +17,4 @@ dump = true
 [modules]
 # This is the version of the modules. It should'n be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.7"
+version = "0.4.8"
diff --git a/cognite_toolkit/_builtin_modules/cdf.toml b/cognite_toolkit/_builtin_modules/cdf.toml
@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.7"
+version = "0.4.8"
 
 
 [plugins]

diff --git a/...in_modules/models/cdf_cdm_extension_full/data_models/containers/Annotation.Container.yaml b/...in_modules/models/cdf_cdm_extension_full/data_models/containers/Annotation.Container.yaml
@@ -11,3 +11,4 @@ properties:
     autoIncrement: false
 constraints: {}
 indexes: {}
+usedFor: edge
diff --git a/..._process_industry_extension_full/data_models/containers/360ImageAnnotation.Container.yaml b/..._process_industry_extension_full/data_models/containers/360ImageAnnotation.Container.yaml
@@ -11,3 +11,4 @@ properties:
     autoIncrement: false
 constraints: {}
 indexes: {}
+usedFor: edge
diff --git a/...dels/cdf_process_industry_extension_full/data_models/containers/Annotation.Container.yaml b/...dels/cdf_process_industry_extension_full/data_models/containers/Annotation.Container.yaml
@@ -11,3 +11,4 @@ properties:
     autoIncrement: false
 constraints: {}
 indexes: {}
+usedFor: edge
diff --git a/...f_process_industry_extension_full/data_models/containers/DiagramAnnotation.Container.yaml b/...f_process_industry_extension_full/data_models/containers/DiagramAnnotation.Container.yaml
@@ -11,3 +11,4 @@ properties:
     autoIncrement: false
 constraints: {}
 indexes: {}
+usedFor: edge
diff --git a/cognite_toolkit/_cdf_tk/commands/deploy.py b/cognite_toolkit/_cdf_tk/commands/deploy.py
@@ -353,15 +353,10 @@ def _create_resources(
         try:
             created = loader.create(resources)
         except CogniteAPIError as e:
-            if e.code == 409:
-                self.warn(LowSeverityWarning("Resource(s) already exist(s), skipping creation."))
-            else:
-                message = f"Failed to create resource(s). Error: {escape(str(e))!s}."
-                if hint := self._environment_variable_hint(
-                    loader.get_ids(resources), environment_variable_warning_by_id
-                ):
-                    message += hint
-                raise ResourceCreationError(message) from e
+            message = f"Failed to create resource(s). Error: {escape(str(e))!s}."
+            if hint := self._environment_variable_hint(loader.get_ids(resources), environment_variable_warning_by_id):
+                message += hint
+            raise ResourceCreationError(message) from e
         except CogniteDuplicatedError as e:
             self.warn(
                 LowSeverityWarning(

diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/auth_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/auth_loaders.py
@@ -347,18 +347,37 @@ def dump_resource(self, resource: Group, local: dict[str, Any] | None = None) ->
     def create(self, items: Sequence[GroupWrite]) -> GroupList:
         if len(items) == 0:
             return GroupList([])
-        return self.client.iam.groups.create(items)
+        return self._create_with_fallback(items, action="create")
 
     def update(self, items: Sequence[GroupWrite]) -> GroupList:
         # We MUST retrieve all the old groups BEFORE we add the new, if not the new will be deleted
         old_groups = self.client.iam.groups.list(all=True)
-        created = self.client.iam.groups.create(items)
+        created = self._create_with_fallback(items, action="update")
         created_names = {g.name for g in created}
         to_delete = GroupList([group for group in old_groups if group.name in created_names])
         if to_delete:
             self._delete(to_delete, check_own_principal=False)
         return created
 
+    def _create_with_fallback(self, items: Sequence[GroupWrite], action: Literal["create", "update"]) -> GroupList:
+        try:
+            return self.client.iam.groups.create(items)
+        except CogniteAPIError as e:
+            if not (e.code == 400 and "buffer" in e.message.lower() and len(items) > 1):
+                raise e
+            # Fallback to create one by one
+            created_list = GroupList([])
+            for item in items:
+                try:
+                    created = self.client.iam.groups.create(item)
+                except CogniteAPIError as e:
+                    HighSeverityWarning(f"Failed to {action} group {item.name}. Error: {escape(str(e))}").print_warning(
+                        include_timestamp=True, console=self.console
+                    )
+                else:
+                    created_list.append(created)
+            return created_list
+
     def retrieve(self, ids: SequenceNotStr[str]) -> GroupList:
         id_set = set(ids)
         remote = self.client.iam.groups.list(all=True)

diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/datamodel_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/datamodel_loaders.py
@@ -72,6 +72,7 @@
 from cognite.client.utils.useful_types import SequenceNotStr
 from rich import print
 from rich.console import Console
+from rich.markup import escape
 from rich.panel import Panel
 
 from cognite_toolkit._cdf_tk._parameters import ANY_INT, ANY_STR, ANYTHING, ParameterSpec, ParameterSpecSet
@@ -88,7 +89,7 @@
     ResourceContainerLoader,
     ResourceLoader,
 )
-from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, LowSeverityWarning
+from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, LowSeverityWarning, MediumSeverityWarning
 from cognite_toolkit._cdf_tk.utils import (
     GraphQLParser,
     calculate_str_or_file_hash,
@@ -324,6 +325,8 @@ def dump_resource(self, resource: Container, local: dict[str, Any] | None = None
             for key, type_default in [("list", False), ("collation", "ucs_basic")]:
                 if cdf_type.get(key) == type_default and key not in local_type:
                     cdf_type.pop(key, None)
+        if "usedFor" not in local:
+            dumped.pop("usedFor", None)
         return dumped
 
     def create(self, items: Sequence[ContainerApply]) -> ContainerList:
@@ -591,7 +594,26 @@ def diff_list(
         return super().diff_list(local, cdf, json_path)
 
     def create(self, items: Sequence[ViewApply]) -> ViewList:
-        return self.client.data_modeling.views.apply(items)
+        try:
+            return self.client.data_modeling.views.apply(items)
+        except CogniteAPIError as e1:
+            if not (isinstance(e1.extra, dict) and "isAutoRetryable" in e1.extra and e1.extra["isAutoRetryable"]):
+                raise
+            # Fallback to creating one by one if the error is auto-retryable.
+            MediumSeverityWarning(
+                f"Failed to create {len(items)} views error:\n{escape(str(e1))}\n\n----------------------------\nTrying to create one by one..."
+            ).print_warning(include_timestamp=True, console=self.console)
+            created_list = ViewList([])
+            for no, item in enumerate(items):
+                try:
+                    created = self.client.data_modeling.views.apply(item)
+                except CogniteAPIError as e2:
+                    e2.failed.extend(items[no + 1 :])
+                    e2.successful.extend(created_list)
+                    raise e2 from e1
+                else:
+                    created_list.append(created)
+            return created_list
 
     def retrieve(self, ids: SequenceNotStr[ViewId]) -> ViewList:
         return self.client.data_modeling.views.retrieve(

diff --git a/cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml b/cognite_toolkit/_repo_files/AzureDevOps/.devops/deploy-pipeline.yml
@@ -14,7 +14,7 @@ jobs:
     pool:
       vmImage: 'ubuntu-latest'
     container: 
-      image: 'cognite/toolkit:0.4.7'
+      image: 'cognite/toolkit:0.4.8'
       env:
         CDF_CLUSTER: $(CDF_CLUSTER)
         CDF_PROJECT: $(CDF_PROJECT)

diff --git a/cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml b/cognite_toolkit/_repo_files/AzureDevOps/.devops/dry-run-pipeline.yml
@@ -16,7 +16,7 @@ jobs:
     pool:
       vmImage: 'ubuntu-latest'
     container: 
-      image: 'cognite/toolkit:0.4.7'
+      image: 'cognite/toolkit:0.4.8'
       env:
         CDF_CLUSTER: $(CDF_CLUSTER)
         CDF_PROJECT: $(CDF_PROJECT)

diff --git a/cognite_toolkit/_repo_files/GitHub/.github/workflows/build.yaml b/cognite_toolkit/_repo_files/GitHub/.github/workflows/build.yaml
@@ -10,7 +10,7 @@ jobs:
     environment: dev
     name: Deploy Dry Run
     container:
-      image: cognite/toolkit:0.4.7
+      image: cognite/toolkit:0.4.8
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

diff --git a/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml b/cognite_toolkit/_repo_files/GitHub/.github/workflows/deploy.yaml
@@ -12,7 +12,7 @@ jobs:
     environment: dev
     name: Deploy
     container:
-      image: cognite/toolkit:0.4.7
+      image: cognite/toolkit:0.4.8
       env:
         CDF_CLUSTER: ${{ vars.CDF_CLUSTER }}
         CDF_PROJECT: ${{ vars.CDF_PROJECT }}

diff --git a/cognite_toolkit/_version.py b/cognite_toolkit/_version.py
@@ -1 +1 @@
-__version__ = "0.4.7"
+__version__ = "0.4.8"
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "cognite_toolkit"
-version = "0.4.7"
+version = "0.4.8"
 description = "Official Cognite Data Fusion tool for project templates and configuration deployment"
 authors = [
     {name ="Cognite AS", email="support@cognite.com"}

diff --git a/tests/data/build_group_with_unknown_acl/_build_environment.yaml b/tests/data/build_group_with_unknown_acl/_build_environment.yaml
@@ -4,4 +4,4 @@ project: pytest-project
 type: dev
 selected:
 - modules
-cdf_toolkit_version: 0.4.7
+cdf_toolkit_version: 0.4.8
diff --git a/tests/data/cdf_toml_data/cdf.toml b/tests/data/cdf_toml_data/cdf.toml
@@ -1,7 +1,7 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.7"
+version = "0.4.8"
 
 [plugins]
 graphql = true

diff --git a/tests/data/complete_org/cdf.toml b/tests/data/complete_org/cdf.toml
@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.7"
+version = "0.4.8"
diff --git a/tests/data/complete_org_alpha_flags/cdf.toml b/tests/data/complete_org_alpha_flags/cdf.toml
@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.7"
+version = "0.4.8"
diff --git a/tests/data/project_no_cognite_modules/cdf.toml b/tests/data/project_no_cognite_modules/cdf.toml
@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.7"
+version = "0.4.8"
diff --git a/tests/data/project_with_bad_modules/cdf.toml b/tests/data/project_with_bad_modules/cdf.toml
@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.7"
+version = "0.4.8"
diff --git a/tests/data/project_with_duplicates/cdf.toml b/tests/data/project_with_duplicates/cdf.toml
@@ -1,4 +1,4 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.7"
+version = "0.4.8"
diff --git a/tests/data/run_data/cdf.toml b/tests/data/run_data/cdf.toml
@@ -1,7 +1,7 @@
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.4.7"
+version = "0.4.8"
 
 [modules.packages]
 cdf_infield = [

diff --git a/tests/test_unit/test_cdf_tk/test_loaders/test_container_loader.py b/tests/test_unit/test_cdf_tk/test_loaders/test_container_loader.py
@@ -1,7 +1,12 @@
+from pathlib import Path
+from unittest.mock import MagicMock
+
 import pytest
+from cognite.client.data_classes.data_modeling import Container, ContainerProperty, Text
 
 from cognite_toolkit._cdf_tk._parameters import read_parameters_from_dict
-from cognite_toolkit._cdf_tk.loaders import ContainerLoader
+from cognite_toolkit._cdf_tk.loaders import ContainerLoader, ResourceWorker
+from tests.test_unit.approval_client import ApprovalToolkitClient
 
 
 class TestContainerLoader:
@@ -35,3 +40,46 @@ def test_valid_spec(self, item: dict):
         extra = dumped - spec
 
         assert not extra, f"Extra keys: {extra}"
+
+    def test_unchanged_used_for_not_set(self, toolkit_client_approval: ApprovalToolkitClient) -> None:
+        loader = ContainerLoader.create_loader(toolkit_client_approval.mock_client)
+        raw_file = """space: sp_enterprise_process_industry_full
+externalId: Toolkit360Image
+properties:
+  UUID:
+    type:
+      list: false
+      collation: ucs_basic
+      type: text
+    immutable: false
+    nullable: true
+    autoIncrement: false
+constraints: {}
+indexes: {}
+"""
+        file = MagicMock(spec=Path)
+        file.read_text.return_value = raw_file
+        cdf_container = Container(
+            space="sp_enterprise_process_industry_full",
+            external_id="Toolkit360Image",
+            last_updated_time=1739469813633,
+            created_time=1739469813633,
+            description=None,
+            name=None,
+            used_for="node",
+            is_global=False,
+            properties={"UUID": ContainerProperty(type=Text())},
+            indexes={},
+            constraints={},
+        )
+
+        toolkit_client_approval.append(Container, [cdf_container])
+
+        worker = ResourceWorker(loader)
+        to_create, to_change, to_delete, unchanged, _ = worker.load_resources([file])
+        assert {
+            "create": len(to_create),
+            "change": len(to_change),
+            "delete": len(to_delete),
+            "unchanged": len(unchanged),
+        } == {"create": 0, "change": 0, "delete": 0, "unchanged": 1}
diff --git a/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_cdm_extension_full.yaml b/tests/test_unit/test_cli/test_build_deploy_snapshots/cdf_cdm_extension_full.yaml
@@ -142,6 +142,7 @@ Container:
         list: false
         type: text
   space: sp_enterprise_schema
+  usedFor: edge
 - constraints: {}
   externalId: ToolkitAsset
   indexes: {}

diff --git a/...s/test_unit/test_cli/test_build_deploy_snapshots/cdf_process_industry_extension_full.yaml b/...s/test_unit/test_cli/test_build_deploy_snapshots/cdf_process_industry_extension_full.yaml
@@ -25,6 +25,7 @@ Container:
         list: false
         type: text
   space: sp_enterprise_process_industry_full
+  usedFor: edge
 - constraints: {}
   externalId: ToolkitIDM360ImageCollection
   indexes: {}
@@ -142,6 +143,7 @@ Container:
         list: false
         type: text
   space: sp_enterprise_process_industry_full
+  usedFor: edge
 - constraints: {}
   externalId: ToolkitIDMAsset
   indexes: {}
@@ -259,6 +261,7 @@ Container:
         list: false
         type: text
   space: sp_enterprise_process_industry_full
+  usedFor: edge
 - constraints: {}
   externalId: ToolkitIDMEquipment
   indexes: {}