Merge branch 'release' into 'master'

Release 2023.09.28

See merge request arenadata/development/adcm!3034

CODEOWNERS

@@ -1,19 +1,20 @@
-/data/ @aas @d.skrynnik @tsd
-/conf/ @aas @d.skrynnik @tsd @a.starovoitov
-/go/ @aas @d.skrynnik @tsd @aer
-/os/ @aas @d.skrynnik @tsd
-/python/ @aas @d.skrynnik @tsd @a.starovoitov
-/spec/ @aas @d.skrynnik @tsd
+tests/ @aer @v.chudasov
+/data/ @aas @d.skrynnik @aer
+/conf/ @aas @d.skrynnik @a.starovoitov @aer
+/go/ @aas @d.skrynnik @aer
+/os/ @aas @d.skrynnik @aer
+/python/ @aas @d.skrynnik @a.starovoitov @aer
+/spec/ @aas @d.skrynnik @aer
 /web/ @v.remizov @d.bardin @k.fedorenko
-.dockerignore @aas @v.remizov @d.skrynnik @tsd
-.gitignore @aas @d.skrynnik @tsd @v.remizov @d.bardin @k.fedorenko
-.gitlab-ci.yaml @aas @v.remizov @d.skrynnik @tsd
-.pre-commit-config.yaml @aas @d.skrynnik @tsd @v.remizov @d.bardin @k.fedorenko
-CODEOWNERS @aas @v.remizov @tsd @d.skrynnik
-COPYRIGHT @aas @v.remizov @d.skrynnik @tsd
-Dockerfile @aas @v.remizov @d.skrynnik @tsd
-LICENSE @aas @v.remizov @d.skrynnik @tsd
-license_checker.py @aas @d.skrynnik @tsd
-Makefile @aas @d.skrynnik @tsd
-pyproject.toml @aas @d.skrynnik @tsd
-README.md @aas @d.skrynnik @tsd @v.remizov @d.bardin @k.fedorenko @a.starovoitov
+.dockerignore @aas @v.remizov @d.skrynnik @aer @v.chudasov
+.gitignore @aas @d.skrynnik @v.remizov @d.bardin @k.fedorenko @aer @v.chudasov
+.gitlab-ci.yaml @aas @v.remizov @d.skrynnik @aer @v.chudasov
+.pre-commit-config.yaml @aas @d.skrynnik @v.remizov @d.bardin @k.fedorenko @aer @v.chudasov
+CODEOWNERS @aas @v.remizov @d.skrynnik @aer @v.chudasov
+COPYRIGHT @aas @v.remizov @d.skrynnik @aer
+Dockerfile @aas @v.remizov @d.skrynnik @aer
+LICENSE @aas @v.remizov @d.skrynnik @aer
+license_checker.py @aas @d.skrynnik @aer
+Makefile @aas @d.skrynnik @aer
+pyproject.toml @aas @d.skrynnik @aer @v.chudasov
+README.md @aas @d.skrynnik @v.remizov @d.bardin @k.fedorenko @a.starovoitov @aer @v.chudasov

Dockerfile

@@ -8,6 +8,7 @@ RUN apk update && \
         bash \
         curl \
         git \
+        gnupg \
         libc6-compat \
         libffi \
         libstdc++ \

Makefile

@@ -25,33 +25,31 @@ build_base:
 build: describe buildss buildjs build_base
 
 unittests_sqlite: describe
-	poetry install --no-root
-	poetry run python/manage.py test python -v 2
+	poetry install --no-root --with unittests
+	poetry run python/manage.py test python -v 2 --parallel
 
 unittests_postgresql: describe
 	docker run -d --rm -e POSTGRES_PASSWORD="postgres" --name postgres -p 5500:5432 postgres:14
 	export DB_HOST="localhost" DB_PORT="5500" DB_NAME="postgres" DB_PASS="postgres" DB_USER="postgres"
-	poetry install --no-root
-	poetry run python/manage.py test python -v 2
+	poetry install --no-root --with unittests
+	poetry run python/manage.py test python -v 2 --parallel
 	docker stop postgres
 
 ng_tests:
 	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster_node16-x64
 	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/web hub.adsw.io/library/functest:3.8.6.slim.buster_node16-x64 ./ng_test.sh
 
 pretty:
+	poetry install --no-root --with lint
 	black license_checker.py python
 	autoflake -r -i --remove-all-unused-imports --exclude apps.py,python/ansible/plugins,python/init_db.py,python/task_runner.py,python/backupdb.py,python/job_runner.py,python/drf_docs.py license_checker.py python
 	isort license_checker.py python
 	python license_checker.py --fix --folders python go
 
 lint:
-	black --check license_checker.py python
-	autoflake --check --quiet -r --remove-all-unused-imports --exclude apps.py,python/ansible/plugins,python/init_db.py,python/task_runner.py,python/backupdb.py,python/job_runner.py,python/drf_docs.py license_checker.py python
-	isort --check license_checker.py python
+	poetry install --no-root --with lint
+	poetry run black --check license_checker.py python
+	poetry run autoflake --check --quiet -r --remove-all-unused-imports --exclude apps.py,python/ansible/plugins,python/init_db.py,python/task_runner.py,python/backupdb.py,python/job_runner.py,python/drf_docs.py license_checker.py python
+	poetry run isort --check license_checker.py python
 	python license_checker.py --folders python go
-	pylint --rcfile pyproject.toml --recursive y python
-
-lint_docker:
-	docker run -i --rm -e DJANGO_SETTINGS_MODULE=adcm.settings $(APP_IMAGE):$(APP_TAG) \
-	sh -c "cd /adcm && poetry install --no-root --with lint && apk add make && make lint"
+	poetry run pylint --rcfile pyproject.toml --recursive y python

conf/adcm/config.yaml

@@ -2,7 +2,7 @@
 
   type: adcm
   name: ADCM
-  version: 2.6
+  version: 3.0
 
   actions:
     run_ldap_sync:
@@ -34,6 +34,26 @@
             We have to know ADCM's Url to send information from host. We try to guess that information from url you enter in browser.
             But if your network has more complicated structure and we guess wrong, please fix that here.
           type: string
+        - name: "verification_public_key"
+          display_name: "Bundle verification public key"
+          description: |
+            Bundle verification GPG public key to verify the signature of the bundle (*.sig file) when uploading the bundle to ADCM.
+          type: file
+          default: "gpg_key.pub"
+    - name: "statistics_collection"
+      display_name: "Statistics Collection"
+      type: "group"
+      activatable: true
+      active: true
+      ui_options:
+        advanced: true
+      subs:
+        - name: "url"
+          description: "URL to send collected statistic"
+          type: string
+          default: adcm-usage-ss.arenadata.io
+          ui_options:
+            invisible: true
     - name: "google_oauth"
       display_name: "Google Auth"
       type: "group"
@@ -46,6 +66,8 @@
           required: false
           ui_options:
             no_confirm: true
+      ui_options:
+        invisible: true
     - name: "yandex_oauth"
       display_name: "Yandex Auth"
       type: "group"
@@ -60,38 +82,10 @@
             no_confirm: true
       ui_options:
         invisible: true
-    - name: "job_log"
-      display_name: "Job Log"
-      type: "group"
-      subs:
-        - name: "log_rotation_on_fs"
-          display_name: "Log rotation from file system"
-          type: integer
-          required: false
-          default: 365
-          min: 0
-          description: |
-            You can set the time (number of days) after which the logs will be deleted from the file system.
-        - name: "log_rotation_in_db"
-          display_name: "Log rotation from database"
-          type: integer
-          required: false
-          default: 365
-          min: 0
-          description: |
-            You can set the time (number of days) after which the logs will be deleted from the database.
     - name: "ansible_settings"
       display_name: "Ansible Settings"
       type: "group"
       subs:
-        - name: "mitogen"
-          display_name: "Use Mitogen"
-          description: |
-            Mitogen for Ansible is a completely redesigned UNIX connection layer and module runtime for Ansible.
-          type: boolean
-          ui_options:
-            invisible: true
-          default: false
         - name: "forks"
           display_name: "Forks"
           description: |
@@ -126,35 +120,44 @@
             Compress the rotated files
           type: boolean
           default: false
-    - name: "config_rotation"
-      display_name: "Configuration rotation"
-      description: |
-        You can enable Clusters/Services/Components configurations deleting mechanism.
+    - name: "audit_data_retention"
+      display_name: "Data retention policy"
       type: "group"
       subs:
+        - name: "log_rotation_on_fs"
+          display_name: "Job log retention period from file system"
+          type: integer
+          required: false
+          default: 365
+          min: 0
+          description: |
+            You can set the time (number of days) after which the logs will be deleted from the file system.
+        - name: "log_rotation_in_db"
+          display_name: "Job log retention period from database"
+          type: integer
+          required: false
+          default: 365
+          min: 0
+          description: |
+            You can set the time (number of days) after which the logs will be deleted from the database.
         - name: "config_rotation_in_db"
-          display_name: "Objects configurations rotation period"
+          display_name: "Objects configurations retention period"
           type: integer
           required: false
           default: 0
           min: 0
           description: |
             You can set the time (number of days) after which the Objects configuration will be deleted from the database. 0 is infinite storing.
-
-    - name: "audit_data_retention"
-      display_name: "Audit data retention"
-      type: "group"
-      subs:
         - name: "retention_period"
-          display_name: "Retention period"
+          display_name: "Audit data retention period"
           description: |
             Data storage period (in days) for operations and authorizations in ADCM.
           type: integer
           default: 1825
         - name: "data_archiving"
-          display_name: "Enable archiving"
+          display_name: "Enable audit data archiving"
           description: |
-            Enable/disable archiving of data on operations and authorizations after the period specified in "retention_period".
+            Enable/disable archiving of data on operations and authorizations after the period specified in "Audit data retention period".
           type: boolean
           default: false
           required: no

conf/adcm/gpg_key.pub

@@ -0,0 +1 @@
+

conf/adcm/python_scripts/run_ldap_sync.py

@@ -69,11 +69,11 @@ def _bind(self) -> ldap.ldapobject.LDAPObject:
     @staticmethod
     def _deactivate_extra_users(ldap_usernames: set):
         django_usernames = set(
-            User.objects.filter(type=OriginType.LDAP, is_active=True).values_list("username", flat=True)
+            User.objects.filter(type=OriginType.LDAP).values_list("username", flat=True)
         )
         for username in django_usernames - ldap_usernames:
             user = User.objects.get(username__iexact=username)
-            sys.stdout.write(f"Deactivate user and his session: {user}\n")
+            sys.stdout.write(f"Delete user: {user}\n")
             user.delete()
 
     def unbind(self) -> None:
@@ -106,6 +106,7 @@ def sync_groups(self) -> list:
             ldap_groups = self.settings["GROUP_SEARCH"].execute(self.conn, {})
             self._sync_ldap_groups(ldap_groups)
             sys.stdout.write("Groups were synchronized\n")
+
         return ldap_groups
 
     def sync_users(self, ldap_groups: list) -> None:
@@ -114,18 +115,21 @@ def sync_users(self, ldap_groups: list) -> None:
             sys.stdout.write("No groups found. Aborting sync users\n")
             self._deactivate_extra_users(set())
             return
+
         group_filter = ""
         for group_dn, _ in ldap_groups:
             group_filter += f"(memberOf={group_dn})"
         if group_filter:
             group_filter = f"(|{group_filter})"
+
         self.settings["USER_SEARCH"].filterstr = (
             f"(&"
             f"(objectClass={self.settings['USER_OBJECT_CLASS']})"
             f"{self.settings['USER_FILTER']}"
             f"{group_filter})"
         )
         ldap_users = self.settings["USER_SEARCH"].execute(self.conn, {"user": "*"}, True)
+
         self._sync_ldap_users(ldap_users, ldap_groups)
         sys.stdout.write("Users were synchronized\n")
 
@@ -164,6 +168,8 @@ def _sync_ldap_users(self, ldap_users: list, ldap_groups: list) -> None:
         ldap_group_names = [group[0].split(",")[0][3:] for group in ldap_groups]
         ldap_usernames = set()
         error_names = []
+        deleted_names: list[str] = []
+
         for cname, ldap_attributes in ldap_users:
             defaults = {}
             for field, ldap_name in self.settings["USER_ATTR_MAP"].items():
@@ -186,12 +192,13 @@ def _sync_ldap_users(self, ldap_users: list, ldap_groups: list) -> None:
                 sys.stdout.write(f"Error creating user {username}: {e}\n")
                 continue
             else:
+                if not self._is_ldap_user_active(ldap_attrs=ldap_attributes):
+                    deleted_names.append(user.username)
+                    user.delete()
+                    continue
+
                 updated = False
-                user.is_active = False
-                if ldap_attributes.get("useraccountcontrol") and not hex(
-                    int(ldap_attributes["useraccountcontrol"][0])
-                ).endswith("2"):
-                    user.is_active = True
+
                 if created:
                     sys.stdout.write(f"Create user: {username}\n")
                     user.set_unusable_password()
@@ -221,8 +228,12 @@ def _sync_ldap_users(self, ldap_users: list, ldap_groups: list) -> None:
                         except (IntegrityError, DataError, Group.DoesNotExist) as e:
                             sys.stdout.write(f"Error getting group {name}: {e}\n")
         self._deactivate_extra_users(ldap_usernames)
+
         msg = "Sync of users ended successfully."
-        msg = f"{msg} Couldn't synchronize users: {error_names}" if error_names else f"{msg}"
+        if error_names:
+            msg = f"{msg}{os.linesep}Couldn't synchronize users: {error_names}"
+        if deleted_names:
+            msg = f"{msg}{os.linesep}Deleted users (inactive in ldap): {deleted_names}"
         logger.debug(msg)
 
     def _process_user_ldap_groups(self, user: User, user_dn: str) -> None:
@@ -242,6 +253,14 @@ def _process_user_ldap_groups(self, user: User, user_dn: str) -> None:
             group.user_set.add(user)
             sys.stdout.write(f"Add user {user} to group {ldap_group_name}\n")
 
+    @staticmethod
+    def _is_ldap_user_active(ldap_attrs: dict) -> bool:
+        target_attr = "useraccountcontrol"
+        if ldap_attrs.get(target_attr) and not hex(int(ldap_attrs[target_attr][0])).endswith("2"):
+            return True
+
+        return False
+
 
 if __name__ == "__main__":
     sync_ldap = SyncLDAP()

os/etc/crontabs/root

@@ -1,6 +1,7 @@
 # DO NOT EDIT THIS FILE - edit the master and reinstall.
 # (/tmp/crontab.nS0S9F/crontab installed on Wed Oct  5 09:29:23 2022)
 # (Cron version -- $Id: crontab.c,v 2.13 1994/01/17 03:20:37 vixie Exp $)
-0	8       */1     *       *       python /adcm/python/manage.py logrotate --target all
+0       8       */1     *       *       python /adcm/python/manage.py logrotate --target all
 0       10      */1     *       *       python /adcm/python/manage.py clearaudit
 */1     *       *       *       *       python /adcm/python/manage.py run_ldap_sync
+0       0       *       *       1       python /adcm/python/manage.py collect_statistics