Merge branch 'develop' into ITJamie-perf_tuning

.github/workflows/test_on_push.yml

@@ -10,20 +10,20 @@ jobs:
       - name: Install black
         run: pip install black
       - name: Build the stack
-        run: docker-compose -f docker-compose-ci.yml up -d
+        run: docker compose -f docker-compose-ci.yml up -d
       - name: Test
         run: bash docker/utils/ci_script.sh
       - name: Check safety
         if: ${{ success() }}
-        run: docker-compose exec -T -u alcali web safety check
+        run: docker compose exec -T -u alcali web safety check
       - name: Check format
         if: ${{ success() }}
         run: black --check .
       - name: Upload Codecov
         if: ${{ success() }}
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-        run: docker-compose exec -T -u alcali -e "$env:CODECOV_TOKEN" web codecov -X s3
+        run: docker compose exec -T -u alcali -e "$env:CODECOV_TOKEN" web codecov -X s3
       - name: Show logs
         if: ${{ failure() }}
-        run: docker-compose logs
+        run: docker compose logs

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7-slim-stretch
+FROM python:3.10-bullseye
 
 MAINTAINER Matt Melquiond
 
@@ -8,7 +8,7 @@ ARG USER_ID=1000
 # Upgrade System and Install dependencies
 RUN apt-get update \
   && seq 1 8 | xargs -I{} mkdir -p /usr/share/man/man{} \
-  && apt-get install -y --no-install-recommends -o DPkg::Options::=--force-confold netcat libmariadbclient-dev libpq-dev build-essential libldap2-dev libsasl2-dev ldap-utils git
+  && apt-get install -y --no-install-recommends -o DPkg::Options::=--force-confold netcat default-libmysqlclient-dev libpq-dev build-essential libldap2-dev libsasl2-dev ldap-utils git pkg-config
 
 # Upgrade pip
 RUN pip install --upgrade pip

README.md

@@ -34,7 +34,7 @@ If you just want to have a look, just clone the [repository](https://github.com/
 ```commandline
 git clone https://github.com/latenighttales/alcali.git
 cd alcali
-docker-compose up --scale minion=2
+docker compose up --scale minion=2
 ```
 
 
@@ -94,6 +94,12 @@ via GitHub Sponsors, or
 
 ## Changelog
 
+## [3006.3.0] dev
+
+- int: bugfix and deps update
+
+- feat: i18n (#353)
+
 ## [3003.1.0] - 2021-04-23
 
 - int: updated deps (#317)

VERSION

@@ -1 +1 @@
-3003.1.0
+3006.3.0

api/management/commands/check.py

@@ -8,7 +8,6 @@ class Command(BaseCommand):
     help = "Check access to database and that env var are set"
 
     def handle(self, *args, **options):
-
         unset = []
         for env in [
             "MASTER_MINION_ID",

api/management/commands/current_version.py

@@ -6,6 +6,5 @@ class Command(BaseCommand):
     help = "Return Alcali version"
 
     def handle(self, *args, **options):
-
         version = settings.VERSION
         self.stdout.write("alcali version {}".format(version))

api/management/commands/location.py

@@ -7,7 +7,6 @@ class Command(BaseCommand):
     help = "Return Alcali location"
 
     def handle(self, *args, **options):
-
         current_path = os.path.dirname(os.path.abspath(__file__))
         current_path = os.path.abspath(os.path.join(current_path, "..", "..", ".."))
         self.stdout.write("{}".format(current_path))

api/migrations/0001_initial.py

@@ -6,7 +6,6 @@
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = [

api/migrations/0002_jobtemplate.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [("api", "0001_initial")]
 
     operations = [

api/migrations/0003_auto_20200317_1719.py

@@ -12,7 +12,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [("api", "0002_jobtemplate")]
     operations = [
         migrations.RemoveField(model_name="usersettings", name="max_notifs"),

api/models.py

@@ -186,7 +186,6 @@ def conformity(self):
         return True
 
     def custom_conformity(self, fun, *args):
-
         # First, filter with fun.
         jobs = SaltReturns.objects.filter(fun=fun, id=self.minion_id).order_by(
             "-alter_time"

api/signals.py

@@ -8,14 +8,12 @@
 
 @receiver(post_save, sender=User)
 def create_user_settings(sender, instance, created, **kwargs):
-
     # Create related UserSettings when a new User is created.
     if created:
         UserSettings.objects.create(user=instance)
 
 
 @receiver(post_save, sender=User)
 def save_user_settings(sender, instance, **kwargs):
-
     # Save UserSettings when user is saved.
     instance.user_settings.save()

api/utils/__init__.py

@@ -95,7 +95,6 @@ def graph_data(period=7, **kwargs):
 
 
 def render_conformity(target=None):
-
     # First, a list of targets.
     if target:
         minions_all = [Minions.objects.get(minion_id=target)]

config/ldap_config.py

@@ -29,8 +29,14 @@
     "email": os.environ.get("AUTH_LDAP_USER_ATTR_MAP_EMAIL", "mail"),
 }
 
-if os.environ.get("AUTH_LDAP_START_TLS"):
-    AUTH_LDAP_START_TLS = True
+AUTH_LDAP_START_TLS = os.environ.get("AUTH_LDAP_START_TLS", "False").lower() in (
+    "y",
+    "yes",
+    "t",
+    "true",
+    "on",
+    "1",
+)
 
 # This is the default.
 AUTH_LDAP_ALWAYS_UPDATE_USER = True

config/settings.py

@@ -97,6 +97,7 @@
 DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.{}".format(os.environ["DB_BACKEND"]),
+        "ATOMIC_REQUESTS": True,
         "NAME": os.environ.get("DB_NAME"),
         "USER": os.environ.get("DB_USER"),
         "PASSWORD": os.environ.get("DB_PASS"),
@@ -107,7 +108,7 @@
 
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
-
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 AUTH_PASSWORD_VALIDATORS = [
     {
         "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"
@@ -153,12 +154,12 @@
 #     'REFRESH_TOKEN_LIFETIME': timedelta(days=1)}
 
 # # TODO!
-# LOGGING = {
-#     "version": 1,
-#     "disable_existing_loggers": False,
-#     "handlers": {"console": {"class": "logging.StreamHandler"}},
-#     "loggers": {"django_auth_ldap": {"level": "DEBUG", "handlers": ["console"]}},
-# }
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {"console": {"class": "logging.StreamHandler", "level": "DEBUG"}},
+    "loggers": {"django_auth_ldap": {"level": "DEBUG", "handlers": ["console"]}},
+}
 #
 # Get version from file.
 try:

config/urls.py

@@ -13,7 +13,6 @@
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
-from django.urls import include
-from django.conf.urls import url
+from django.urls import include, re_path
 
-urlpatterns = [url(r"^", include("api.urls"))]
+urlpatterns = [re_path(r"^", include("api.urls"))]

docker-compose-ci.yml

@@ -34,6 +34,7 @@ services:
     command: honcho start
     depends_on:
       - web
+      - db
     environment:
       - PYTHONUNBUFFERED=true
     ports:

docker/Dockerfile-dev

@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.10-slim-bullseye
 
 MAINTAINER Matt Melquiond
 
@@ -8,7 +8,7 @@ ARG USER_ID=1000
 # Upgrade System and Install dependencies
 RUN apt-get update \
   && seq 1 8 | xargs -I{} mkdir -p /usr/share/man/man{} \
-  && apt-get install -y --no-install-recommends -o DPkg::Options::=--force-confold netcat libmariadbclient-dev libpq-dev build-essential libldap2-dev libsasl2-dev ldap-utils
+  && apt-get install -y --no-install-recommends -o DPkg::Options::=--force-confold netcat default-libmysqlclient-dev libpq-dev build-essential libldap2-dev libsasl2-dev ldap-utils pkg-config
 
 # Upgrade pip
 RUN pip install --upgrade pip

docker/Dockerfile-master

@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.10-bullseye
 
 # Upgrade System and Install dependencies
 RUN apt-get update && \
@@ -13,8 +13,9 @@ COPY saltconfig/etc/minion /etc/salt/minion
 RUN echo "id: master">>/etc/salt/minion
 
 # Install python dependencies
-RUN apt-get install -y -o DPkg::Options::=--force-confold salt-api python3-openssl
+RUN apt-get install -y -o DPkg::Options::=--force-confold salt-api python3-openssl default-libmysqlclient-dev pkg-config
 RUN pip install honcho
+RUN salt-pip install mysqlclient
 
 # Copy needed files
 COPY saltconfig/etc/master /etc/salt/master

docker/Dockerfile-minion

@@ -1,12 +1,12 @@
-FROM python:3.7-slim-buster
+FROM python:3.10-bullseye
 
 # Upgrade System and Install dependencies
 RUN apt-get update && \
   apt-get upgrade -y -o DPkg::Options::=--force-confold && \
   apt-get install -y -o DPkg::Options::=--force-confold curl netcat
 
 # Install Latest Salt from the stable Branch
-RUN curl -L https://bootstrap.saltstack.com | sh -s -- -x python3 stable latest
+RUN curl -L https://bootstrap.saltstack.com | sh -s -- -D -r -P stable
 
 # Set master
 COPY saltconfig/etc/minion /etc/salt/minion

docker/saltconfig/etc/master

@@ -235,9 +235,22 @@ mysql.port: 3306
 #  pam:
 #    fred:
 #      - test.*
+# client_acl:
+#   admin:
+#     - .*
+
+
+eauth_acl_module: alcali
+keep_acl_in_token: true
+
 external_auth:
   rest:
     ^url: http://web:8000/api/token/verify/
+    admin:
+      - '.*'
+      - '@runner'
+      - '@wheel'
+      - '@jobs'
     apiuser:
       - test.ping
       - minion\*:
@@ -249,11 +262,13 @@ external_auth:
           - state.*
       - 'G@os:RedHat':
           - kmod.*
+  alcali:
     admin:
-      - .*
+      - '.*'
+      - '@local'
       - '@runner'
       - '@wheel'
-  alcali:
+      - '@jobs'
     apiuser:
       - test.ping
       - minion\*:
@@ -265,10 +280,6 @@ external_auth:
         - state.*
       - 'G@os:RedHat':
         - kmod.*
-    admin:
-      - .*
-      - '@runner'
-      - '@wheel'
 
 # Time (in seconds) for a newly generated token to live. Default: 12 hours
 #token_expire: 43200
@@ -691,3 +702,14 @@ rest_cherrypy:
   debug: True
   ssl_crt: /etc/pki/tls/certs/localhost.crt
   ssl_key: /etc/pki/tls/certs/localhost.key
+
+netapi_enable_clients:
+  - local
+  - local_async
+  - local_batch
+  - local_subset
+  - runner
+  - runner_async
+  - ssh
+  - wheel
+  - wheel_async