Merge pull request #1178 from arenadata/develop

Release 2021.10.20

.flake8

@@ -3,6 +3,8 @@
 ignore =
     I801,
     E501,
+    W503,
+    PT004,
 
 exclude =
     .pytest_cache,

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+* @arenadata/python
+/tests/functional/ @arenadata/python-qa
+/tests/ui_tests/ @arenadata/python-qa
+/tests/api/ @arenadata/python-qa
+/web/ @arenadata/angular

.gitignore

@@ -6,6 +6,7 @@
 .cache
 .coverage
 .idea/
+.venv
 .version
 .vscode
 /allure-results/

.pre-commit-config.yaml

@@ -5,20 +5,3 @@ repos:
       - id: black
         language_version: python3.8
         files: '^python/'
-  - repo: https://github.com/pre-commit/mirrors-pylint
-    rev: 'v3.0.0a3'
-    hooks:
-    - id: pylint
-      additional_dependencies:
-        - ansible==2.8.8
-        - croniter
-        - django
-        - django-background-tasks
-        - django-filter
-        - djangorestframework
-        - drf-flex-fields
-        - pylint-django
-        - pytest
-        - simplejson
-        - social-auth-app-django
-        - version_utils

Makefile

@@ -2,7 +2,7 @@
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
 
 ADCMBASE_IMAGE ?= hub.arenadata.io/adcm/base
-ADCMBASE_TAG ?= 20210317134752
+ADCMBASE_TAG ?= 20210927130343
 
 APP_IMAGE ?= hub.adsw.io/adcm/adcm
 APP_TAG ?= $(subst /,_,$(BRANCH_NAME))
@@ -28,7 +28,7 @@ describe: ## Create .version file with output of describe
 	./gues_version.sh
 
 buildss: ## Build status server
-	@docker run -i --rm -v $(CURDIR)/go:/code -w /code  golang:1.15-alpine sh -c "apk --update add make git && make && rm -f /code/adcm/go.sum"
+	@docker run -i --rm -v $(CURDIR)/go:/code -w /code  golang:1.15-alpine3.13 sh -c "apk --update add make git && make && rm -f /code/adcm/go.sum"
 
 buildjs: ## Build client side js/html/css in directory wwwroot
 	@docker run -i --rm -v $(CURDIR)/wwwroot:/wwwroot -v $(CURDIR)/web:/code -w /code  node:12-alpine ./build.sh
@@ -57,32 +57,37 @@ unittests: ## Run unittests
 	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/tests/base $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) /bin/sh -e ./run_test.sh
 
 pytest: ## Run functional tests
-	docker pull ci.arenadata.io/functest:3.8.6.slim.buster-x64
+	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster-x64
 	docker run -i --rm --shm-size=4g -v /var/run/docker.sock:/var/run/docker.sock --network=host -v $(CURDIR)/:/adcm -w /adcm/ \
 	-e BUILD_TAG=${BUILD_TAG} -e ADCMPATH=/adcm/ -e PYTHONPATH=${PYTHONPATH}:python/ \
 	-e SELENOID_HOST="${SELENOID_HOST}" -e SELENOID_PORT="${SELENOID_PORT}" \
-	ci.arenadata.io/functest:3.8.6.slim.buster-x64 /bin/sh -e \
-	./pytest.sh --adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))'
+	hub.adsw.io/library/functest:3.8.6.slim.buster-x64 /bin/sh -e \
+	./pytest.sh -m "not full" --adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))'
 
 pytest_release: ## Run functional tests on release
-	docker pull ci.arenadata.io/functest:3.8.6.slim.buster.firefox-x64
+	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster.firefox-x64
 	docker run -i --rm --shm-size=4g -v /var/run/docker.sock:/var/run/docker.sock --network=host -v $(CURDIR)/:/adcm -w /adcm/ \
 	-e BUILD_TAG=${BUILD_TAG} -e ADCMPATH=/adcm/ -e PYTHONPATH=${PYTHONPATH}:python/ \
 	-e SELENOID_HOST="${SELENOID_HOST}" -e SELENOID_PORT="${SELENOID_PORT}" \
-	ci.arenadata.io/functest:3.8.6.slim.buster.firefox-x64 /bin/sh -e \
-	./pytest.sh --firefox --adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))'
+	hub.adsw.io/library/functest:3.8.6.slim.buster.firefox-x64 /bin/sh -e \
+	./pytest.sh --adcm-image='hub.adsw.io/adcm/adcm:$(subst /,_,$(BRANCH_NAME))'
 
 ng_tests: ## Run Angular tests
-	docker pull ci.arenadata.io/functest:3.8.6.slim.buster-x64
-	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/web ci.arenadata.io/functest:3.8.6.slim.buster-x64 ./ng_test.sh
+	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster-x64
+	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/web hub.adsw.io/library/functest:3.8.6.slim.buster-x64 ./ng_test.sh
 
 linters : ## Run linters
-	docker pull ci.arenadata.io/pr-builder:3-x64
-	docker run -i --rm -v $(CURDIR)/:/source -w /source ci.arenadata.io/pr-builder:3-x64 /linters.sh shellcheck pylint pep8
+	docker pull hub.adsw.io/library/pr-builder:3-x64
+	docker run -i --rm -v $(CURDIR)/:/source -w /source hub.adsw.io/library/pr-builder:3-x64 \
+        /bin/bash -xeo pipefail -c "/linters.sh shellcheck pylint pep8 && \
+        /linters.sh -b ./tests -f ../tests pylint && \
+        /linters.sh -f ./tests black && \
+        /linters.sh -f ./tests/functional flake8_pytest_style && \
+        /linters.sh -f ./tests/ui_tests flake8_pytest_style"
 
 npm_check: ## Run npm-check
 	docker run -i --rm -v $(CURDIR)/wwwroot:/wwwroot -v $(CURDIR)/web:/code -w /code  node:12-alpine ./npm_check.sh
 
 django_tests : ## Run django tests.
 	docker pull $(ADCMBASE_IMAGE):$(ADCMBASE_TAG)
-	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/ $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) python python/manage.py test cm
+	docker run -e DJANGO_SETTINGS_MODULE=adcm.test -i --rm -v $(CURDIR)/:/adcm -w /adcm/ $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) python python/manage.py test cm

README.md

@@ -4,7 +4,7 @@
 
 # Arenadata Cluster Manager
 
-That is Areandata Cluster Manager Project (aka Chapelnik)
+That is Arenadata Cluster Manager Project (aka Chapelnik)
 
 # Documentation
 
@@ -29,7 +29,7 @@ There is a Makefile in repo. It could be used for building application.
 
 ## Fast start with make
 
-You have to have GNU Make on your host and Docker daemon accessable for a user. Besides you have to have access to ci.arenadata.io
+You have to have GNU Make on your host and Docker daemon accessible for a user. Also, you have to have access to ci.arenadata.io
 
 ```sh
 # Clone repo
@@ -41,11 +41,11 @@ cd adcm
 make build
 ```
 
-That will be image ci.arenadata.io/adcm:<branch_name> as a result of the operation above.
+That will be an image hub.adsw.io/adcm/adcm:<branch_name> as a result of the operation above.
 
 ## Makefile description
 
-Makefile has selfdocumented help message. Just type.
+Makefile has self-documented help message. Just type.
 
 ```sh
 $ make
@@ -62,7 +62,7 @@ And check out the description for every operation available.
 
 ## Pre-commit hook
 
-We are using black, pylint and pre-commit to care about code formating and linting.
+We are using black, pylint and pre-commit to care about code formatting and linting.
 
 So you have to install pre-commit hook before you do something with code.
 

assemble/base/build.sh

@@ -8,4 +8,4 @@ pip install --no-cache-dir -r /requirements-base.txt
 rm /requirements-base.txt
 apk del git
 apk del .pynacl_deps
-rm /etc/nginx/conf.d/default.conf
+rm /etc/nginx/http.d/default.conf

assemble/base/requirements-base.txt

@@ -6,10 +6,14 @@ django-cors-headers
 django-filter
 django-rest-swagger
 djangorestframework
+drf-flex-fields==0.9.1
+drf-extensions
 django-background-tasks
 social-auth-app-django
 git+git://github.com/arenadata/django-generate-secret-key.git
-jinja2
+# TODO jinja2 and markupsafe should be unpinned after https://arenadata.atlassian.net/browse/ADCM-2089 fixup
+Jinja2==2.11.3
+MarkupSafe==1.1.1
 markdown
 mitogen
 pyyaml
@@ -24,3 +28,5 @@ apache-libcloud
 jmespath
 lxml
 pycrypto
+--extra-index-url https://ci.arenadata.io/artifactory/api/pypi/python-packages/simple
+adwp-events==0.1.7

go/adcm/status/init.go

@@ -96,6 +96,10 @@ func startHTTP(httpPort string, hub Hub) {
 	router.GET("/api/v1/cluster/", authWrap(hub, clusterList))
 	router.GET("/api/v1/cluster/:clusterid/", authWrap(hub, showCluster))
 	router.GET("/api/v1/cluster/:clusterid/service/:serviceid/", authWrap(hub, showService))
+	router.GET(
+		"/api/v1/cluster/:clusterid/service/:serviceid/component/:compid/",
+		authWrap(hub, showComp),
+	)
 
 	router.GET("/api/v1/servicemap/", authWrap(hub, showServiceMap))
 	router.POST("/api/v1/servicemap/", authWrap(hub, postServiceMap))

os/etc/nginx/http.d/adcm.conf

@@ -0,0 +1,38 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+upstream django {
+    server unix:///run/adcm.sock; 
+}
+
+
+# That variable sets to True if we have non HTTP 200 response.
+# That is usefull to disable noise from recurent operations.
+map $status $abnormal {
+    ~^200  0;
+    default 1;
+}
+
+
+server {
+    listen      8000;
+
+    include "http.d/proxy.inc";
+}
+
+server {
+    listen      8443 ssl;
+
+    ssl_certificate         /adcm/data/conf/ssl/cert.pem;
+    ssl_certificate_key     /adcm/data/conf/ssl/key.pem;
+
+    include "http.d/proxy.inc";
+}

conf/nginx/adcm.conf → os/etc/nginx/http.d/proxy.inc

@@ -1,37 +1,11 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-upstream django {
-    server unix:///run/adcm.sock; 
-}
-
-
-# That variable sets to True if we have non HTTP 200 response.
-# That is usefull to disable noise from recurent operations.
-map $status $abnormal {
-    ~^200  0;
-    default 1;
-}
-
-
-server {
-    listen      8000;
     charset     utf-8;
 
+    root /adcm/wwwroot;
+
     client_max_body_size 100M;
     client_body_temp_path /adcm/data/tmp/nginx_client_temp 1 2;
     client_body_timeout 300s;
 
-    root /adcm/wwwroot;
-
     location / {
         if ($arg_nocache) {
             add_header Cache-Control no-cache;
@@ -72,4 +46,3 @@ server {
         proxy_pass http://127.0.0.1:8020/;
         proxy_set_header Host $http_host;
     }
-}

os/etc/service/init/run

@@ -41,9 +41,16 @@ if [ ! -f "${adcmvar}"/cluster.db ]; then
     "${adcmroot}"/python/manage.py generate_secret_key "${adcmsecretfile}"
 fi
 
+if [ ! \( -f "${adcmuserconf}/ssl/cert.pem" -a -f "${adcmuserconf}/ssl/key.pem" \) ]; then
+    mkdir -p "${adcmuserconf}/ssl"
+    cd "${adcmuserconf}/ssl"
+    openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 3650 -subj "/C=RU/ST=Moscow/L=Moscow/O=Arenadata Software LLC/OU=Release/CN=ADCM" -nodes
+
+fi
+
+"${adcmroot}"/python/backupdb.py
 "${adcmroot}"/python/manage.py migrate
 "${adcmroot}"/python/init_db.py
-"${adcmroot}"/python/backupdb.py
 touch "${initreadyfile}"
 
 # That is sleep forever case because I have no idea how to force

pylintrc

@@ -1,5 +1,8 @@
 [MASTER]
 
+# ignore tests dirs because they contain their pylint settings and run separately
+ignore-paths=^tests/.*$
+
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code

pytest.sh

@@ -18,7 +18,7 @@ pip3 install -r requirements-test.txt
 find . -name "*.pyc" -type f -delete
 find . -name "__pycache__" -type d -delete
 { # try
-    pytest tests/functional tests/ui_tests -s -v -n auto --maxfail 30 \
+    pytest tests/api tests/functional tests/ui_tests -s -v -n auto --maxfail 30 \
     --showlocals --alluredir ./allure-results/ --durations=20 -p allure_pytest \
     --reruns 2 --remote-executor-host "$SELENOID_HOST" --timeout=1080 "$@" &&
     chmod -R o+xw allure-results