diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..26ef8e2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,33 @@ +name: Tests + +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9'] + db-type: ['pgsql', 'mysql'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Make scripts executable + run: | + chmod +x test_in_docker.sh + chmod +x test.sh + if [ -f "tests/db/${{ matrix.db-type }}/init_db.sh" ]; then + chmod +x "tests/db/${{ matrix.db-type }}/init_db.sh" + fi + + - name: Run tests in Docker + run: | + ./test_in_docker.sh \ + --python-version "${{ matrix.python-version }}" \ + --db-type "${{ matrix.db-type }}" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dec36e4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.8-bullseye + +# Install required Python packages +RUN pip install foliantcontrib.utils requests psycopg2-binary pyodbc + +WORKDIR /app +ENTRYPOINT ["./test.sh"] diff --git a/README.md b/README.md index 3197c29..a1c9d1d 100755 --- a/README.md +++ b/README.md @@ -104,6 +104,8 @@ preprocessors: password: !env DBDOC_PASS doc: True scheme: True + strict: False + trusted_connection: False filters: ... doc_template: dbdoc.j2 @@ -141,6 +143,12 @@ preprocessors: `scheme` : If `true` — the platuml code for database scheme will be generated. Default: `true` +`strict` +: If `true` — the build will fail if connection to database cannot be established. If `false` — the preprocessor will skip the tag with warning. Default: `false` + +`trusted_connection` +: Specific option for MS SQL Server. If true - will use Windows Authentication (Trusted Connection) instead of username/password. Default: false. Requires proper ODBC driver configuration. + `filters` : SQL-like operators for filtering the results. More info in the **Filters** section. @@ -309,6 +317,47 @@ If you wish to create your own template, the default ones may be a good starting * [Default **SQL Server doc** template.](https://github.com/foliant-docs/foliantcontrib.dbdoc/blob/master/foliant/preprocessors/dbdoc/mssql/templates/doc.j2) * [Default **SQL Server scheme** template.](https://github.com/foliant-docs/foliantcontrib.dbdoc/blob/master/foliant/preprocessors/dbdoc/mssql/templates/doc.j2) +## Tests + +For run tests, use: +```bash +./test_in_docker.sh --python-version "3.9" --db-type "mysql" +``` + +**Options:** +`--python-version ` – Specifies Python version for test environment. Available_: 3.8, 3.9, 3.10 etc. + +`--db-type ` – Chooses database type for testing. Available_: mysql, pgsql. + +**Usage Examples** + +```bash +# Basic usage with defaults +./test_in_docker.sh + +# Specific Python and database +./test_in_docker.sh --python-version "3.10" --db-type "pgsql" + +# Only change database type +./test_in_docker.sh --db-type "mysql" + +# Only change Python version +./test_in_docker.sh --python-version "3.9" +``` + +**What It Does:** +1. Starts Docker container with specified Python version. +2. Initializes chosen database type with test data. +3. Runs test suite. +4. Cleans up resources after completion. +5. Returns exit code based on test results. + +**Notes** +- Requires Docker installed; +- Test data is automatically loaded from `test_data/` directory; +- Results are displayed in console with color formatting; +- Exit code 0 = success, 1 = test failures. + ## Troubleshooting If you get errors during build, especially errors concerning connection to the database, you have to make sure that you are supplying the right parameters. @@ -379,3 +428,28 @@ con = pyodbc.connect( "UID=Usernam;PWD=Password_0" ) ``` + +**Microsoft SQL Server Authentication Issues** + +When using MS SQL Server, you have two authentication options: + +1. SQL Server Authentication (username/password): + + ```yaml + trusted_connection: false + user: your_username + password: your_password + ``` + +2. Windows Authentication (Trusted Connection): + + ```yaml + trusted_connection: true + # no user/password needed + ``` + +For Windows Authentication to work: + +- Make sure your ODBC driver supports Trusted Connections. +- The account running Foliant must have proper database permissions. +- On Linux/Mac, you may need to configure Kerberos for cross-platform authentication. diff --git a/changelog.md b/changelog.md index 5f8e7f8..4744c2a 100755 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,7 @@ +# 0.1.9 +- Add: strict option. +- Add: tests. + # 0.1.8 - DBMS python connectors are only imported on use. diff --git a/foliant/preprocessors/dbdoc/mssql/main.py b/foliant/preprocessors/dbdoc/mssql/main.py index 31c6f59..7b2c06a 100644 --- a/foliant/preprocessors/dbdoc/mssql/main.py +++ b/foliant/preprocessors/dbdoc/mssql/main.py @@ -1,3 +1,4 @@ +import os from copy import deepcopy from logging import getLogger @@ -9,6 +10,8 @@ from .queries import TriggersQuery from .queries import ViewsQuery from foliant.preprocessors.dbdoc.base.main import DBRendererBase +from foliant.utils import output + logger = getLogger('unbound.dbdoc.mssql') @@ -29,7 +32,8 @@ class MSSQLRenderer(DBRendererBase): 'functions', 'triggers', 'views' - ] + ], + 'strict': False } module_name = __name__ @@ -46,24 +50,34 @@ def connect(self): 'and make sure that MS SQL Server is installed on the machine' ) - if self.options['trusted_connection']: - connection_string = ( - f"DRIVER={self.options['driver']};" - f"SERVER={self.options['host']},{self.options['port']};" - f"DATABASE={self.options['dbname']};Trusted_Connection=yes" - ) + try: + if self.options['trusted_connection']: + connection_string = ( + f"DRIVER={self.options['driver']};" + f"SERVER={self.options['host']},{self.options['port']};" + f"DATABASE={self.options['dbname']};Trusted_Connection=yes" + ) - else: - connection_string = ( - f"DRIVER={self.options['driver']};" - f"SERVER={self.options['host']},{self.options['port']};" - f"DATABASE={self.options['dbname']};" - f"UID={self.options['user']};PWD={self.options['password']}" + else: + connection_string = ( + f"DRIVER={self.options['driver']};" + f"SERVER={self.options['host']},{self.options['port']};" + f"DATABASE={self.options['dbname']};" + f"UID={self.options['user']};PWD={self.options['password']}" + ) + logger.debug( + f"Trying to connect: {connection_string}" ) - logger.debug( - f"Trying to connect: {connection_string}" - ) - self.con = pyodbc.connect(connection_string) + self.con = pyodbc.connect(connection_string) + logger.debug("Successfully connected to the database") + except pyodbc.Error as e: + msg = f"\MS SQL database database connection error: {e}" + if self.options['strict']: + logger.error(msg) + output(f'ERROR: {msg}. Exit.') + os._exit(1) + else: + logger.debug(f'{msg}. Skipping.') def collect_datasets(self) -> dict: diff --git a/foliant/preprocessors/dbdoc/mysql/main.py b/foliant/preprocessors/dbdoc/mysql/main.py index cfc7b5b..3528f8e 100644 --- a/foliant/preprocessors/dbdoc/mysql/main.py +++ b/foliant/preprocessors/dbdoc/mysql/main.py @@ -1,3 +1,4 @@ +import os from copy import deepcopy from logging import getLogger @@ -9,6 +10,8 @@ from .queries import TriggersQuery from .queries import ViewsQuery from foliant.preprocessors.dbdoc.base.main import DBRendererBase +from foliant.utils import output + logger = getLogger('unbound.dbdoc.mysql') @@ -27,7 +30,8 @@ class MySQLRenderer(DBRendererBase): 'functions', 'triggers', 'views' - ] + ], + 'strict': False } module_name = __name__ @@ -45,18 +49,27 @@ def connect(self): 'and make sure that MySQL Client is installed on the machine' ) - logger.debug( - f"Trying to connect: host={self.options['host']} port={self.options['port']}" - f" dbname={self.options['dbname']}, user={self.options['user']} " - f"password={self.options['password']}." - ) - self.con = _mysql.connect( - host=self.options['host'], - port=self.options['port'], - user=self.options['user'], - passwd=self.options['password'], - db=self.options['dbname'] - ) + try: + logger.debug( + f"Trying to connect: host={self.options['host']} port={self.options['port']}" + f" dbname={self.options['dbname']}, user={self.options['user']} " + f"password={self.options['password']}." + ) + self.con = _mysql.connect( + host=self.options['host'], + port=self.options['port'], + user=self.options['user'], + password=self.options['password'], + database=self.options['dbname'] + ) + except _mysql.Error as e: + msg = f"\nMySQL database error: {e}" + if self.options['strict']: + logger.error(msg) + output(f'ERROR: {msg}. Exit.') + os._exit(1) + else: + logger.debug(f'{msg}. Skipping.') def collect_datasets(self) -> dict: @@ -94,7 +107,6 @@ def collect_datasets(self) -> dict: q_triggers = TriggersQuery(self.con, filters) logger.debug(f'Triggers query:\n\n {q_triggers.sql}') result['triggers'] = q_triggers.run() - return result def collect_tables(self, diff --git a/foliant/preprocessors/dbdoc/oracle/main.py b/foliant/preprocessors/dbdoc/oracle/main.py index f90cfc3..c2e1d18 100644 --- a/foliant/preprocessors/dbdoc/oracle/main.py +++ b/foliant/preprocessors/dbdoc/oracle/main.py @@ -1,3 +1,4 @@ +import os from copy import deepcopy from logging import getLogger @@ -9,6 +10,7 @@ from .queries import ViewsQuery from ..base.main import LibraryNotInstalledError from foliant.preprocessors.dbdoc.base.main import DBRendererBase +from foliant.utils import output logger = getLogger('unbound.dbdoc.oracle') @@ -28,7 +30,8 @@ class OracleRenderer(DBRendererBase): 'functions', 'triggers', 'views' - ] + ], + 'strict': False } module_name = __name__ @@ -50,13 +53,22 @@ def connect(self): f" dbname={self.options['dbname']}, user={self.options['user']} " f"password={self.options['password']}." ) - self.con = cx_Oracle.connect( - f"{self.options['user']}/{self.options['password']}@" - f"{self.options['host']}:{self.options['port']}/" - f"{self.options['dbname']}", - encoding='UTF-8', - nencoding='UTF-8' - ) + try: + self.con = cx_Oracle.connect( + f"{self.options['user']}/{self.options['password']}@" + f"{self.options['host']}:{self.options['port']}/" + f"{self.options['dbname']}", + encoding='UTF-8', + nencoding='UTF-8' + ) + except cx_Oracle.Error as e: + msg = f"\nOracle database connection error: {e}" + if self.options['strict']: + logger.error(msg) + output(f'ERROR: {msg}. Exit.') + os._exit(1) + else: + logger.debug(f"{msg}. Skipping.") def collect_datasets(self) -> dict: diff --git a/foliant/preprocessors/dbdoc/oracle/queries.py b/foliant/preprocessors/dbdoc/oracle/queries.py index 6845518..cbffcc8 100644 --- a/foliant/preprocessors/dbdoc/oracle/queries.py +++ b/foliant/preprocessors/dbdoc/oracle/queries.py @@ -129,7 +129,7 @@ class ColumnsQuery(QueryBase): col.DATA_LENGTH, col.DATA_PRECISION, com.COMMENTS - FROM user_tab_columns col + FROM all_tab_columns col JOIN all_tables tab ON col.TABLE_NAME = tab.TABLE_NAME LEFT JOIN user_col_comments com diff --git a/foliant/preprocessors/dbdoc/pgsql/main.py b/foliant/preprocessors/dbdoc/pgsql/main.py index 649601a..b2b192c 100644 --- a/foliant/preprocessors/dbdoc/pgsql/main.py +++ b/foliant/preprocessors/dbdoc/pgsql/main.py @@ -1,3 +1,4 @@ +import os from copy import deepcopy from logging import getLogger @@ -10,6 +11,7 @@ from .queries import TriggersQuery from .queries import ViewsQuery from foliant.preprocessors.dbdoc.base.main import DBRendererBase +from foliant.utils import output logger = getLogger('unbound.dbdoc.pgsql') @@ -29,7 +31,8 @@ class PGSQLRenderer(DBRendererBase): 'views', 'functions', 'triggers' - ] + ], + 'strict': False } module_name = __name__ @@ -50,13 +53,23 @@ def connect(self): f" dbname={self.options['dbname']}, user={self.options['user']} " f"password={self.options['password']}." ) - self.con = psycopg2.connect( - f"host='{self.options['host']}' " - f"port='{self.options['port']}' " - f"dbname='{self.options['dbname']}' " - f"user='{self.options['user']}'" - f"password='{self.options['password']}'" - ) + + try: + self.con = psycopg2.connect( + f"host='{self.options['host']}' " + f"port='{self.options['port']}' " + f"dbname='{self.options['dbname']}' " + f"user='{self.options['user']}'" + f"password='{self.options['password']}'" + ) + except psycopg2.Error as e: + msg = f"\nPostgreSQL database connection error: {e}" + if self.options['strict']: + logger.error(msg) + output(f'ERROR: {msg}. Exit.') + os._exit(1) + else: + logger.debug(f"{msg}. Skipping.") def collect_datasets(self) -> dict: diff --git a/setup.py b/setup.py index 78d8aee..c116fb3 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ description=SHORT_DESCRIPTION, long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown', - version='0.1.8', + version='0.1.9', author='Daniil Minukhin', author_email='ddddsa@gmail.com', packages=find_namespace_packages(exclude=['*.test', 'foliant', '*.templates']), @@ -30,7 +30,7 @@ platforms='any', install_requires=[ 'foliant>=1.0.5', - 'foliantcontrib.utils>=1.0.2', + 'foliantcontrib.utils==1.0.3', 'foliantcontrib.plantuml>=1.0.10', 'jinja2', 'PyYAML' diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..e4a11d5 --- /dev/null +++ b/test.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +db_type=${1-pgsql} + +# before testing make sure that you have installed the fresh version of preprocessor: +pip3 install . +# also make sure that fresh version of test framework is installed: +pip3 install --upgrade foliantcontrib.test_framework + +# install dependencies +pip3 install sqlalchemy mysqlclient + +python3 -m unittest discover -v -p "*${db_type}*" diff --git a/test_in_docker.sh b/test_in_docker.sh new file mode 100755 index 0000000..133d1a9 --- /dev/null +++ b/test_in_docker.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +set -e # Exit on error + +# Default values +PYTHON_VERSIONS=("3.8" "3.9") +DB_TYPE=("pgsql" "mysql") + +CONTAINER_NAME="testdb" +NETWORK_NAME="testnetwork" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --python-version) + PYTHON_VERSIONS=("$2") + shift 2 + ;; + --db-type) + DB_TYPE=("$2") + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +echo "=== Test Configuration ===" +echo "Python versions: ${PYTHON_VERSIONS[*]}" +echo "Database type: ${DB_TYPE[*]}" +echo "==========================" + +for version in "${PYTHON_VERSIONS[@]}"; do + for type in "${DB_TYPE[@]}"; do + echo "=== Testing with Python ${version} with DB ${type} ===" + + cat > Dockerfile << EOF +FROM python:${version}-bullseye + +# Install required Python packages +RUN pip install foliantcontrib.utils requests psycopg2-binary pyodbc + +WORKDIR /app +ENTRYPOINT ["./test.sh"] +EOF + echo "Building test image..." + docker build . -t test-foliant:${version} --no-cache + + if ! docker network inspect "${NETWORK_NAME}" >/dev/null 2>&1; then + docker network create "${NETWORK_NAME}" + fi + + ./tests/db/${type}/init_db.sh ${CONTAINER_NAME} ${NETWORK_NAME} || exit 1 + + echo "Running tests with Docker access..." + docker rm -f test-foliant:${version} 2>/dev/null || true + docker run --rm \ + --network ${NETWORK_NAME} \ + -v "./:/app/" \ + -w /app \ + test-foliant:${version} \ + "${type}" + + echo "Cleaning up..." + rm -f Dockerfile + docker rm -f ${CONTAINER_NAME} 2>/dev/null || true + docker rmi test-foliant:${version} 2>/dev/null || true + docker network rm ${NETWORK_NAME} 2>/dev/null || true + + echo "=== Completed Python ${version} with ${type} tests ===" + echo "" + done +done \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/db/mysql/data/01-init.sql b/tests/db/mysql/data/01-init.sql new file mode 100644 index 0000000..79e4941 --- /dev/null +++ b/tests/db/mysql/data/01-init.sql @@ -0,0 +1,19 @@ +CREATE TABLE users ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100), + email VARCHAR(100) UNIQUE +); + +INSERT INTO users (name, email) VALUES +('john_doe', 'john@example.com'), +('jane_smith', 'jane@example.com'); + +CREATE TABLE products ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100), + price DECIMAL(10,2) +); + +INSERT INTO products (name, price) VALUES +('apple', 50000.00), +('banana', 1500.50); diff --git a/tests/db/mysql/init_db.sh b/tests/db/mysql/init_db.sh new file mode 100755 index 0000000..1760ede --- /dev/null +++ b/tests/db/mysql/init_db.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +container=${1:-testdb} +network=${2:-testnetwork} + +docker rm -f ${container} +docker run -d --name ${container} \ + --network ${network} \ + -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ + -e MYSQL_USER=testuser \ + -e MYSQL_PASSWORD=testpassword \ + -e MYSQL_DATABASE=testdb \ + -v $(pwd)/tests/db/mysql/data:/docker-entrypoint-initdb.d \ + -p 3306:3306 \ + mysql:8 diff --git a/tests/db/pgsql/data/01-init.sql b/tests/db/pgsql/data/01-init.sql new file mode 100644 index 0000000..2a40dbf --- /dev/null +++ b/tests/db/pgsql/data/01-init.sql @@ -0,0 +1,19 @@ +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + email VARCHAR(100) UNIQUE +); + +INSERT INTO users (name, email) VALUES +('john_doe', 'john@example.com'), +('jane_smith','jane@example.com'); + +CREATE TABLE products ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + price DECIMAL(10,2) +); + +INSERT INTO products (name, price) VALUES +('apple', 50000.00), +('banana', 1500.50); diff --git a/tests/db/pgsql/init_db.sh b/tests/db/pgsql/init_db.sh new file mode 100755 index 0000000..05d4013 --- /dev/null +++ b/tests/db/pgsql/init_db.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +container=${1:-testdb} +network=${2:-testnetwork} + +docker rm -f ${container} +docker run -d --name ${container} \ + --network ${network} \ + -e POSTGRES_PASSWORD=password \ + -e POSTGRES_DB=testdb \ + -v $(pwd)/tests/db/pgsql/data:/docker-entrypoint-initdb.d \ + -p 5432:5432 \ + postgres:15 + diff --git a/tests/test_mysql_basic.py b/tests/test_mysql_basic.py new file mode 100644 index 0000000..c45c634 --- /dev/null +++ b/tests/test_mysql_basic.py @@ -0,0 +1,84 @@ +from unittest import TestCase +from unittest.mock import patch +from foliant_test.preprocessor import PreprocessorTestFramework + +class TestDbdocMySQL(TestCase): + """MySQL tests""" + def setUp(self): + self.ptf = PreprocessorTestFramework('dbdoc') + self.ptf.options = {} + + def test_simple_documentation_mysql(self): + """mysql test""" + self.ptf.options = { + 'dbms': 'mysql', + 'host': 'testdb', + 'dbname': 'testdb', + 'user': 'testuser', + 'password': 'testpassword', + 'port': 3306, + 'doc': True, + 'scheme': False, + 'filters':{ + 'eq': {'table_name':'users'}, + }, + 'components':['tables'] + } + + input_files = { + 'index.md': '# Database Documentation\n\n' + } + expected_files = { + 'index.md': '''# Database Documentation\n\n + + +# Tables + + +## users + + + +column | nullable | type | descr | fkey +------ | -------- | ---- | ----- | ---- +id | NO | int | | +name | YES | varchar | | +email | YES | varchar | | + + + + + + + + +''' + } + + self.ptf.test_preprocessor( + input_mapping=input_files, + expected_mapping=expected_files + ) + + def test_strict_mysql(self): + """mysql test strict mode""" + self.ptf.options = { + 'dbms': 'mysql', + 'host': 'invalid-host-name', + 'dbname': 'testdb', + 'user': 'testuser', + 'password': 'testpassword', + 'port': 3306, + 'strict': True + } + + input_files = { + 'index.md': '# Database Documentation\n\n' + } + + with patch('os._exit') as mock_exit: + result = self.ptf.test_preprocessor( + input_mapping=input_files, + expected_mapping=input_files + ) + mock_exit.assert_called_once_with(1) diff --git a/tests/test_pgsql_basic.py b/tests/test_pgsql_basic.py new file mode 100644 index 0000000..9560479 --- /dev/null +++ b/tests/test_pgsql_basic.py @@ -0,0 +1,77 @@ +from unittest import TestCase +from unittest.mock import patch +from foliant_test.preprocessor import PreprocessorTestFramework + +class TestDbdocPostgres(TestCase): + """Postgres tests""" + def setUp(self): + self.ptf = PreprocessorTestFramework('dbdoc') + self.ptf.options = {} + + def test_simple_documentation_pgsql(self): + """pgsql test""" + self.ptf.options = { + 'dbms': 'pgsql', + 'host': 'testdb', + 'dbname': 'testdb', + 'user': 'postgres', + 'password': 'password', + 'port': 5432, + 'doc': True, + 'scheme': False, + 'filters':{ + 'eq': + {'table_name':'users'}, + }, + 'components':[ + 'tables' + ] + } + + input_files = { + 'index.md': '# Database Documentation\n\n' + } + expected_files = { + 'index.md': '''# Database Documentation\n\n +# Tables + + +## users + + + +column | nullable | type | descr | fkey +------ | -------- | ---- | ----- | ---- +id | NO | integer | | +name | YES | character varying | | +email | YES | character varying | | + +''' + } + + self.ptf.test_preprocessor( + input_mapping=input_files, + expected_mapping=expected_files + ) + def test_strict_pgsql(self): + """pgsql test strict mode""" + self.ptf.options = { + 'dbms': 'pgsql', + 'host': 'invalid-host-name', + 'dbname': 'testdb', + 'user': 'postgres', + 'password': 'password', + 'port': 5432, + 'strict': True + } + + input_files = { + 'index.md': '# Database Documentation\n\n' + } + + with patch('os._exit') as mock_exit: + result = self.ptf.test_preprocessor( + input_mapping=input_files, + expected_mapping=input_files + ) + mock_exit.assert_called_once_with(1)