From 941cebfa1e545429f8dcfb6e86427759242e5f56 Mon Sep 17 00:00:00 2001 From: holamgadol Date: Tue, 17 Jan 2023 16:47:32 +0700 Subject: [PATCH 1/9] fix: all_tab_columns instead user_tab_columns in oracle column query --- foliant/preprocessors/dbdoc/oracle/queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foliant/preprocessors/dbdoc/oracle/queries.py b/foliant/preprocessors/dbdoc/oracle/queries.py index 6845518..cbffcc8 100644 --- a/foliant/preprocessors/dbdoc/oracle/queries.py +++ b/foliant/preprocessors/dbdoc/oracle/queries.py @@ -129,7 +129,7 @@ class ColumnsQuery(QueryBase): col.DATA_LENGTH, col.DATA_PRECISION, com.COMMENTS - FROM user_tab_columns col + FROM all_tab_columns col JOIN all_tables tab ON col.TABLE_NAME = tab.TABLE_NAME LEFT JOIN user_col_comments com From a5fd492bea88a0c4afc8e7fa336738d5ecfb955c Mon Sep 17 00:00:00 2001 From: Timur Osmanov Date: Fri, 1 Aug 2025 11:53:12 +0300 Subject: [PATCH 2/9] add: strict option --- foliant/preprocessors/dbdoc/mssql/main.py | 47 ++++++++++++++-------- foliant/preprocessors/dbdoc/mysql/main.py | 38 +++++++++++------ foliant/preprocessors/dbdoc/oracle/main.py | 29 ++++++++----- 3 files changed, 75 insertions(+), 39 deletions(-) diff --git a/foliant/preprocessors/dbdoc/mssql/main.py b/foliant/preprocessors/dbdoc/mssql/main.py index 31c6f59..7ea1f3d 100644 --- a/foliant/preprocessors/dbdoc/mssql/main.py +++ b/foliant/preprocessors/dbdoc/mssql/main.py @@ -1,3 +1,4 @@ +import os from copy import deepcopy from logging import getLogger @@ -9,6 +10,7 @@ from .queries import TriggersQuery from .queries import ViewsQuery from foliant.preprocessors.dbdoc.base.main import DBRendererBase +from foliant.utils import output logger = getLogger('unbound.dbdoc.mssql') @@ -29,7 +31,8 @@ class MSSQLRenderer(DBRendererBase): 'functions', 'triggers', 'views' - ] + ], + 'strict': False } module_name = __name__ @@ -46,24 +49,34 @@ def connect(self): 'and make sure that MS SQL Server is installed on the machine' ) - if self.options['trusted_connection']: - connection_string = ( - f"DRIVER={self.options['driver']};" - f"SERVER={self.options['host']},{self.options['port']};" - f"DATABASE={self.options['dbname']};Trusted_Connection=yes" - ) + try: + if self.options['trusted_connection']: + connection_string = ( + f"DRIVER={self.options['driver']};" + f"SERVER={self.options['host']},{self.options['port']};" + f"DATABASE={self.options['dbname']};Trusted_Connection=yes" + ) - else: - connection_string = ( - f"DRIVER={self.options['driver']};" - f"SERVER={self.options['host']},{self.options['port']};" - f"DATABASE={self.options['dbname']};" - f"UID={self.options['user']};PWD={self.options['password']}" + else: + connection_string = ( + f"DRIVER={self.options['driver']};" + f"SERVER={self.options['host']},{self.options['port']};" + f"DATABASE={self.options['dbname']};" + f"UID={self.options['user']};PWD={self.options['password']}" + ) + logger.debug( + f"Trying to connect: {connection_string}" ) - logger.debug( - f"Trying to connect: {connection_string}" - ) - self.con = pyodbc.connect(connection_string) + self.con = pyodbc.connect(connection_string) + logger.debug("Successfully connected to the database") + except pyodbc.Error as e: + msg = f"\MS SQL database database connection error: {e}" + if self.options['strict']: + logger.error(msg) + output(f'ERROR: {msg}') + os._exit(1) + else: + logger.debug(f'{msg}. Skipping.') def collect_datasets(self) -> dict: diff --git a/foliant/preprocessors/dbdoc/mysql/main.py b/foliant/preprocessors/dbdoc/mysql/main.py index cfc7b5b..13f381f 100644 --- a/foliant/preprocessors/dbdoc/mysql/main.py +++ b/foliant/preprocessors/dbdoc/mysql/main.py @@ -1,3 +1,4 @@ +import os from copy import deepcopy from logging import getLogger @@ -9,6 +10,7 @@ from .queries import TriggersQuery from .queries import ViewsQuery from foliant.preprocessors.dbdoc.base.main import DBRendererBase +from foliant.utils import output logger = getLogger('unbound.dbdoc.mysql') @@ -27,7 +29,8 @@ class MySQLRenderer(DBRendererBase): 'functions', 'triggers', 'views' - ] + ], + 'strict': False } module_name = __name__ @@ -45,18 +48,27 @@ def connect(self): 'and make sure that MySQL Client is installed on the machine' ) - logger.debug( - f"Trying to connect: host={self.options['host']} port={self.options['port']}" - f" dbname={self.options['dbname']}, user={self.options['user']} " - f"password={self.options['password']}." - ) - self.con = _mysql.connect( - host=self.options['host'], - port=self.options['port'], - user=self.options['user'], - passwd=self.options['password'], - db=self.options['dbname'] - ) + try: + logger.debug( + f"Trying to connect: host={self.options['host']} port={self.options['port']}" + f" dbname={self.options['dbname']}, user={self.options['user']} " + f"password={self.options['password']}." + ) + self.con = _mysql.connect( + host=self.options['host'], + port=self.options['port'], + user=self.options['user'], + passwd=self.options['password'], + db=self.options['dbname'] + ) + except _mysql.Error as e: + msg = f"\nMySQL database error: {e}" + if self.options['strict']: + logger.error(msg) + output(f'ERROR: {msg}') + os._exit(1) + else: + logger.debug(f'{msg}. Skipping.') def collect_datasets(self) -> dict: diff --git a/foliant/preprocessors/dbdoc/oracle/main.py b/foliant/preprocessors/dbdoc/oracle/main.py index f90cfc3..00f7667 100644 --- a/foliant/preprocessors/dbdoc/oracle/main.py +++ b/foliant/preprocessors/dbdoc/oracle/main.py @@ -1,3 +1,4 @@ +import os from copy import deepcopy from logging import getLogger @@ -9,7 +10,7 @@ from .queries import ViewsQuery from ..base.main import LibraryNotInstalledError from foliant.preprocessors.dbdoc.base.main import DBRendererBase - +from foliant.utils import output logger = getLogger('unbound.dbdoc.oracle') @@ -28,7 +29,8 @@ class OracleRenderer(DBRendererBase): 'functions', 'triggers', 'views' - ] + ], + 'strict': False } module_name = __name__ @@ -50,13 +52,22 @@ def connect(self): f" dbname={self.options['dbname']}, user={self.options['user']} " f"password={self.options['password']}." ) - self.con = cx_Oracle.connect( - f"{self.options['user']}/{self.options['password']}@" - f"{self.options['host']}:{self.options['port']}/" - f"{self.options['dbname']}", - encoding='UTF-8', - nencoding='UTF-8' - ) + try: + self.con = cx_Oracle.connect( + f"{self.options['user']}/{self.options['password']}@" + f"{self.options['host']}:{self.options['port']}/" + f"{self.options['dbname']}", + encoding='UTF-8', + nencoding='UTF-8' + ) + except cx_Oracle.Error as e: + msg = f"\nOracle database connection error: {e}" + if self.options['strict']: + logger.error(msg) + output(f"ERROR: {msg}") + os._exit(1) + else: + logger.debug(f"{msg}. Skipping.") def collect_datasets(self) -> dict: From dbe2f51689b8c6608829f3d665fd5027861ba76e Mon Sep 17 00:00:00 2001 From: Timur Osmanov Date: Fri, 8 Aug 2025 10:26:30 +0300 Subject: [PATCH 3/9] update: default options --- foliant/preprocessors/dbdoc/base/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/foliant/preprocessors/dbdoc/base/main.py b/foliant/preprocessors/dbdoc/base/main.py index 80cb8c6..db93c03 100644 --- a/foliant/preprocessors/dbdoc/base/main.py +++ b/foliant/preprocessors/dbdoc/base/main.py @@ -27,7 +27,8 @@ def process(self, tag_options) -> str: self.options = CombinedOptions( { 'config': self.config, - 'tag': tag_options + 'tag': tag_options, + 'strict': True }, priority='tag', defaults=self.defaults From 4b7a24ceec518df4d589e4b5c2c35e5de3241fa4 Mon Sep 17 00:00:00 2001 From: Timur Osmanov Date: Fri, 8 Aug 2025 16:24:33 +0300 Subject: [PATCH 4/9] update: README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3197c29..bb264ac 100755 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ preprocessors: password: !env DBDOC_PASS doc: True scheme: True + strict: False filters: ... doc_template: dbdoc.j2 @@ -141,6 +142,9 @@ preprocessors: `scheme` : If `true` — the platuml code for database scheme will be generated. Default: `true` +`strict` +: If `true` — the build will fail if connection to database cannot be established. If `false` — the preprocessor will skip the tag with warning. Default: `false` + `filters` : SQL-like operators for filtering the results. More info in the **Filters** section. From 3bbc1a679bdde992ca5f4a15c36371212d7fb320 Mon Sep 17 00:00:00 2001 From: Timur Osmanov Date: Fri, 8 Aug 2025 16:35:52 +0300 Subject: [PATCH 5/9] update: README.md added trusted_connection description --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index bb264ac..ecbe977 100755 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ preprocessors: doc: True scheme: True strict: False + trusted_connection: False filters: ... doc_template: dbdoc.j2 @@ -145,6 +146,9 @@ preprocessors: `strict` : If `true` — the build will fail if connection to database cannot be established. If `false` — the preprocessor will skip the tag with warning. Default: `false` +`trusted_connection` +: Specific option for MS SQL Server. If true - will use Windows Authentication (Trusted Connection) instead of username/password. Default: false. Requires proper ODBC driver configuration. + `filters` : SQL-like operators for filtering the results. More info in the **Filters** section. @@ -383,3 +387,28 @@ con = pyodbc.connect( "UID=Usernam;PWD=Password_0" ) ``` + +**Microsoft SQL Server Authentication Issues** + +When using MS SQL Server, you have two authentication options: + +1. SQL Server Authentication (username/password): + + ```yaml + trusted_connection: false + user: your_username + password: your_password + ``` + +2. Windows Authentication (Trusted Connection): + + ```yaml + trusted_connection: true + # no user/password needed + ``` + +For Windows Authentication to work: + +- Make sure your ODBC driver supports Trusted Connections. +- The account running Foliant must have proper database permissions. +- On Linux/Mac, you may need to configure Kerberos for cross-platform authentication. From 457398a077ff696e9ec99d869c347d3278d0c25f Mon Sep 17 00:00:00 2001 From: Timur Osmanov Date: Tue, 26 Aug 2025 12:34:31 +0300 Subject: [PATCH 6/9] add: tests --- .github/workflows/test.yml | 33 +++++++++++ foliant/preprocessors/dbdoc/base/main.py | 3 +- test.sh | 11 ++++ test_in_docker.sh | 75 ++++++++++++++++++++++++ tests/__init__.py | 0 tests/db/mysql/data/01-init.sql | 19 ++++++ tests/db/mysql/init_db.sh | 14 +++++ tests/db/psql/data/01-init.sql | 19 ++++++ tests/db/psql/init_db.sh | 12 ++++ tests/test_dbdoc_basic.py | 58 ++++++++++++++++++ 10 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100755 test.sh create mode 100755 test_in_docker.sh create mode 100644 tests/__init__.py create mode 100644 tests/db/mysql/data/01-init.sql create mode 100644 tests/db/mysql/init_db.sh create mode 100644 tests/db/psql/data/01-init.sql create mode 100755 tests/db/psql/init_db.sh create mode 100644 tests/test_dbdoc_basic.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..71b21de --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,33 @@ +name: Tests + +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9'] + db-type: ['psql'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Make scripts executable + run: | + chmod +x test_in_docker.sh + chmod +x test.sh + if [ -f "tests/db/${{ matrix.db-type }}/init_db.sh" ]; then + chmod +x "tests/db/${{ matrix.db-type }}/init_db.sh" + fi + + - name: Run tests in Docker + run: | + ./test_in_docker.sh \ + --python-version "${{ matrix.python-version }}" \ + --db-type "${{ matrix.db-type }}" \ No newline at end of file diff --git a/foliant/preprocessors/dbdoc/base/main.py b/foliant/preprocessors/dbdoc/base/main.py index db93c03..80cb8c6 100644 --- a/foliant/preprocessors/dbdoc/base/main.py +++ b/foliant/preprocessors/dbdoc/base/main.py @@ -27,8 +27,7 @@ def process(self, tag_options) -> str: self.options = CombinedOptions( { 'config': self.config, - 'tag': tag_options, - 'strict': True + 'tag': tag_options }, priority='tag', defaults=self.defaults diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..d11dd44 --- /dev/null +++ b/test.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# before testing make sure that you have installed the fresh version of preprocessor: +pip3 install . +# also make sure that fresh version of test framework is installed: +pip3 install --upgrade foliantcontrib.test_framework + +# install dependencies +pip3 install sqlalchemy + +python3 -m unittest discover -v diff --git a/test_in_docker.sh b/test_in_docker.sh new file mode 100755 index 0000000..43c1d03 --- /dev/null +++ b/test_in_docker.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +set -e # Exit on error + +# Default values +PYTHON_VERSIONS=("3.8" "3.9") +DB_TYPE=("psql") + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --python-version) + PYTHON_VERSIONS=("$2") + shift 2 + ;; + --db-type) + DB_TYPE=("$2") + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +echo "=== Test Configuration ===" +echo "Python versions: ${PYTHON_VERSIONS[*]}" +echo "Database type: ${DB_TYPE[*]}" +echo "==========================" + +for version in "${PYTHON_VERSIONS[@]}"; do + for db_type in "${DB_TYPE[@]}"; do + echo "=== Testing with Python ${version} ===" + + # Write Dockerfile + cat > Dockerfile << EOF +FROM python:${version}-bullseye + +# Install required Python packages +RUN pip install foliantcontrib.utils requests psycopg2-binary pyodbc + +# Create working directory +WORKDIR /app + +EOF + + echo "Building test image..." + docker build . -t test-foliant:${version} --no-cache + + container_name="testdb" + network_name="testnetwork" + + docker network create ${network_name} + + ./tests/db/${db_type}/init_db.sh ${container_name} ${network_name} || exit 1 + + echo "Running tests with Docker access..." + docker run --rm \ + --network ${network_name}\ + -v "./:/app/" \ + -w /app \ + test-foliant:${version} \ + "./test.sh" + + echo "Cleaning up..." + rm -f Dockerfile + docker rm -f ${container_name} 2>/dev/null || true + docker rmi test-foliant:${version}-${DB_TYPE} 2>/dev/null || true + docker network rm ${network_name} 2>/dev/null || true + + echo "=== Completed Python ${version} with ${DB_TYPE} tests ===" + echo "" + done +done \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/db/mysql/data/01-init.sql b/tests/db/mysql/data/01-init.sql new file mode 100644 index 0000000..79e4941 --- /dev/null +++ b/tests/db/mysql/data/01-init.sql @@ -0,0 +1,19 @@ +CREATE TABLE users ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100), + email VARCHAR(100) UNIQUE +); + +INSERT INTO users (name, email) VALUES +('john_doe', 'john@example.com'), +('jane_smith', 'jane@example.com'); + +CREATE TABLE products ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100), + price DECIMAL(10,2) +); + +INSERT INTO products (name, price) VALUES +('apple', 50000.00), +('banana', 1500.50); diff --git a/tests/db/mysql/init_db.sh b/tests/db/mysql/init_db.sh new file mode 100644 index 0000000..62d605b --- /dev/null +++ b/tests/db/mysql/init_db.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +container=${1:-testdb-mysql} +network=${2:-testnetwork} + +docker run -d --name ${container} \ + --network ${network} \ + -e MYSQL_ROOT_PASSWORD=password \ + -e MYSQL_DATABASE=testdb \ + -v $(pwd)/tests/db/mysql/data:/docker-entrypoint-initdb.d \ + -p 3306:3306 \ + mysql:8 + +rm -rf init-scripts diff --git a/tests/db/psql/data/01-init.sql b/tests/db/psql/data/01-init.sql new file mode 100644 index 0000000..2a40dbf --- /dev/null +++ b/tests/db/psql/data/01-init.sql @@ -0,0 +1,19 @@ +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + email VARCHAR(100) UNIQUE +); + +INSERT INTO users (name, email) VALUES +('john_doe', 'john@example.com'), +('jane_smith','jane@example.com'); + +CREATE TABLE products ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + price DECIMAL(10,2) +); + +INSERT INTO products (name, price) VALUES +('apple', 50000.00), +('banana', 1500.50); diff --git a/tests/db/psql/init_db.sh b/tests/db/psql/init_db.sh new file mode 100755 index 0000000..af2d163 --- /dev/null +++ b/tests/db/psql/init_db.sh @@ -0,0 +1,12 @@ +container=${1:-testdb} +network=${2:-testnetwork} + +docker run -d --name ${container} \ + --network ${network} \ + -e POSTGRES_PASSWORD=password \ + -e POSTGRES_DB=testdb \ + -v $(pwd)/tests/db/psql/data:/docker-entrypoint-initdb.d \ + -p 5432:5432 \ + postgres:15 + +rm -rf init-scripts diff --git a/tests/test_dbdoc_basic.py b/tests/test_dbdoc_basic.py new file mode 100644 index 0000000..2ca5979 --- /dev/null +++ b/tests/test_dbdoc_basic.py @@ -0,0 +1,58 @@ +# tests/test_dbdoc_basic.py +from unittest import TestCase +from foliant_test.preprocessor import PreprocessorTestFramework + +class TestDbdocBasic(TestCase): + """Basic tests""" + def setUp(self): + self.ptf = PreprocessorTestFramework('dbdoc') + self.ptf.options = {} + + def test_simple_documentation(self): + """pgsql test""" + self.ptf.options = { + 'dbms': 'pgsql', + 'host': 'testdb', + 'dbname': 'testdb', + 'user': 'postgres', + 'password': 'password', + 'port': 5432, + 'doc': True, + 'scheme': False, + 'filters':{ + 'eq': + {'table_name':'users'}, + }, + 'components':[ + 'tables' + ] + } + # Входные файлы + input_files = { + 'index.md': '# Database Documentation\n\n' + } + + # Ожидаемые файлы с функцией проверки + expected_files = { + 'index.md': '''# Database Documentation\n\n +# Tables + + +## users + + + +column | nullable | type | descr | fkey +------ | -------- | ---- | ----- | ---- +id | NO | integer | | +name | YES | character varying | | +email | YES | character varying | | + +''' + } + + # Запускаем тест + self.ptf.test_preprocessor( + input_mapping=input_files, + expected_mapping=expected_files + ) From 323d1f9370d563bb33856c30240a29d15c4dd0f4 Mon Sep 17 00:00:00 2001 From: Timur Osmanov Date: Tue, 26 Aug 2025 19:42:11 +0300 Subject: [PATCH 7/9] update: tests --- .github/workflows/test.yml | 2 +- README.md | 41 +++++++++ foliant/preprocessors/dbdoc/mssql/main.py | 2 +- foliant/preprocessors/dbdoc/mysql/main.py | 7 +- foliant/preprocessors/dbdoc/oracle/main.py | 2 +- test.sh | 6 +- test_in_docker.sh | 36 ++++---- tests/db/mysql/init_db.sh | 11 ++- tests/db/psql/init_db.sh | 6 +- tests/test_mysql_basic.py | 84 +++++++++++++++++++ ...test_dbdoc_basic.py => test_psql_basic.py} | 34 ++++++-- 11 files changed, 192 insertions(+), 39 deletions(-) mode change 100644 => 100755 tests/db/mysql/init_db.sh create mode 100644 tests/test_mysql_basic.py rename tests/{test_dbdoc_basic.py => test_psql_basic.py} (61%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71b21de..5b85c7c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: python-version: ['3.8', '3.9'] - db-type: ['psql'] + db-type: ['psql', 'mysql'] steps: - name: Checkout code diff --git a/README.md b/README.md index ecbe977..15c5bb4 100755 --- a/README.md +++ b/README.md @@ -317,6 +317,47 @@ If you wish to create your own template, the default ones may be a good starting * [Default **SQL Server doc** template.](https://github.com/foliant-docs/foliantcontrib.dbdoc/blob/master/foliant/preprocessors/dbdoc/mssql/templates/doc.j2) * [Default **SQL Server scheme** template.](https://github.com/foliant-docs/foliantcontrib.dbdoc/blob/master/foliant/preprocessors/dbdoc/mssql/templates/doc.j2) +## Tests + +For run tests, use: +```bash +./test_in_docker.sh --python-version "3.9" --db-type "mysql" +``` + +**Options:** +`--python-version ` – Specifies Python version for test environment. Available_: 3.8, 3.9, 3.10 etc. + +`--db-type ` – Chooses database type for testing. Available_: mysql, psql. + +**Usage Examples** + +```bash +# Basic usage with defaults +./test_in_docker.sh + +# Specific Python and database +./test_in_docker.sh --python-version "3.10" --db-type "psql" + +# Only change database type +./test_in_docker.sh --db-type "mysql" + +# Only change Python version +./test_in_docker.sh --python-version "3.9" +``` + +**What It Does:** +1. Starts Docker container with specified Python version. +2. Initializes chosen database type with test data. +3. Runs test suite. +4. Cleans up resources after completion. +5. Returns exit code based on test results. + +**Notes** +- Requires Docker installed; +- Test data is automatically loaded from `test_data/` directory; +- Results are displayed in console with color formatting; +- Exit code 0 = success, 1 = test failures. + ## Troubleshooting If you get errors during build, especially errors concerning connection to the database, you have to make sure that you are supplying the right parameters. diff --git a/foliant/preprocessors/dbdoc/mssql/main.py b/foliant/preprocessors/dbdoc/mssql/main.py index 7ea1f3d..5365d7d 100644 --- a/foliant/preprocessors/dbdoc/mssql/main.py +++ b/foliant/preprocessors/dbdoc/mssql/main.py @@ -73,7 +73,7 @@ def connect(self): msg = f"\MS SQL database database connection error: {e}" if self.options['strict']: logger.error(msg) - output(f'ERROR: {msg}') + output(f'ERROR: {msg}. Exit.') os._exit(1) else: logger.debug(f'{msg}. Skipping.') diff --git a/foliant/preprocessors/dbdoc/mysql/main.py b/foliant/preprocessors/dbdoc/mysql/main.py index 13f381f..1ba9343 100644 --- a/foliant/preprocessors/dbdoc/mysql/main.py +++ b/foliant/preprocessors/dbdoc/mysql/main.py @@ -58,14 +58,14 @@ def connect(self): host=self.options['host'], port=self.options['port'], user=self.options['user'], - passwd=self.options['password'], - db=self.options['dbname'] + password=self.options['password'], + database=self.options['dbname'] ) except _mysql.Error as e: msg = f"\nMySQL database error: {e}" if self.options['strict']: logger.error(msg) - output(f'ERROR: {msg}') + output(f'ERROR: {msg}. Exit.') os._exit(1) else: logger.debug(f'{msg}. Skipping.') @@ -106,7 +106,6 @@ def collect_datasets(self) -> dict: q_triggers = TriggersQuery(self.con, filters) logger.debug(f'Triggers query:\n\n {q_triggers.sql}') result['triggers'] = q_triggers.run() - return result def collect_tables(self, diff --git a/foliant/preprocessors/dbdoc/oracle/main.py b/foliant/preprocessors/dbdoc/oracle/main.py index 00f7667..c321480 100644 --- a/foliant/preprocessors/dbdoc/oracle/main.py +++ b/foliant/preprocessors/dbdoc/oracle/main.py @@ -64,7 +64,7 @@ def connect(self): msg = f"\nOracle database connection error: {e}" if self.options['strict']: logger.error(msg) - output(f"ERROR: {msg}") + output(f'ERROR: {msg}. Exit.') os._exit(1) else: logger.debug(f"{msg}. Skipping.") diff --git a/test.sh b/test.sh index d11dd44..874432a 100755 --- a/test.sh +++ b/test.sh @@ -1,11 +1,13 @@ #!/bin/bash +db_type=${1-psql} + # before testing make sure that you have installed the fresh version of preprocessor: pip3 install . # also make sure that fresh version of test framework is installed: pip3 install --upgrade foliantcontrib.test_framework # install dependencies -pip3 install sqlalchemy +pip3 install sqlalchemy mysqlclient -python3 -m unittest discover -v +python3 -m unittest discover -v -p "*${db_type}*" diff --git a/test_in_docker.sh b/test_in_docker.sh index 43c1d03..2a05778 100755 --- a/test_in_docker.sh +++ b/test_in_docker.sh @@ -4,7 +4,10 @@ set -e # Exit on error # Default values PYTHON_VERSIONS=("3.8" "3.9") -DB_TYPE=("psql") +DB_TYPE=("psql" "mysql") + +CONTAINER_NAME="testdb" +NETWORK_NAME="testnetwork" # Parse command line arguments while [[ $# -gt 0 ]]; do @@ -30,46 +33,43 @@ echo "Database type: ${DB_TYPE[*]}" echo "==========================" for version in "${PYTHON_VERSIONS[@]}"; do - for db_type in "${DB_TYPE[@]}"; do - echo "=== Testing with Python ${version} ===" + for type in "${DB_TYPE[@]}"; do + echo "=== Testing with Python ${version} with DB ${type} ===" - # Write Dockerfile cat > Dockerfile << EOF FROM python:${version}-bullseye # Install required Python packages RUN pip install foliantcontrib.utils requests psycopg2-binary pyodbc -# Create working directory WORKDIR /app - +ENTRYPOINT ["./test.sh"] EOF - echo "Building test image..." docker build . -t test-foliant:${version} --no-cache - container_name="testdb" - network_name="testnetwork" - - docker network create ${network_name} + if ! docker network inspect "${NETWORK_NAME}" >/dev/null 2>&1; then + docker network create "${NETWORK_NAME}" + fi - ./tests/db/${db_type}/init_db.sh ${container_name} ${network_name} || exit 1 + ./tests/db/${type}/init_db.sh ${CONTAINER_NAME} ${NETWORK_NAME} || exit 1 echo "Running tests with Docker access..." + docker rm -f test-foliant:${version} 2>/dev/null || true docker run --rm \ - --network ${network_name}\ + --network ${NETWORK_NAME}\ -v "./:/app/" \ -w /app \ test-foliant:${version} \ - "./test.sh" + "${type}" echo "Cleaning up..." rm -f Dockerfile - docker rm -f ${container_name} 2>/dev/null || true - docker rmi test-foliant:${version}-${DB_TYPE} 2>/dev/null || true - docker network rm ${network_name} 2>/dev/null || true + docker rm -f ${CONTAINER_NAME} 2>/dev/null || true + docker rmi test-foliant:${version} 2>/dev/null || true + docker network rm ${NETWORK_NAME} 2>/dev/null || true - echo "=== Completed Python ${version} with ${DB_TYPE} tests ===" + echo "=== Completed Python ${version} with ${type} tests ===" echo "" done done \ No newline at end of file diff --git a/tests/db/mysql/init_db.sh b/tests/db/mysql/init_db.sh old mode 100644 new mode 100755 index 62d605b..1760ede --- a/tests/db/mysql/init_db.sh +++ b/tests/db/mysql/init_db.sh @@ -1,14 +1,17 @@ #!/bin/bash -container=${1:-testdb-mysql} +set -e + +container=${1:-testdb} network=${2:-testnetwork} +docker rm -f ${container} docker run -d --name ${container} \ --network ${network} \ - -e MYSQL_ROOT_PASSWORD=password \ + -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ + -e MYSQL_USER=testuser \ + -e MYSQL_PASSWORD=testpassword \ -e MYSQL_DATABASE=testdb \ -v $(pwd)/tests/db/mysql/data:/docker-entrypoint-initdb.d \ -p 3306:3306 \ mysql:8 - -rm -rf init-scripts diff --git a/tests/db/psql/init_db.sh b/tests/db/psql/init_db.sh index af2d163..56c9e94 100755 --- a/tests/db/psql/init_db.sh +++ b/tests/db/psql/init_db.sh @@ -1,6 +1,11 @@ +#!/bin/bash + +set -e + container=${1:-testdb} network=${2:-testnetwork} +docker rm -f ${container} docker run -d --name ${container} \ --network ${network} \ -e POSTGRES_PASSWORD=password \ @@ -9,4 +14,3 @@ docker run -d --name ${container} \ -p 5432:5432 \ postgres:15 -rm -rf init-scripts diff --git a/tests/test_mysql_basic.py b/tests/test_mysql_basic.py new file mode 100644 index 0000000..a1902b3 --- /dev/null +++ b/tests/test_mysql_basic.py @@ -0,0 +1,84 @@ +from unittest import TestCase +from unittest.mock import patch +from foliant_test.preprocessor import PreprocessorTestFramework + +class TestDbdocMySQL(TestCase): + """MySQL tests""" + def setUp(self): + self.ptf = PreprocessorTestFramework('dbdoc') + self.ptf.options = {} + + def test_simple_documentation_mysql(self): + """mysql test""" + self.ptf.options = { + 'dbms': 'mysql', + 'host': 'testdb', + 'dbname': 'testdb', + 'user': 'testuser', + 'password': 'testpassword', + 'port': 3306, + 'doc': True, + 'scheme': False, + 'filters':{ + 'eq': {'table_name':'users'}, + }, + 'components':['tables'] + } + + input_files = { + 'index.md': '# Database Documentation\n\n' + } + expected_files = { + 'index.md': '''# Database Documentation\n\n + + +# Tables + + +## users + + + +column | nullable | type | descr | fkey +------ | -------- | ---- | ----- | ---- +id | NO | int | | +name | YES | varchar | | +email | YES | varchar | | + + + + + + + + +''' + } + + self.ptf.test_preprocessor( + input_mapping=input_files, + expected_mapping=expected_files + ) + + def test_strict_mysql(self): + """mysql test strict mode""" + self.ptf.options = { + 'dbms': 'mysql', + 'host': 'invalid-host-name', + 'dbname': 'testdb', + 'user': 'testuser', + 'password': 'testpassword', + 'port': 3306, + 'strict': True + } + + input_files = { + 'index.md': '# Database Documentation\n\n' + } + + with patch('os._exit') as mock_exit: + result = self.ptf.test_preprocessor( + input_mapping=input_files, + expected_mapping=input_files + ) + mock_exit.assert_called_once_with(1) diff --git a/tests/test_dbdoc_basic.py b/tests/test_psql_basic.py similarity index 61% rename from tests/test_dbdoc_basic.py rename to tests/test_psql_basic.py index 2ca5979..420302d 100644 --- a/tests/test_dbdoc_basic.py +++ b/tests/test_psql_basic.py @@ -1,9 +1,9 @@ -# tests/test_dbdoc_basic.py from unittest import TestCase +from unittest.mock import patch from foliant_test.preprocessor import PreprocessorTestFramework -class TestDbdocBasic(TestCase): - """Basic tests""" +class TestDbdocPostgres(TestCase): + """Postgres tests""" def setUp(self): self.ptf = PreprocessorTestFramework('dbdoc') self.ptf.options = {} @@ -27,12 +27,10 @@ def test_simple_documentation(self): 'tables' ] } - # Входные файлы + input_files = { 'index.md': '# Database Documentation\n\n' } - - # Ожидаемые файлы с функцией проверки expected_files = { 'index.md': '''# Database Documentation\n\n # Tables @@ -51,8 +49,30 @@ def test_simple_documentation(self): ''' } - # Запускаем тест self.ptf.test_preprocessor( input_mapping=input_files, expected_mapping=expected_files ) + def test_strict_mysql(self): + """mysql test strict mode""" + self.ptf.options = { + 'dbms': 'pgsql', + 'host': 'invalid-host-name', + 'host': 'testdb', + 'dbname': 'testdb', + 'user': 'postgres', + 'password': 'password', + 'port': 5432, + 'strict': True + } + + input_files = { + 'index.md': '# Database Documentation\n\n' + } + + with patch('os._exit') as mock_exit: + result = self.ptf.test_preprocessor( + input_mapping=input_files, + expected_mapping=input_files + ) + mock_exit.assert_called_once_with(1) From a2a4395b02d3e6e2c49c651588da583207f5f8ac Mon Sep 17 00:00:00 2001 From: Timur Osmanov Date: Wed, 17 Sep 2025 16:42:31 +0300 Subject: [PATCH 8/9] update: tests and pgsql --- changelog.md | 4 +++ foliant/preprocessors/dbdoc/mssql/main.py | 1 + foliant/preprocessors/dbdoc/mysql/main.py | 1 + foliant/preprocessors/dbdoc/oracle/main.py | 1 + foliant/preprocessors/dbdoc/pgsql/main.py | 29 ++++++++++++++++------ setup.py | 2 +- tests/test_psql_basic.py | 8 +++--- 7 files changed, 33 insertions(+), 13 deletions(-) diff --git a/changelog.md b/changelog.md index 5f8e7f8..4744c2a 100755 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,7 @@ +# 0.1.9 +- Add: strict option. +- Add: tests. + # 0.1.8 - DBMS python connectors are only imported on use. diff --git a/foliant/preprocessors/dbdoc/mssql/main.py b/foliant/preprocessors/dbdoc/mssql/main.py index 5365d7d..7b2c06a 100644 --- a/foliant/preprocessors/dbdoc/mssql/main.py +++ b/foliant/preprocessors/dbdoc/mssql/main.py @@ -12,6 +12,7 @@ from foliant.preprocessors.dbdoc.base.main import DBRendererBase from foliant.utils import output + logger = getLogger('unbound.dbdoc.mssql') diff --git a/foliant/preprocessors/dbdoc/mysql/main.py b/foliant/preprocessors/dbdoc/mysql/main.py index 1ba9343..3528f8e 100644 --- a/foliant/preprocessors/dbdoc/mysql/main.py +++ b/foliant/preprocessors/dbdoc/mysql/main.py @@ -12,6 +12,7 @@ from foliant.preprocessors.dbdoc.base.main import DBRendererBase from foliant.utils import output + logger = getLogger('unbound.dbdoc.mysql') diff --git a/foliant/preprocessors/dbdoc/oracle/main.py b/foliant/preprocessors/dbdoc/oracle/main.py index c321480..c2e1d18 100644 --- a/foliant/preprocessors/dbdoc/oracle/main.py +++ b/foliant/preprocessors/dbdoc/oracle/main.py @@ -12,6 +12,7 @@ from foliant.preprocessors.dbdoc.base.main import DBRendererBase from foliant.utils import output + logger = getLogger('unbound.dbdoc.oracle') diff --git a/foliant/preprocessors/dbdoc/pgsql/main.py b/foliant/preprocessors/dbdoc/pgsql/main.py index 649601a..b2b192c 100644 --- a/foliant/preprocessors/dbdoc/pgsql/main.py +++ b/foliant/preprocessors/dbdoc/pgsql/main.py @@ -1,3 +1,4 @@ +import os from copy import deepcopy from logging import getLogger @@ -10,6 +11,7 @@ from .queries import TriggersQuery from .queries import ViewsQuery from foliant.preprocessors.dbdoc.base.main import DBRendererBase +from foliant.utils import output logger = getLogger('unbound.dbdoc.pgsql') @@ -29,7 +31,8 @@ class PGSQLRenderer(DBRendererBase): 'views', 'functions', 'triggers' - ] + ], + 'strict': False } module_name = __name__ @@ -50,13 +53,23 @@ def connect(self): f" dbname={self.options['dbname']}, user={self.options['user']} " f"password={self.options['password']}." ) - self.con = psycopg2.connect( - f"host='{self.options['host']}' " - f"port='{self.options['port']}' " - f"dbname='{self.options['dbname']}' " - f"user='{self.options['user']}'" - f"password='{self.options['password']}'" - ) + + try: + self.con = psycopg2.connect( + f"host='{self.options['host']}' " + f"port='{self.options['port']}' " + f"dbname='{self.options['dbname']}' " + f"user='{self.options['user']}'" + f"password='{self.options['password']}'" + ) + except psycopg2.Error as e: + msg = f"\nPostgreSQL database connection error: {e}" + if self.options['strict']: + logger.error(msg) + output(f'ERROR: {msg}. Exit.') + os._exit(1) + else: + logger.debug(f"{msg}. Skipping.") def collect_datasets(self) -> dict: diff --git a/setup.py b/setup.py index 78d8aee..60591e8 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ description=SHORT_DESCRIPTION, long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown', - version='0.1.8', + version='0.1.9', author='Daniil Minukhin', author_email='ddddsa@gmail.com', packages=find_namespace_packages(exclude=['*.test', 'foliant', '*.templates']), diff --git a/tests/test_psql_basic.py b/tests/test_psql_basic.py index 420302d..8c568d3 100644 --- a/tests/test_psql_basic.py +++ b/tests/test_psql_basic.py @@ -8,7 +8,7 @@ def setUp(self): self.ptf = PreprocessorTestFramework('dbdoc') self.ptf.options = {} - def test_simple_documentation(self): + def test_simple_documentation_pgsql(self): """pgsql test""" self.ptf.options = { 'dbms': 'pgsql', @@ -53,8 +53,8 @@ def test_simple_documentation(self): input_mapping=input_files, expected_mapping=expected_files ) - def test_strict_mysql(self): - """mysql test strict mode""" + def test_strict_pgsql(self): + """pgsql test strict mode""" self.ptf.options = { 'dbms': 'pgsql', 'host': 'invalid-host-name', @@ -67,7 +67,7 @@ def test_strict_mysql(self): } input_files = { - 'index.md': '# Database Documentation\n\n' + 'index.md': '# Database Documentation\n\n' } with patch('os._exit') as mock_exit: From a9b19822d9296fb5c26e47492f200728a10774b2 Mon Sep 17 00:00:00 2001 From: Timur Osmanov Date: Wed, 17 Sep 2025 18:35:37 +0300 Subject: [PATCH 9/9] update: tests --- .github/workflows/test.yml | 2 +- Dockerfile | 7 +++++++ README.md | 4 ++-- setup.py | 2 +- test.sh | 2 +- test_in_docker.sh | 4 ++-- tests/db/{psql => pgsql}/data/01-init.sql | 0 tests/db/{psql => pgsql}/init_db.sh | 2 +- tests/test_mysql_basic.py | 6 +++--- tests/{test_psql_basic.py => test_pgsql_basic.py} | 7 +++---- 10 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 Dockerfile rename tests/db/{psql => pgsql}/data/01-init.sql (100%) rename tests/db/{psql => pgsql}/init_db.sh (80%) rename tests/{test_psql_basic.py => test_pgsql_basic.py} (94%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b85c7c..26ef8e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: python-version: ['3.8', '3.9'] - db-type: ['psql', 'mysql'] + db-type: ['pgsql', 'mysql'] steps: - name: Checkout code diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dec36e4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.8-bullseye + +# Install required Python packages +RUN pip install foliantcontrib.utils requests psycopg2-binary pyodbc + +WORKDIR /app +ENTRYPOINT ["./test.sh"] diff --git a/README.md b/README.md index 15c5bb4..a1c9d1d 100755 --- a/README.md +++ b/README.md @@ -327,7 +327,7 @@ For run tests, use: **Options:** `--python-version ` – Specifies Python version for test environment. Available_: 3.8, 3.9, 3.10 etc. -`--db-type ` – Chooses database type for testing. Available_: mysql, psql. +`--db-type ` – Chooses database type for testing. Available_: mysql, pgsql. **Usage Examples** @@ -336,7 +336,7 @@ For run tests, use: ./test_in_docker.sh # Specific Python and database -./test_in_docker.sh --python-version "3.10" --db-type "psql" +./test_in_docker.sh --python-version "3.10" --db-type "pgsql" # Only change database type ./test_in_docker.sh --db-type "mysql" diff --git a/setup.py b/setup.py index 60591e8..c116fb3 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ platforms='any', install_requires=[ 'foliant>=1.0.5', - 'foliantcontrib.utils>=1.0.2', + 'foliantcontrib.utils==1.0.3', 'foliantcontrib.plantuml>=1.0.10', 'jinja2', 'PyYAML' diff --git a/test.sh b/test.sh index 874432a..e4a11d5 100755 --- a/test.sh +++ b/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -db_type=${1-psql} +db_type=${1-pgsql} # before testing make sure that you have installed the fresh version of preprocessor: pip3 install . diff --git a/test_in_docker.sh b/test_in_docker.sh index 2a05778..133d1a9 100755 --- a/test_in_docker.sh +++ b/test_in_docker.sh @@ -4,7 +4,7 @@ set -e # Exit on error # Default values PYTHON_VERSIONS=("3.8" "3.9") -DB_TYPE=("psql" "mysql") +DB_TYPE=("pgsql" "mysql") CONTAINER_NAME="testdb" NETWORK_NAME="testnetwork" @@ -57,7 +57,7 @@ EOF echo "Running tests with Docker access..." docker rm -f test-foliant:${version} 2>/dev/null || true docker run --rm \ - --network ${NETWORK_NAME}\ + --network ${NETWORK_NAME} \ -v "./:/app/" \ -w /app \ test-foliant:${version} \ diff --git a/tests/db/psql/data/01-init.sql b/tests/db/pgsql/data/01-init.sql similarity index 100% rename from tests/db/psql/data/01-init.sql rename to tests/db/pgsql/data/01-init.sql diff --git a/tests/db/psql/init_db.sh b/tests/db/pgsql/init_db.sh similarity index 80% rename from tests/db/psql/init_db.sh rename to tests/db/pgsql/init_db.sh index 56c9e94..05d4013 100755 --- a/tests/db/psql/init_db.sh +++ b/tests/db/pgsql/init_db.sh @@ -10,7 +10,7 @@ docker run -d --name ${container} \ --network ${network} \ -e POSTGRES_PASSWORD=password \ -e POSTGRES_DB=testdb \ - -v $(pwd)/tests/db/psql/data:/docker-entrypoint-initdb.d \ + -v $(pwd)/tests/db/pgsql/data:/docker-entrypoint-initdb.d \ -p 5432:5432 \ postgres:15 diff --git a/tests/test_mysql_basic.py b/tests/test_mysql_basic.py index a1902b3..c45c634 100644 --- a/tests/test_mysql_basic.py +++ b/tests/test_mysql_basic.py @@ -78,7 +78,7 @@ def test_strict_mysql(self): with patch('os._exit') as mock_exit: result = self.ptf.test_preprocessor( - input_mapping=input_files, - expected_mapping=input_files - ) + input_mapping=input_files, + expected_mapping=input_files + ) mock_exit.assert_called_once_with(1) diff --git a/tests/test_psql_basic.py b/tests/test_pgsql_basic.py similarity index 94% rename from tests/test_psql_basic.py rename to tests/test_pgsql_basic.py index 8c568d3..9560479 100644 --- a/tests/test_psql_basic.py +++ b/tests/test_pgsql_basic.py @@ -58,7 +58,6 @@ def test_strict_pgsql(self): self.ptf.options = { 'dbms': 'pgsql', 'host': 'invalid-host-name', - 'host': 'testdb', 'dbname': 'testdb', 'user': 'postgres', 'password': 'password', @@ -72,7 +71,7 @@ def test_strict_pgsql(self): with patch('os._exit') as mock_exit: result = self.ptf.test_preprocessor( - input_mapping=input_files, - expected_mapping=input_files - ) + input_mapping=input_files, + expected_mapping=input_files + ) mock_exit.assert_called_once_with(1)