Initial tests (#14)

Author: math280h

.flake8

@@ -3,5 +3,5 @@ per-file-ignores = tests/*:S101,*__init__.py:F401 D104,*helpers.py:S311
 application-import-names = redactdump,tests
 import-order-style = google
 select = B,B9,BLK,C,E,F,I,S,W, D, A
-ignore = E203,E501,W503,ANN101,D100,D107,D401,S404,S607,S603,S605,S101,S602, S608
+ignore = E203,E501,W503,ANN101,D100,D107,D401,S404,S607,S603,S605,S101,S602,S608,B905
 max-line-length = 120

.github/workflows/test.yaml

@@ -0,0 +1,22 @@
+name: "Test"
+
+on:
+  push:
+
+jobs:
+  test:
+    name: test
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: python -m pip install -r dev-requirements.txt
+
+    - name: Run tests
+      run: python -m pytest

.github/workflows/type-lint.yaml

@@ -2,13 +2,6 @@ name: "Typing and Linting"
 
 on:
   push:
-    branches: [ main ]
-    paths-ignore:
-      - "README.md"
-  pull_request:
-    branches: [ main ]
-    paths-ignore:
-      - "README.md"
 
 jobs:
   lint:

dev-requirements.txt

@@ -3,7 +3,9 @@ rich==10.15.2
 setuptools~=59.5.0
 PyYAML~=6.0
 schema~=0.7.5
-psycopg2==2.9.2
+psycopg2-binary==2.9.3
 SQLAlchemy~=1.4.27
 faker==10.0.0
 pymysql==1.0.2
+pytest==6.2.5
+nox==2022.11.21

noxfile.py

@@ -1,7 +1,7 @@
 import nox
 from nox.sessions import Session
 
-locations = "redactdump", "noxfile.py"
+locations = "redactdump", "noxfile.py", "tests"
 python_versions = ["3.9"]
 nox.options.sessions = "lint", "mypy"
 

redactdump/core/redactor.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 import re
-from typing import Any, List, Pattern
+from typing import Any, List, Pattern, Union
 
 from faker import Faker
 
@@ -61,7 +61,7 @@ def load_rules(self) -> None:
                             )
                         )
 
-    def get_replacement(self, replacement: str) -> Any:
+    def get_replacement(self, replacement: str) -> Union[str, Any]:
         """
         Get replacement value.
 

setup.py

@@ -26,10 +26,10 @@
     install_requires=[
         'rich==10.15.2',
         'PyYAML==6.0',
-        'schema==0.7.4',
+        'schema==0.7.5',
         'configargparse==1.5.3',
         'SQLAlchemy~=1.4.27',
-        'psycopg2==2.9.2',
+        'psycopg2-binary==2.9.3',
         'pymysql==1.0.2',
         'faker==10.0.0'
     ],

tests/.gitignore

@@ -0,0 +1 @@
+!config.yaml

tests/config.yaml

@@ -0,0 +1,21 @@
+connection:
+  type: pgsql
+  host: 127.0.0.1
+  port: 5432
+  database: postgres
+
+redact:
+  patterns:
+    column:
+      - pattern: '^[a-zA-Z]+_name'
+        replacement: name
+    data:
+      - pattern: '192.168.0.1'
+        replacement: ipv4
+      - pattern: 'John Doe'
+        replacement: name
+
+output:
+  type: multi_file
+  naming: 'dump-[table_name]-[timestamp]' # Default: [table_name]-[timestamp]
+  location: 'tests/tmp/test_output'

tests/test_config.py

@@ -0,0 +1,11 @@
+from configargparse import Namespace
+
+from redactdump.core.config import Config
+
+
+def test_config_load() -> None:
+    """Test Config loading."""
+    config = Config(Namespace(config="tests/config.yaml"))
+
+    assert type(config) is Config
+    assert type(config.config) is dict

tests/test_database.py

@@ -0,0 +1,36 @@
+from configargparse import Namespace
+
+from redactdump.core import Config
+from redactdump.core.redactor import Redactor
+
+
+def test_redaction() -> None:
+    """Test data redaction."""
+    config = Config(Namespace(config="tests/config.yaml", debug=False))
+    redactor = Redactor(config)
+
+    data = [
+        {
+            "full_name": "Test Name",
+            "secondary_name": "John Doe",
+            "ip": "192.168.0.1",
+            "email": "my@email.com",
+        }
+    ]
+    original = [
+        {
+            "full_name": "Test Name",
+            "secondary_name": "John Doe",
+            "ip": "192.168.0.1",
+            "email": "my@email.com",
+        }
+    ]
+
+    for idx, item in enumerate(data):
+
+        if redactor.data_rules or redactor.column_rules:
+            redactor.redact(item, ["full_name", "email"])
+            assert data[idx]["full_name"] != original[idx]["full_name"]
+            assert data[idx]["secondary_name"] != original[idx]["secondary_name"]
+            assert data[idx]["ip"] != original[idx]["ip"]
+            assert data[idx]["email"] == original[idx]["email"]