diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d1477c9..0000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:3.8 -RUN pip install --upgrade pip -COPY ./requirements-dev.txt requirements-dev.txt -COPY ./requirements.txt requirements.txt - -RUN pip install -r requirements-dev.txt -RUN pip install -r requirements.txt - - diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index b5a7f1a..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,15 +0,0 @@ -node { - checkout scm - docker.image('postgres').withRun('-e "POSTGRES_PASSWORD=test -e POSTGRES_USER=test"') { c -> - docker.withRegistry('https://gitlab.aofl.com:5001') { - docker.image( - 'engineering-automation_tools/automation_images/dbt-unit-test-ci:latest' - ).withRun('-v "$(pwd)":/dbt-unit-test').inside("--link ${c.id}:db") { - pip install -e /dbt-unit-test - tox - sh 'while ! nc -z db 5432; do sleep 1; done;' - dut run --log-level debug - } - } - } -} \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 24d63aa..c8bc73b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include dbt_unit_test/templates/**/* -include dbt_unit_test/templates/* \ No newline at end of file +include src/dbt_unit_test/assets/**/* +include src/dbt_unit_test/assets/* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b81be6f --- /dev/null +++ b/Makefile @@ -0,0 +1,47 @@ + +help: + @grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "%-30s %s\n", $$1, $$2}' + +# Local installation +.PHONY: init clean lock update install + +install: ## Initalise the virtual env installing deps + pipenv install --dev + +clean: ## Remove all the unwanted clutter + find src -type d -name __pycache__ | xargs rm -rf + find src -type d -name '*.egg-info' | xargs rm -rf + pipenv clean + rm -rf build + rm -rf dist + +lock: ## Lock dependencies + pipenv lock + +update: ## Update dependencies (whole tree) + pipenv update --dev + +sync: ## Install dependencies as per the lock file + pipenv sync --dev + +# Linting and formatting +.PHONY: lint format + +lint: ## Lint files with flake and mypy + pipenv run flake8 src tests functional + pipenv run mypy src tests functional + pipenv run black --check src tests functional + pipenv run isort --check-only src tests functional + +format: ## Run black and isort + pipenv run black src tests functional + pipenv run isort src tests functional + +# Testing + +.PHONY: test functional +test: ## Run unit tests + TZ=UTC pipenv run pytest tests + +functional: ## Run functional tests + TZ=UTC pipenv run pytest functional diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..e4ac9d0 --- /dev/null +++ b/Pipfile @@ -0,0 +1,21 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +click = "*" +dbt-core = "*" + +[dev-packages] +black = "*" +flake8 = "*" +isort = "*" +mypy = "*" +pytest = "*" +pytest-cov = "*" +pytest-mock = "*" +dbt-unit-test = { path = ".", editable = true } + +[requires] +python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..ac14e39 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,836 @@ +{ + "_meta": { + "hash": { + "sha256": "f6ef5efbbfa587ec31f0c79ab0a70219ffcba69a0b2387adfcf273b30b277bc4" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "agate": { + "hashes": [ + "sha256:2d568fd68a8eb8b56c805a1299ba4bc30ca0434563be1bea309c9d1c1c8401f4", + "sha256:e0f2f813f7e12311a4cdccc97d6ba0a6781e9c1aa8eca0ab00d5931c0113a308" + ], + "version": "==1.6.3" + }, + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.4.0" + }, + "babel": { + "hashes": [ + "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2", + "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13" + ], + "markers": "python_version >= '3.6'", + "version": "==2.10.1" + }, + "certifi": { + "hashes": [ + "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", + "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a" + ], + "markers": "python_version >= '3.6'", + "version": "==2022.5.18.1" + }, + "cffi": { + "hashes": [ + "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3", + "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2", + "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636", + "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20", + "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728", + "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27", + "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66", + "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443", + "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0", + "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7", + "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39", + "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605", + "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a", + "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37", + "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029", + "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139", + "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc", + "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df", + "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14", + "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880", + "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2", + "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a", + "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e", + "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474", + "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024", + "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8", + "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0", + "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e", + "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a", + "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e", + "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032", + "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6", + "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e", + "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b", + "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e", + "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954", + "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962", + "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c", + "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4", + "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55", + "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962", + "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023", + "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c", + "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6", + "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8", + "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382", + "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7", + "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc", + "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997", + "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796" + ], + "version": "==1.15.0" + }, + "charset-normalizer": { + "hashes": [ + "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", + "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" + ], + "markers": "python_version >= '3'", + "version": "==2.0.12" + }, + "click": { + "hashes": [ + "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", + "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" + ], + "index": "pypi", + "version": "==8.1.3" + }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.4" + }, + "dbt-core": { + "hashes": [ + "sha256:3e33ce5be74e0d4f9f28ad76b4b1de55cb343d4c53c3a8f98c24551881aedf3e", + "sha256:4af9124e6ec188db2ce9a32d6b26757fd42c2e14238770ca0661d8963f1ba7ea" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "dbt-extractor": { + "hashes": [ + "sha256:037907a7c7ae0391045d81338ca77ddaef899a91d80f09958f09fe374594e19b", + "sha256:34783d788b133f223844e280e37b3f5244f2fb60acc457aa75c2667e418d5442", + "sha256:35265a0ae0a250623b0c2e3308b2738dc8212e40e0aa88407849e9ea090bb312", + "sha256:3fe8d8e28a7bd3e0884896147269ca0202ca432d8733113386bdc84c824561bf", + "sha256:4dc715bd740e418d8dc1dd418fea508e79208a24cf5ab110b0092a3cbe96bf71", + "sha256:554d27741a54599c39e5c0b7dbcab77400d83f908caba284a3e960db812e5814", + "sha256:75b1c665699ec0f1ffce1ba3d776f7dfce802156f22e70a7b9c8f0b4d7e80f42", + "sha256:76872cdee659075d6ce2df92dc62e59a74ba571be62acab2e297ca478b49d766", + "sha256:7c291f9f483eae4f60dd5859097d7ba51d5cb6c4725f08973ebd18cdea89d758", + "sha256:7d7c47774dc051b8c18690281a55e2e3d3320e823b17e04b06bc3ff81b1874ba", + "sha256:81435841610be1b07806d72cd89b1956c6e2a84c360b9ceb3f949c62a546d569", + "sha256:822b1e911db230e1b9701c99896578e711232001027b518c44c32f79a46fa3f9", + "sha256:9da211869a1220ea55c5552c1567a3ea5233a6c52fa89ca87a22465481c37bc9", + "sha256:a805d51a25317f53cbff951c79b9cf75421cf48e4b3e1dfb3e9e8de6d824b76c", + "sha256:bc9e0050e3a2f4ea9fe58e8794bc808e6709a0c688ed710fc7c5b6ef3e5623ec", + "sha256:cad90ddc708cb4182dc16fe2c87b1f088a1679877b93e641af068eb68a25d582" + ], + "markers": "python_full_version >= '3.6.1'", + "version": "==0.4.1" + }, + "future": { + "hashes": [ + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.18.2" + }, + "hologram": { + "hashes": [ + "sha256:2911b59115bebd0504eb089532e494fa22ac704989afe41371c5361780433bfe", + "sha256:fd67bd069e4681e1d2a447df976c65060d7a90fee7f6b84d133fd9958db074ec" + ], + "version": "==0.0.14" + }, + "idna": { + "hashes": [ + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + ], + "markers": "python_version >= '3.5'", + "version": "==3.3" + }, + "importlib-metadata": { + "hashes": [ + "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700", + "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec" + ], + "markers": "python_version >= '3.7'", + "version": "==4.11.4" + }, + "isodate": { + "hashes": [ + "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", + "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" + ], + "version": "==0.6.1" + }, + "jinja2": { + "hashes": [ + "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", + "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.11.3" + }, + "jsonschema": { + "hashes": [ + "sha256:2fa0684276b6333ff3c0b1b27081f4b2305f0a36cf702a23db50edb141893c3f", + "sha256:94c0a13b4a0616458b42529091624e66700a17f847453e52279e35509a5b7631" + ], + "version": "==3.1.1" + }, + "leather": { + "hashes": [ + "sha256:5e741daee96e9f1e9e06081b8c8a10c4ac199301a0564cdd99b09df15b4603d2", + "sha256:b43e21c8fa46b2679de8449f4d953c06418666dc058ce41055ee8a8d3bb40918" + ], + "version": "==0.3.4" + }, + "logbook": { + "hashes": [ + "sha256:0cf2cdbfb65a03b5987d19109dacad13417809dcf697f66e1a7084fb21744ea9", + "sha256:2dc85f1510533fddb481e97677bb7bca913560862734c0b3b289bfed04f78c92", + "sha256:56ee54c11df3377314cedcd6507638f015b4b88c0238c2e01b5eb44fd3a6ad1b", + "sha256:66f454ada0f56eae43066f604a222b09893f98c1adc18df169710761b8f32fe8", + "sha256:7c533eb728b3d220b1b5414ba4635292d149d79f74f6973b4aa744c850ca944a", + "sha256:8f76a2e7b1f72595f753228732f81ce342caf03babc3fed6bbdcf366f2f20f18", + "sha256:94e2e11ff3c2304b0d09a36c6208e5ae756eb948b210e5cbd63cd8d27f911542", + "sha256:97fee1bd9605f76335b169430ed65e15e457a844b2121bd1d90a08cf7e30aba0", + "sha256:e18f7422214b1cf0240c56f884fd9c9b4ff9d0da2eabca9abccba56df7222f66" + ], + "version": "==1.5.3" + }, + "markupsafe": { + "hashes": [ + "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", + "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", + "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", + "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194", + "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", + "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", + "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724", + "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", + "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646", + "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", + "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6", + "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a", + "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6", + "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad", + "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", + "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38", + "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac", + "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", + "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6", + "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047", + "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", + "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", + "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b", + "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", + "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", + "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a", + "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", + "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1", + "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9", + "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864", + "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", + "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee", + "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f", + "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", + "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", + "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", + "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", + "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b", + "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", + "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86", + "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6", + "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", + "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", + "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", + "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28", + "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e", + "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", + "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", + "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f", + "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d", + "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", + "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", + "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145", + "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", + "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c", + "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1", + "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a", + "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207", + "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", + "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53", + "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd", + "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134", + "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85", + "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9", + "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", + "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", + "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", + "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", + "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "mashumaro": { + "hashes": [ + "sha256:343b6e2d3e432e31973688c4c8821dcd6ef41fd33264b992afc4aecbfd155f18", + "sha256:f616df410d82936b8bb2b4d32af570556685d77f49acf4228134b50230a69799" + ], + "markers": "python_version >= '3.6'", + "version": "==2.9" + }, + "minimal-snowplow-tracker": { + "hashes": [ + "sha256:acabf7572db0e7f5cbf6983d495eef54081f71be392330eb3aadb9ccb39daaa4" + ], + "version": "==0.0.2" + }, + "msgpack": { + "hashes": [ + "sha256:0d8c332f53ffff01953ad25131272506500b14750c1d0ce8614b17d098252fbc", + "sha256:1c58cdec1cb5fcea8c2f1771d7b5fec79307d056874f746690bd2bdd609ab147", + "sha256:2c3ca57c96c8e69c1a0d2926a6acf2d9a522b41dc4253a8945c4c6cd4981a4e3", + "sha256:2f30dd0dc4dfe6231ad253b6f9f7128ac3202ae49edd3f10d311adc358772dba", + "sha256:2f97c0f35b3b096a330bb4a1a9247d0bd7e1f3a2eba7ab69795501504b1c2c39", + "sha256:36a64a10b16c2ab31dcd5f32d9787ed41fe68ab23dd66957ca2826c7f10d0b85", + "sha256:3d875631ecab42f65f9dce6f55ce6d736696ced240f2634633188de2f5f21af9", + "sha256:40fb89b4625d12d6027a19f4df18a4de5c64f6f3314325049f219683e07e678a", + "sha256:47d733a15ade190540c703de209ffbc42a3367600421b62ac0c09fde594da6ec", + "sha256:494471d65b25a8751d19c83f1a482fd411d7ca7a3b9e17d25980a74075ba0e88", + "sha256:51fdc7fb93615286428ee7758cecc2f374d5ff363bdd884c7ea622a7a327a81e", + "sha256:6eef0cf8db3857b2b556213d97dd82de76e28a6524853a9beb3264983391dc1a", + "sha256:6f4c22717c74d44bcd7af353024ce71c6b55346dad5e2cc1ddc17ce8c4507c6b", + "sha256:73a80bd6eb6bcb338c1ec0da273f87420829c266379c8c82fa14c23fb586cfa1", + "sha256:89908aea5f46ee1474cc37fbc146677f8529ac99201bc2faf4ef8edc023c2bf3", + "sha256:8a3a5c4b16e9d0edb823fe54b59b5660cc8d4782d7bf2c214cb4b91a1940a8ef", + "sha256:96acc674bb9c9be63fa8b6dabc3248fdc575c4adc005c440ad02f87ca7edd079", + "sha256:973ad69fd7e31159eae8f580f3f707b718b61141838321c6fa4d891c4a2cca52", + "sha256:9b6f2d714c506e79cbead331de9aae6837c8dd36190d02da74cb409b36162e8a", + "sha256:9c0903bd93cbd34653dd63bbfcb99d7539c372795201f39d16fdfde4418de43a", + "sha256:9fce00156e79af37bb6db4e7587b30d11e7ac6a02cb5bac387f023808cd7d7f4", + "sha256:a598d0685e4ae07a0672b59792d2cc767d09d7a7f39fd9bd37ff84e060b1a996", + "sha256:b0a792c091bac433dfe0a70ac17fc2087d4595ab835b47b89defc8bbabcf5c73", + "sha256:bb87f23ae7d14b7b3c21009c4b1705ec107cb21ee71975992f6aca571fb4a42a", + "sha256:bf1e6bfed4860d72106f4e0a1ab519546982b45689937b40257cfd820650b920", + "sha256:c1ba333b4024c17c7591f0f372e2daa3c31db495a9b2af3cf664aef3c14354f7", + "sha256:c2140cf7a3ec475ef0938edb6eb363fa704159e0bf71dde15d953bacc1cf9d7d", + "sha256:c7e03b06f2982aa98d4ddd082a210c3db200471da523f9ac197f2828e80e7770", + "sha256:d02cea2252abc3756b2ac31f781f7a98e89ff9759b2e7450a1c7a0d13302ff50", + "sha256:da24375ab4c50e5b7486c115a3198d207954fe10aaa5708f7b65105df09109b2", + "sha256:e4c309a68cb5d6bbd0c50d5c71a25ae81f268c2dc675c6f4ea8ab2feec2ac4e2", + "sha256:f01b26c2290cbd74316990ba84a14ac3d599af9cebefc543d241a66e785cf17d", + "sha256:f201d34dc89342fabb2a10ed7c9a9aaaed9b7af0f16a5923f1ae562b31258dea", + "sha256:f74da1e5fcf20ade12c6bf1baa17a2dc3604958922de8dc83cbe3eff22e8b611" + ], + "version": "==1.0.3" + }, + "networkx": { + "hashes": [ + "sha256:51d6ae63c24dcd33901357688a2ad20d6bcd38f9a4c5307720048d3a8081059c", + "sha256:ae99c9b0d35e5b4a62cf1cfea01e5b3633d8d02f4a0ead69685b6e7de5b85eab" + ], + "markers": "python_version >= '3.8'", + "version": "==2.8.2" + }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "markers": "python_version >= '3.6'", + "version": "==21.3" + }, + "parsedatetime": { + "hashes": [ + "sha256:3d817c58fb9570d1eec1dd46fa9448cd644eeed4fb612684b02dfda3a79cb84b", + "sha256:9ee3529454bf35c40a77115f5a596771e59e1aee8c53306f346c461b8e913094" + ], + "version": "==2.4" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "pyparsing": { + "hashes": [ + "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", + "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + ], + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.9" + }, + "pyrsistent": { + "hashes": [ + "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c", + "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc", + "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e", + "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26", + "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec", + "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286", + "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045", + "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec", + "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8", + "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c", + "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca", + "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22", + "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a", + "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96", + "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc", + "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1", + "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07", + "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6", + "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b", + "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5", + "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6" + ], + "markers": "python_version >= '3.7'", + "version": "==0.18.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "python-slugify": { + "hashes": [ + "sha256:272d106cb31ab99b3496ba085e3fea0e9e76dcde967b5e9992500d1f785ce4e1", + "sha256:7b2c274c308b62f4269a9ba701aa69a797e9bca41aeee5b3a9e79e36b6656927" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==6.1.2" + }, + "pytimeparse": { + "hashes": [ + "sha256:04b7be6cc8bd9f5647a6325444926c3ac34ee6bc7e69da4367ba282f076036bd", + "sha256:e86136477be924d7e670646a98561957e8ca7308d44841e21f5ddea757556a0a" + ], + "version": "==1.1.8" + }, + "pytz": { + "hashes": [ + "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", + "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + ], + "version": "==2022.1" + }, + "pyyaml": { + "hashes": [ + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "markers": "python_version >= '3.6'", + "version": "==6.0" + }, + "requests": { + "hashes": [ + "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", + "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==2.27.1" + }, + "setuptools": { + "hashes": [ + "sha256:68e45d17c9281ba25dc0104eadd2647172b3472d9e01f911efa57965e8d51a36", + "sha256:a43bdedf853c670e5fed28e5623403bad2f73cf02f9a2774e91def6bda8265a7" + ], + "markers": "python_version >= '3.7'", + "version": "==62.3.2" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "sqlparse": { + "hashes": [ + "sha256:0c00730c74263a94e5a9919ade150dfc3b19c574389985446148402998287dae", + "sha256:48719e356bb8b42991bdbb1e8b83223757b93789c00910a616a071910ca4a64d" + ], + "markers": "python_version >= '3.5'", + "version": "==0.4.2" + }, + "text-unidecode": { + "hashes": [ + "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", + "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93" + ], + "version": "==1.3" + }, + "typing-extensions": { + "hashes": [ + "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708", + "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376" + ], + "markers": "python_version >= '3.7'", + "version": "==4.2.0" + }, + "urllib3": { + "hashes": [ + "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", + "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.9" + }, + "werkzeug": { + "hashes": [ + "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6", + "sha256:72a4b735692dd3135217911cbeaa1be5fa3f62bffb8745c5215420a03dc55255" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.2" + }, + "zipp": { + "hashes": [ + "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad", + "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099" + ], + "markers": "python_version >= '3.7'", + "version": "==3.8.0" + } + }, + "develop": { + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.4.0" + }, + "black": { + "hashes": [ + "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b", + "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176", + "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09", + "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a", + "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015", + "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79", + "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb", + "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20", + "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464", + "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968", + "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82", + "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21", + "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0", + "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265", + "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b", + "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a", + "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72", + "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce", + "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0", + "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a", + "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163", + "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad", + "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d" + ], + "index": "pypi", + "version": "==22.3.0" + }, + "click": { + "hashes": [ + "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", + "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" + ], + "index": "pypi", + "version": "==8.1.3" + }, + "coverage": { + "extras": [ + "toml" + ], + "hashes": [ + "sha256:00c8544510f3c98476bbd58201ac2b150ffbcce46a8c3e4fb89ebf01998f806a", + "sha256:016d7f5cf1c8c84f533a3c1f8f36126fbe00b2ec0ccca47cc5731c3723d327c6", + "sha256:03014a74023abaf5a591eeeaf1ac66a73d54eba178ff4cb1fa0c0a44aae70383", + "sha256:033ebec282793bd9eb988d0271c211e58442c31077976c19c442e24d827d356f", + "sha256:21e6686a95025927775ac501e74f5940cdf6fe052292f3a3f7349b0abae6d00f", + "sha256:26f8f92699756cb7af2b30720de0c5bb8d028e923a95b6d0c891088025a1ac8f", + "sha256:2e76bd16f0e31bc2b07e0fb1379551fcd40daf8cdf7e24f31a29e442878a827c", + "sha256:341e9c2008c481c5c72d0e0dbf64980a4b2238631a7f9780b0fe2e95755fb018", + "sha256:3cfd07c5889ddb96a401449109a8b97a165be9d67077df6802f59708bfb07720", + "sha256:4002f9e8c1f286e986fe96ec58742b93484195defc01d5cc7809b8f7acb5ece3", + "sha256:50ed480b798febce113709846b11f5d5ed1e529c88d8ae92f707806c50297abf", + "sha256:543e172ce4c0de533fa892034cce260467b213c0ea8e39da2f65f9a477425211", + "sha256:5a78cf2c43b13aa6b56003707c5203f28585944c277c1f3f109c7b041b16bd39", + "sha256:5cd698341626f3c77784858427bad0cdd54a713115b423d22ac83a28303d1d95", + "sha256:60c2147921da7f4d2d04f570e1838db32b95c5509d248f3fe6417e91437eaf41", + "sha256:62d382f7d77eeeaff14b30516b17bcbe80f645f5cf02bb755baac376591c653c", + "sha256:69432946f154c6add0e9ede03cc43b96e2ef2733110a77444823c053b1ff5166", + "sha256:727dafd7f67a6e1cad808dc884bd9c5a2f6ef1f8f6d2f22b37b96cb0080d4f49", + "sha256:742fb8b43835078dd7496c3c25a1ec8d15351df49fb0037bffb4754291ef30ce", + "sha256:750e13834b597eeb8ae6e72aa58d1d831b96beec5ad1d04479ae3772373a8088", + "sha256:7b546cf2b1974ddc2cb222a109b37c6ed1778b9be7e6b0c0bc0cf0438d9e45a6", + "sha256:83bd142cdec5e4a5c4ca1d4ff6fa807d28460f9db919f9f6a31babaaa8b88426", + "sha256:8d2e80dd3438e93b19e1223a9850fa65425e77f2607a364b6fd134fcd52dc9df", + "sha256:9229d074e097f21dfe0643d9d0140ee7433814b3f0fc3706b4abffd1e3038632", + "sha256:968ed5407f9460bd5a591cefd1388cc00a8f5099de9e76234655ae48cfdbe2c3", + "sha256:9c82f2cd69c71698152e943f4a5a6b83a3ab1db73b88f6e769fabc86074c3b08", + "sha256:a00441f5ea4504f5abbc047589d09e0dc33eb447dc45a1a527c8b74bfdd32c65", + "sha256:a022394996419142b33a0cf7274cb444c01d2bb123727c4bb0b9acabcb515dea", + "sha256:af5b9ee0fc146e907aa0f5fb858c3b3da9199d78b7bb2c9973d95550bd40f701", + "sha256:b5578efe4038be02d76c344007b13119b2b20acd009a88dde8adec2de4f630b5", + "sha256:b84ab65444dcc68d761e95d4d70f3cfd347ceca5a029f2ffec37d4f124f61311", + "sha256:c53ad261dfc8695062fc8811ac7c162bd6096a05a19f26097f411bdf5747aee7", + "sha256:cc173f1ce9ffb16b299f51c9ce53f66a62f4d975abe5640e976904066f3c835d", + "sha256:d548edacbf16a8276af13063a2b0669d58bbcfca7c55a255f84aac2870786a61", + "sha256:d55fae115ef9f67934e9f1103c9ba826b4c690e4c5bcf94482b8b2398311bf9c", + "sha256:d8099ea680201c2221f8468c372198ceba9338a5fec0e940111962b03b3f716a", + "sha256:e35217031e4b534b09f9b9a5841b9344a30a6357627761d4218818b865d45055", + "sha256:e4f52c272fdc82e7c65ff3f17a7179bc5f710ebc8ce8a5cadac81215e8326740", + "sha256:e637ae0b7b481905358624ef2e81d7fb0b1af55f5ff99f9ba05442a444b11e45", + "sha256:eef5292b60b6de753d6e7f2d128d5841c7915fb1e3321c3a1fe6acfe76c38052", + "sha256:fb45fe08e1abc64eb836d187b20a59172053999823f7f6ef4f18a819c44ba16f" + ], + "markers": "python_version >= '3.7'", + "version": "==6.4" + }, + "dbt-unit-test": { + "editable": true, + "path": "." + }, + "flake8": { + "hashes": [ + "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", + "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + ], + "index": "pypi", + "version": "==4.0.1" + }, + "iniconfig": { + "hashes": [ + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" + ], + "version": "==1.1.1" + }, + "isort": { + "hashes": [ + "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7", + "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951" + ], + "index": "pypi", + "version": "==5.10.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "mypy": { + "hashes": [ + "sha256:0ebfb3f414204b98c06791af37a3a96772203da60636e2897408517fcfeee7a8", + "sha256:239d6b2242d6c7f5822163ee082ef7a28ee02e7ac86c35593ef923796826a385", + "sha256:29dc94d9215c3eb80ac3c2ad29d0c22628accfb060348fd23d73abe3ace6c10d", + "sha256:2c7f8bb9619290836a4e167e2ef1f2cf14d70e0bc36c04441e41487456561409", + "sha256:33d53a232bb79057f33332dbbb6393e68acbcb776d2f571ba4b1d50a2c8ba873", + "sha256:3a3e525cd76c2c4f90f1449fd034ba21fcca68050ff7c8397bb7dd25dd8b8248", + "sha256:3eabcbd2525f295da322dff8175258f3fc4c3eb53f6d1929644ef4d99b92e72d", + "sha256:481f98c6b24383188c928f33dd2f0776690807e12e9989dd0419edd5c74aa53b", + "sha256:7a76dc4f91e92db119b1be293892df8379b08fd31795bb44e0ff84256d34c251", + "sha256:7d390248ec07fa344b9f365e6ed9d205bd0205e485c555bed37c4235c868e9d5", + "sha256:826a2917c275e2ee05b7c7b736c1e6549a35b7ea5a198ca457f8c2ebea2cbecf", + "sha256:85cf2b14d32b61db24ade8ac9ae7691bdfc572a403e3cb8537da936e74713275", + "sha256:8d645e9e7f7a5da3ec3bbcc314ebb9bb22c7ce39e70367830eb3c08d0140b9ce", + "sha256:925aa84369a07846b7f3b8556ccade1f371aa554f2bd4fb31cb97a24b73b036e", + "sha256:a85a20b43fa69efc0b955eba1db435e2ffecb1ca695fe359768e0503b91ea89f", + "sha256:bfd4f6536bd384c27c392a8b8f790fd0ed5c0cf2f63fc2fed7bce56751d53026", + "sha256:cb7752b24528c118a7403ee955b6a578bfcf5879d5ee91790667c8ea511d2085", + "sha256:cc537885891382e08129d9862553b3d00d4be3eb15b8cae9e2466452f52b0117", + "sha256:d4fccf04c1acf750babd74252e0f2db6bd2ac3aa8fe960797d9f3ef41cf2bfd4", + "sha256:f1ba54d440d4feee49d8768ea952137316d454b15301c44403db3f2cb51af024", + "sha256:f47322796c412271f5aea48381a528a613f33e0a115452d03ae35d673e6064f8", + "sha256:fbfb873cf2b8d8c3c513367febde932e061a5f73f762896826ba06391d932b2a", + "sha256:ffdad80a92c100d1b0fe3d3cf1a4724136029a29afe8566404c0146747114382" + ], + "index": "pypi", + "version": "==0.960" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "markers": "python_version >= '3.6'", + "version": "==21.3" + }, + "pathspec": { + "hashes": [ + "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a", + "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1" + ], + "version": "==0.9.0" + }, + "platformdirs": { + "hashes": [ + "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", + "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" + ], + "markers": "python_version >= '3.7'", + "version": "==2.5.2" + }, + "pluggy": { + "hashes": [ + "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", + "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" + ], + "markers": "python_version >= '3.6'", + "version": "==1.0.0" + }, + "py": { + "hashes": [ + "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", + "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.11.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" + }, + "pyflakes": { + "hashes": [ + "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", + "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.0" + }, + "pyparsing": { + "hashes": [ + "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", + "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + ], + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.9" + }, + "pytest": { + "hashes": [ + "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c", + "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45" + ], + "index": "pypi", + "version": "==7.1.2" + }, + "pytest-cov": { + "hashes": [ + "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6", + "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "pytest-mock": { + "hashes": [ + "sha256:5112bd92cc9f186ee96e1a92efc84969ea494939c3aead39c50f421c4cc69534", + "sha256:6cff27cec936bf81dc5ee87f07132b807bcda51106b5ec4b90a04331cba76231" + ], + "index": "pypi", + "version": "==3.7.0" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708", + "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376" + ], + "markers": "python_version >= '3.7'", + "version": "==4.2.0" + } + } +} diff --git a/README.md b/README.md index cb18264..947e9d0 100644 --- a/README.md +++ b/README.md @@ -260,4 +260,4 @@ The generic version of this code is not so easy on the eyes, but what is importa ) }} ``` -Now we can write many tests for this reusable code, and use it with confidence because we have tested it thoroughly! \ No newline at end of file +Now we can write many tests for this reusable code, and use it with confidence because we have tested it thoroughly! diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index c61977e..0000000 --- a/codecov.yml +++ /dev/null @@ -1,8 +0,0 @@ -comment: off - -coverage: - status: - project: - default: - target: auto - threshold: 5 diff --git a/dbt_unit_test/app.py b/dbt_unit_test/app.py deleted file mode 100644 index 09a9148..0000000 --- a/dbt_unit_test/app.py +++ /dev/null @@ -1,81 +0,0 @@ -"""dbt_unit_test.app: CLI for running unit tests on dbt macros.""" - -import os, sys -import click -import yaml -import shutil - - -from .log_setup import logger, console, LOG_LEVELS -from . import operations as ops - -@click.group() -def dut(): - pass - -@click.command() -def init(): - if not os.path.exists('dbt_unit_test.yml'): - with open('dbt_unit_test.yml', 'w') as conf_file: - conf_file.write(ops.render_template('default_config.yml')) - example_test_path = os.path.join(os.path.dirname(__file__), 'templates/example_test') - if os.path.exists('unit_tests/example_test'): - shutil.rmtree('unit_tests/example_test') - shutil.copytree(example_test_path, 'unit_tests/example_test') - click.secho('DUT setup complete.') - click.secho('Please set up a dbt profile called "unit_test".', fg='blue') - - -@click.command() -@click.option('--tests', help='tests to run.') -@click.option('--batches', default=2, help='batches to run.') -@click.option('--log-level', default='info', help='Set log level.') -def run(tests, batches, log_level): - """Run unit tests on a dbt models.""" - # use defaults if there is no config file. - - with open('dbt_unit_test.yml', 'r') as conf_file: - config = yaml.safe_load(conf_file.read()) - - console.setLevel(LOG_LEVELS.get(log_level, 'info')) - - profile = ['--profile', config['unit_test_profile']] - - unit_test_dir = config['unit_test_dir'] - - model = ['--model'] - model += [f'+{test}_model+' for test in tests.split(',')] if tests else [f'+path:models/{unit_test_dir}+'] - select = ['--select'] + model[1:] - - ops.remove_files(**config) - ops.copy_files(**config) - - errors = 0 - - errors += ops.dbt_sp(['dbt', 'seed', '--full-refresh'] + select + profile) - - for batch in range(1, batches+1): - vars_ = [] - if batch < batches: - vars_ += ['--vars', f"batch: {batch}"] - - if batch == 1: - vars_ += ['--full-refresh'] - - errors += ops.dbt_sp(['dbt', 'run'] + model + profile + vars_) - - errors += ops.dbt_sp(['dbt', 'test'] + model + profile) - - if log_level != 'debug': - ops.remove_files(**config) - - if errors != 0: - sys.exit(os.EX_SOFTWARE) - else: - logger.info('All tests passed!') - -dut.add_command(run) -dut.add_command(init) - -if __name__ == '__main__': - dut() \ No newline at end of file diff --git a/dbt_unit_test/log_setup.py b/dbt_unit_test/log_setup.py deleted file mode 100644 index 51772dd..0000000 --- a/dbt_unit_test/log_setup.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging, os - -LOG_LEVELS = { - 'critical': logging.CRITICAL, - 'error': logging.ERROR, - 'warn': logging.WARNING, - 'warning': logging.WARNING, - 'info': logging.INFO, - 'debug': logging.DEBUG -} - -os.makedirs('logs', exist_ok=True) - -logging.basicConfig(level=logging.DEBUG, - format='%(name)-3s %(levelname)-8s %(message)s', - filename='logs/dut.log', - filemode='w') - -console = logging.StreamHandler() -console.setLevel(logging.INFO) - -formatter = logging.Formatter('%(message)s') - -console.setFormatter(formatter) - -logger = logging.getLogger('dut') - -logger.addHandler(console) \ No newline at end of file diff --git a/dbt_unit_test/operations.py b/dbt_unit_test/operations.py deleted file mode 100644 index 0b3b971..0000000 --- a/dbt_unit_test/operations.py +++ /dev/null @@ -1,104 +0,0 @@ -import os -import subprocess -import logging -import glob -import shutil -import jinja2 - -from .log_setup import logger - -def dbt_sp(cmd, log_level=logging.INFO): - sp = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd='.', - close_fds=True) - logger.info(' '.join(cmd)) - line, done_line = '', '' - for line in iter(sp.stdout.readline, b''): - line = line.decode().rstrip() - if 'Done.' in line: - done_line = line - if 'FAIL' in line or 'Finished' in line: - logger.info(line) - else: - logger.debug(line) - sp.wait() - if sp.returncode: - logger.warning(f"\033[31mdbt {cmd[1]} failures. {done_line[6:]}\033[0m") - return 1 - return 0 - - -def map_dbt_file_to_dut_file(dbt_dir, dut_file): - return os.path.join(dbt_dir, '_'.join(dut_file.rsplit('/', 1))) - -def get_test_name_from_dbt_model_path(p): - return p.split('/')[-1].rsplit('_',1)[0] - - -def write_derived_file(original_file_name, derived_file_type): - test_name = get_test_name_from_dbt_model_path(original_file_name) - - derived_file_name = original_file_name.replace('model.sql', derived_file_type) - - if not os.path.exists(derived_file_name): - with open(derived_file_name, 'w') as f: - f.write( - render_template(derived_file_type, test_name=test_name) - ) - - -def copy_files( - unit_test_dir='unit_tests', - models_dir='models', - data_dir='data', - macros_dir='macros', - **kw -): - for f in glob.glob(unit_test_dir + '/**/*.[ys][mq]l', recursive=True): - dbt_file = map_dbt_file_to_dut_file(models_dir, f) - os.makedirs(os.path.dirname(dbt_file), exist_ok=True) - shutil.copy(f, dbt_file) - write_derived_file(dbt_file, 'model.yml') - write_derived_file(dbt_file, 'batch.sql') - - for f in glob.glob(unit_test_dir + '/**/*.csv', recursive=True): - dbt_file = map_dbt_file_to_dut_file(data_dir, f) - os.makedirs(os.path.dirname(dbt_file), exist_ok=True) - shutil.copy(f, dbt_file) - - input_file_name = dbt_file.replace('expect.csv', 'input.csv') - if 'expect' in dbt_file and not os.path.exists(input_file_name): - with open(input_file_name, 'w') as dummy_input_file: - dummy_input_file.write('batch') - - macro_filepath = os.path.join(macros_dir, unit_test_dir, 'test_macros.sql') - os.makedirs(os.path.dirname(macro_filepath), exist_ok=True) - with open(macro_filepath, 'w') as macro_file: - macro_file.write( - render_template('diff_macro.sql')+ - render_template('drop_schema_macro.sql') - ) - -def remove_files( - unit_test_dir='unit_tests', - models_dir='models', - data_dir='data', - macros_dir='macros', - **kw -): - shutil.rmtree(os.path.join(models_dir, unit_test_dir), ignore_errors=True) - shutil.rmtree(os.path.join(data_dir, unit_test_dir), ignore_errors=True) - shutil.rmtree(os.path.join(macros_dir, unit_test_dir), ignore_errors=True) - - -def render_template(template, **kw): - templates_dir = os.path.join(os.path.dirname(__file__), 'templates') - with open(os.path.join(templates_dir, template)) as t: - out = t.read() - if kw: - return jinja2.Template(out).render(**kw) - return out - diff --git a/dbt_unit_test/templates/batch.sql b/dbt_unit_test/templates/batch.sql deleted file mode 100644 index 8c6feae..0000000 --- a/dbt_unit_test/templates/batch.sql +++ /dev/null @@ -1,9 +0,0 @@ -{% raw %}{{ - config( - materialized='ephemeral', - tags=['unit_test', '{% endraw %}{{ test_name }}{% raw %}'] - ) -}} -SELECT * FROM {{ ref('{% endraw %}{{ test_name }}{% raw %}_input') }} -WHERE batch <= {{ var('batch', 100) }}{% endraw %} --- {% raw %}{{ ref('{% endraw %}{{ test_name }}{% raw %}_expect') }}{% endraw %} diff --git a/dbt_unit_test/templates/diff_macro.sql b/dbt_unit_test/templates/diff_macro.sql deleted file mode 100644 index b737a1a..0000000 --- a/dbt_unit_test/templates/diff_macro.sql +++ /dev/null @@ -1,10 +0,0 @@ -{% macro test_diff(model, test) %} -with extra_rows as ( - SELECT * FROM {{ model }} EXCEPT SELECT * FROM {{ test }} -), -missing_rows as ( - SELECT * FROM {{ test }} EXCEPT SELECT * FROM {{ model }} -) -SELECT count(*) -FROM (SELECT * FROM extra_rows UNION ALL SELECT * FROM missing_rows) a -{% endmacro %} diff --git a/dbt_unit_test/templates/drop_schema_macro.sql b/dbt_unit_test/templates/drop_schema_macro.sql deleted file mode 100644 index ecb783b..0000000 --- a/dbt_unit_test/templates/drop_schema_macro.sql +++ /dev/null @@ -1,6 +0,0 @@ -{% macro drop_schema(name) %} -{% set sql %} -DROP SCHEMA IF EXISTS {{ name }} -{% endset %} -{% do run_query(sql) %} -{% endmacro %} diff --git a/dbt_unit_test/templates/model.yml b/dbt_unit_test/templates/model.yml deleted file mode 100644 index 9b874f0..0000000 --- a/dbt_unit_test/templates/model.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -models: - - name: {{ test_name }}_model - tests: - - diff: - test: ref('{{ test_name }}_expect') diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index eb13ce6..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: "3" -services: - db: - image: postgres - restart: always - env_file: .env - - dut-ci: - image: gitlab.aofl.com:5001/engineering-automation_tools/automation_images/dbt-unit-test-ci:latest - working_dir: /dbt-unit-test/tests/test_project - env_file: .env - command: | - /bin/bash -c " - pip install -e /dbt-unit-test - tox - dut run --log-level debug - " - depends_on: - - db - volumes: - - type: bind - source: . - target: /dbt-unit-test diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dfdac69 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.isort] +profile = "black" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 707c971..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -pytest -pytest-cov -pre-commit -dbt -tox diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 36ee380..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -click -dbt \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..6e2adb2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,24 @@ +[bdist_wheel] +universal = 1 + +[flake8] +exclude = logs, .*/*.py, *.egg-info +max-line-length = 99 +max-complexity = 10 +ignore = F405,W391,W503 + +[mypy] +incremental = True +warn_unused_configs = True +warn_unused_ignores = True +check_untyped_defs = True +warn_redundant_casts = True +ignore_missing_imports = True + +[tool:pytest] +python_files = test_*.py +python_classes = Test +python_functions = test_* + +addopts = --junitxml=coverage.xml --cov-report=xml --cov-report=term --cov=./src + diff --git a/setup.py b/setup.py index 496047b..8a97a1c 100644 --- a/setup.py +++ b/setup.py @@ -1,40 +1,36 @@ -"""dbt-unit-test module.""" - +#!/usr/bin/env python +# -*- coding: utf-8 -*- from setuptools import find_packages, setup -with open("README.md") as f: - readme = f.read() - -# Runtime Requirements. -inst_reqs = ["click", "jinja2"] - -# Dev Requirements -extra_reqs = { - "test": ["pytest", "pytest-cov"], - "dev": ["pytest", "pytest-cov", "pre-commit"], -} +PACKAGE_NAME = "dbt-unit-test" +__version__ = "0.1.0" -setup( - name="dbt-unit-test", - version="0.0.5", - description=u"A tiny framework for testing reusable code inside of dbt models", - long_description=readme, - long_description_content_type="text/markdown", +setup_args = dict( + # Description + name=PACKAGE_NAME, + version=__version__, + description="A framework for dbt macro testing", python_requires=">=3.6", + # Credentials + author="Benjamin Ryon", + author_email="benjamin.ryon@aofl.com", + url="https://github.com/AgeOfLearning/dbt-unit-test", classifiers=[ "Intended Audience :: Information Technology", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.8", ], - keywords="sql dbt test unittest", - author="Benjamin Ryon", - author_email="benjamin.ryon@aofl.com", - url="https://github.com/AgeOfLearning/dbt-unit-test", - packages=find_packages(exclude=["ez_setup", "examples", "tests"]), + license="MIT", + # Package data + package_dir={"": "src"}, + packages=find_packages( + "src", + ), include_package_data=True, zip_safe=False, - install_requires=inst_reqs, - extras_require=extra_reqs, - entry_points={"console_scripts": ["dut = dbt_unit_test.app:dut"]}, + entry_points={"console_scripts": ["dut=dbt_unit_test.app:dut"]}, ) + +if __name__ == "__main__": + setup(**setup_args) diff --git a/dbt_unit_test/__init__.py b/src/dbt_unit_test/__init__.py similarity index 100% rename from dbt_unit_test/__init__.py rename to src/dbt_unit_test/__init__.py diff --git a/src/dbt_unit_test/app.py b/src/dbt_unit_test/app.py new file mode 100644 index 0000000..6815733 --- /dev/null +++ b/src/dbt_unit_test/app.py @@ -0,0 +1,114 @@ +"""dbt_unit_test.app: CLI for running unit tests on dbt macros.""" + +import os +import shutil +import sys + +import click +import yaml + +from dbt_unit_test import operations as ops +from dbt_unit_test.log_setup import LOG_LEVELS, console, logger + + +@click.group() +def dut(): + pass + + +@click.command() +def init(): + if not os.path.exists("dbt_unit_test.yml"): + with open("dbt_unit_test.yml", "w") as conf_file: + conf_file.write(ops.render_template("default_config.yml")) + example_test_path = os.path.join(os.path.dirname(__file__), "assets/example_test") + if os.path.exists("unit_tests/example_test"): + shutil.rmtree("unit_tests/example_test") + shutil.copytree(example_test_path, "unit_tests/example_test") + click.secho("DUT setup complete.") + # TODO Check if the appropriate profile exists already + click.secho('Please set up a dbt profile called "unit_test".', fg="blue") + + +@click.command() +@click.option("--tests", help="tests to run.") +@click.option("--batches", default=2, help="batches to run.") +@click.option("--log-level", default="info", help="Set log level.") +@click.option( + "--cleanup/--no-cleanup", + is_flag=True, + default=True, + help="Cleanup generated test files.", +) +def run(tests, batches, log_level, cleanup): + """Run unit tests on a dbt models.""" + + # use defaults if there is no config file. + + with open("dbt_unit_test.yml", "r") as conf_file: + config = yaml.safe_load(conf_file.read()) + + console.setLevel(LOG_LEVELS.get(log_level, "info")) + + profile = ["--profile", config["unit_test_profile"]] + + seed_dir = os.path.join(config["seed_dir"], config["unit_test_dir"]) + models_dir = os.path.join(config["models_dir"], config["unit_test_dir"]) + + # TODO: do we need the +model+? There shouldn't be dependencies for unit tests + seed = ( + [f"+{test}_model+" for test in tests.split(" ")] + if tests + else [f"+path:{seed_dir}+"] + ) + model = ( + [f"+{test}_model+" for test in tests.split(" ")] + if tests + else [f"+path:{models_dir}+"] + ) + select_seed = ["--select"] + seed + select_model = ["--select"] + model + + ops.remove_files(**config) + ops.copy_files(**config) + + errors = 0 + + errors += ops.dbt_sp(["dbt", "seed", "--full-refresh"] + select_seed + profile) + + for batch in range(1, batches + 1): + vars_ = [] + if batch <= batches: + vars_ += ["--vars", f"{{batch: {batch}}}"] + + if batch == 1: + vars_ += ["--full-refresh"] + + errors += ops.dbt_sp(["dbt", "run"] + select_model + profile + vars_) + + errors += ops.dbt_sp(["dbt", "test"] + select_model + profile) + + # TODO make this a cleanup flag (default: True) + if cleanup: + ops.remove_files(**config) + + if errors != 0: + sys.exit(os.EX_SOFTWARE) + else: + logger.info("All tests passed!") + + +@click.command() +def cleanup(): + """Clean up files generated by the dut tool.""" + with open("dbt_unit_test.yml", "r") as conf_file: + config = yaml.safe_load(conf_file.read()) + ops.remove_files(**config) + + +dut.add_command(run) +dut.add_command(init) +dut.add_command(cleanup) + +if __name__ == "__main__": + dut() diff --git a/src/dbt_unit_test/assets/batch.sql b/src/dbt_unit_test/assets/batch.sql new file mode 100644 index 0000000..207897e --- /dev/null +++ b/src/dbt_unit_test/assets/batch.sql @@ -0,0 +1,13 @@ + +{% raw %} +{{ + config( + materialized='ephemeral', + tags=['unit_test', '{% endraw %}{{ test_name }}{% raw %}'] + ) +}} + +select * +from {{ ref('{% endraw %}{{ test_name }}{% raw %}_input') }} +where batch = {{ var('batch') }} +{% endraw %} diff --git a/dbt_unit_test/templates/default_config.yml b/src/dbt_unit_test/assets/default_config.yml similarity index 95% rename from dbt_unit_test/templates/default_config.yml rename to src/dbt_unit_test/assets/default_config.yml index c25db51..67c8938 100644 --- a/dbt_unit_test/templates/default_config.yml +++ b/src/dbt_unit_test/assets/default_config.yml @@ -6,7 +6,7 @@ unit_test_dir: unit_tests # these must match (or at least be includied in) the settings of your dbt_project.yml models_dir: models macros_dir: macros -data_dir: data +seed_dir: seeds # this is the profile that will run your tests unit_test_profile: unit_test diff --git a/src/dbt_unit_test/assets/drop_schema_macro.sql b/src/dbt_unit_test/assets/drop_schema_macro.sql new file mode 100644 index 0000000..ce317cc --- /dev/null +++ b/src/dbt_unit_test/assets/drop_schema_macro.sql @@ -0,0 +1,9 @@ +{% macro drop_schema(name) %} + + {% set sql %} + drop schema if exists {{ name }} + {% endset %} + + {% do run_query(sql) %} + +{% endmacro %} diff --git a/dbt_unit_test/templates/example_test/expect.csv b/src/dbt_unit_test/assets/example_test/expect.csv similarity index 100% rename from dbt_unit_test/templates/example_test/expect.csv rename to src/dbt_unit_test/assets/example_test/expect.csv diff --git a/dbt_unit_test/templates/example_test/input.csv b/src/dbt_unit_test/assets/example_test/input.csv similarity index 100% rename from dbt_unit_test/templates/example_test/input.csv rename to src/dbt_unit_test/assets/example_test/input.csv diff --git a/dbt_unit_test/templates/example_test/model.sql b/src/dbt_unit_test/assets/example_test/model.sql similarity index 100% rename from dbt_unit_test/templates/example_test/model.sql rename to src/dbt_unit_test/assets/example_test/model.sql diff --git a/src/dbt_unit_test/assets/model.yml b/src/dbt_unit_test/assets/model.yml new file mode 100644 index 0000000..d301a58 --- /dev/null +++ b/src/dbt_unit_test/assets/model.yml @@ -0,0 +1,8 @@ +--- +version: 2 + +models: + - name: {{ test_name }}_model + tests: + - dbt_utils.equality: + compare_model: ref('{{ test_name }}_expect') diff --git a/src/dbt_unit_test/log_setup.py b/src/dbt_unit_test/log_setup.py new file mode 100644 index 0000000..f0873c1 --- /dev/null +++ b/src/dbt_unit_test/log_setup.py @@ -0,0 +1,31 @@ +import logging +import os + +LOG_LEVELS = { + "critical": logging.CRITICAL, + "error": logging.ERROR, + "warn": logging.WARNING, + "warning": logging.WARNING, + "info": logging.INFO, + "debug": logging.DEBUG, +} + +os.makedirs("logs", exist_ok=True) + +logging.basicConfig( + level=logging.DEBUG, + format="%(name)-3s %(levelname)-8s %(message)s", + filename="logs/dut.log", + filemode="w", +) + +console = logging.StreamHandler() +console.setLevel(logging.INFO) + +formatter = logging.Formatter("%(message)s") + +console.setFormatter(formatter) + +logger = logging.getLogger("dut") + +logger.addHandler(console) diff --git a/src/dbt_unit_test/operations.py b/src/dbt_unit_test/operations.py new file mode 100644 index 0000000..bca613e --- /dev/null +++ b/src/dbt_unit_test/operations.py @@ -0,0 +1,94 @@ +import glob +import logging +import os +import shutil +import subprocess + +import jinja2 + +from dbt_unit_test.log_setup import logger + + +def dbt_sp(cmd, log_level=logging.INFO): + sp = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=".", close_fds=True + ) + logger.info(" ".join(cmd)) + line, done_line = "", "" + for line in iter(sp.stdout.readline, b""): + line = line.decode().rstrip() + if "Done." in line: + done_line = line + if "FAIL" in line or "Finished" in line: + logger.info(line) + else: + logger.debug(line) + sp.wait() + if sp.returncode: + logger.warning(f"\033[31mdbt {cmd[1]} failures. {done_line[6:]}\033[0m") + return 1 + return 0 + + +def map_dbt_file_to_dut_file(dbt_dir, dut_file): + return os.path.join(dbt_dir, "_".join(dut_file.rsplit("/", 1))) + + +def get_test_name_from_dbt_model_path(p): + return p.split("/")[-1].rsplit("_", 1)[0] + + +def write_derived_file(original_file_name, derived_file_type): + test_name = get_test_name_from_dbt_model_path(original_file_name) + + derived_file_name = original_file_name.replace("model.sql", derived_file_type) + + if not os.path.exists(derived_file_name): + with open(derived_file_name, "w") as f: + f.write(render_template(derived_file_type, test_name=test_name)) + + +def copy_files( + unit_test_dir="unit_tests", + models_dir="models", + seed_dir="seeds", + macros_dir="macros", + **kw, +): + for f in glob.glob(unit_test_dir + "/**/*.[ys][mq]l", recursive=True): + dbt_file = map_dbt_file_to_dut_file(models_dir, f) + os.makedirs(os.path.dirname(dbt_file), exist_ok=True) + shutil.copy(f, dbt_file) + write_derived_file(dbt_file, "model.yml") + write_derived_file(dbt_file, "batch.sql") + + for f in glob.glob(unit_test_dir + "/**/*.csv", recursive=True): + dbt_file = map_dbt_file_to_dut_file(seed_dir, f) + os.makedirs(os.path.dirname(dbt_file), exist_ok=True) + shutil.copy(f, dbt_file) + + input_file_name = dbt_file.replace("expect.csv", "input.csv") + if "expect" in dbt_file and not os.path.exists(input_file_name): + with open(input_file_name, "w") as dummy_input_file: + dummy_input_file.write("batch") + + +def remove_files( + unit_test_dir="unit_tests", + models_dir="models", + seed_dir="seeds", + macros_dir="macros", + **kw, +): + shutil.rmtree(os.path.join(models_dir, unit_test_dir), ignore_errors=True) + shutil.rmtree(os.path.join(seed_dir, unit_test_dir), ignore_errors=True) + shutil.rmtree(os.path.join(macros_dir, unit_test_dir), ignore_errors=True) + + +def render_template(template, **kw): + assets_dir = os.path.join(os.path.dirname(__file__), "assets") + with open(os.path.join(assets_dir, template)) as t: + out = t.read() + if kw: + return jinja2.Template(out).render(**kw) + return out diff --git a/tests/test_operations.py b/tests/test_operations.py index cd68fc9..87061a8 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -1,80 +1,92 @@ -import pytest, os, glob, shutil +import glob +import os +import shutil + +import pytest from dbt_unit_test import operations -os.chdir(os.path.join(os.path.dirname(__file__), 'test_project')) +os.chdir(os.path.join(os.path.dirname(__file__), "test_project")) + @pytest.fixture def dirs(): return dict( - unit_test_dir='unit_tests', - models_dir='_models', - data_dir='_data', - macros_dir='_macros' + unit_test_dir="unit_tests", + models_dir="_models", + seed_dir="_seeds", + macros_dir="_macros", ) + @pytest.fixture(autouse=True) def clean_up(): - shutil.rmtree('_models', ignore_errors=True) - shutil.rmtree('_data', ignore_errors=True) - shutil.rmtree('_macros', ignore_errors=True) + shutil.rmtree("_models", ignore_errors=True) + shutil.rmtree("_seeds", ignore_errors=True) + shutil.rmtree("_macros", ignore_errors=True) yield - shutil.rmtree('_models', ignore_errors=True) - shutil.rmtree('_data', ignore_errors=True) - shutil.rmtree('_macros', ignore_errors=True) + shutil.rmtree("_models", ignore_errors=True) + shutil.rmtree("_seeds", ignore_errors=True) + shutil.rmtree("_macros", ignore_errors=True) def test__remove_files(dirs): operations.copy_files(**dirs) operations.remove_files(**dirs) - assert glob.glob('_models/unit_tests/**/*', recursive=True) == [] + assert glob.glob("_models/unit_tests/**/*", recursive=True) == [] + def test__copy_files(dirs): operations.copy_files(**dirs) - assert set(glob.glob('_models/unit_tests/test1*', recursive=True)) == \ - { - '_models/unit_tests/test1_model.yml', - '_models/unit_tests/test1_model.sql', - '_models/unit_tests/test1_batch.sql' - } - assert set(glob.glob('_data/unit_tests/test1*', recursive=True)) == \ - { - '_data/unit_tests/test1_input.csv', - '_data/unit_tests/test1_expect.csv' - } - assert set(glob.glob('_data/unit_tests/test2*', recursive=True)) == \ - { - '_data/unit_tests/test2_input.csv', - '_data/unit_tests/test2_expect.csv' - } - assert os.path.exists('_macros/unit_tests/test_macros.sql') + assert set(glob.glob("_models/unit_tests/test1*", recursive=True)) == { + "_models/unit_tests/test1_model.yml", + "_models/unit_tests/test1_model.sql", + "_models/unit_tests/test1_batch.sql", + } + assert set(glob.glob("_seeds/unit_tests/test1*", recursive=True)) == { + "_seeds/unit_tests/test1_input.csv", + "_seeds/unit_tests/test1_expect.csv", + } + assert set(glob.glob("_seeds/unit_tests/test2*", recursive=True)) == { + "_seeds/unit_tests/test2_input.csv", + "_seeds/unit_tests/test2_expect.csv", + } + def test__write_derived_file(dirs): - dbt_file = '_models/unit_tests/derived_unit_test_model.sql' - batch_file = '_models/unit_tests/derived_unit_test_batch.sql' + dbt_file = "_models/unit_tests/derived_unit_test_model.sql" + batch_file = "_models/unit_tests/derived_unit_test_batch.sql" os.makedirs(os.path.dirname(dbt_file), exist_ok=True) - operations.write_derived_file(dbt_file, 'batch.sql') + operations.write_derived_file(dbt_file, "batch.sql") assert os.path.exists(batch_file) with open(batch_file) as f: derived_unit_test_batch = f.read() assert "tags=['unit_test', 'derived_unit_test']" in derived_unit_test_batch + def test__get_test_name_from_dbt_model_path(): - assert operations.get_test_name_from_dbt_model_path( - '_models/unit_tests/test1_testy_test_model.sql' - ) == 'test1_testy_test' + assert ( + operations.get_test_name_from_dbt_model_path( + "_models/unit_tests/test1_testy_test_model.sql" + ) + == "test1_testy_test" + ) + def test__map_dbt_file_to_dut_file(): - assert operations.map_dbt_file_to_dut_file( - 'dbt_dir', 'unit_tests/test2/model.sql' - ) == 'dbt_dir/unit_tests/test2_model.sql' + assert ( + operations.map_dbt_file_to_dut_file("dbt_dir", "unit_tests/test2/model.sql") + == "dbt_dir/unit_tests/test2_model.sql" + ) + def test__render_template(): - rendered = operations.render_template('model.yml', test_name='my_model') - assert "test: ref('my_model_expect')" in rendered + rendered = operations.render_template("model.yml", test_name="my_model") + assert "compare_model: ref('my_model_expect')" in rendered + def test__dbt_sp(): - assert operations.dbt_sp('ls -la'.split()) == 0 - assert operations.dbt_sp('ls -not-a-command'.split()) == 1 - assert operations.dbt_sp(['echo', '"Done. FAIL"']) == 0 - assert operations.dbt_sp(['echo', '"FAIL."']) == 0 \ No newline at end of file + assert operations.dbt_sp("ls -la".split()) == 0 + assert operations.dbt_sp("ls -not-a-command".split()) == 1 + assert operations.dbt_sp(["echo", '"Done. FAIL"']) == 0 + assert operations.dbt_sp(["echo", '"FAIL."']) == 0 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 24508fb..0000000 --- a/tox.ini +++ /dev/null @@ -1,55 +0,0 @@ -[tox] -envlist = py38 - -[testenv] -extras = test -commands= - python -m pytest --cov dbt_unit_test --cov-report xml --cov-report term-missing --ignore=venv - -# Lint -[flake8] -exclude = .git,__pycache__,docs/source/conf.py,old,build,dist -max-line-length = 90 - -[mypy] -no_strict_optional = True -ignore_missing_imports = True - -[tool:isort] -include_trailing_comma = True -multi_line_output = 3 -line_length = 90 -known_first_party = dbt_unit_test -default_section = THIRDPARTY - -# Autoformatter -[testenv:black] -basepython = python3 -skip_install = true -deps = - black -commands = - black - -# Release tooling -[testenv:build] -basepython = python3 -skip_install = true -deps = - wheel - setuptools -commands = - python setup.py sdist - -[testenv:release] -basepython = python3 -skip_install = true -setenv = - TWINE_USERNAME = {env:TWINE_USERNAME} - TWINE_PASSWORD = {env:TWINE_PASSWORD} -deps = - {[testenv:build]deps} - twine >= 1.5.0 -commands = - {[testenv:build]commands} - twine upload --skip-existing dist/*