From a3d49ce372a4c0751d9d3591ff7817f448742a80 Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Mon, 6 Dec 2021 14:37:50 +0300 Subject: [PATCH 01/11] Add methods to test interface --- models/test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/test.go b/models/test.go index d4368df..d4659a6 100644 --- a/models/test.go +++ b/models/test.go @@ -35,6 +35,7 @@ type TestInterface interface { SetForm(form *Form) SetResponses(map[int]string) SetHeaders(map[string]string) + SetDbQueryString(string) // comparison properties NeedsCheckingValues() bool From defac8baa5a0492dd6bcb8e9567285f4c36e5a85 Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Mon, 6 Dec 2021 14:38:18 +0300 Subject: [PATCH 02/11] Implement all test interface methods into yaml_file.Test --- testloader/yaml_file/test.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/testloader/yaml_file/test.go b/testloader/yaml_file/test.go index 003ca87..1f96020 100644 --- a/testloader/yaml_file/test.go +++ b/testloader/yaml_file/test.go @@ -9,13 +9,13 @@ type Test struct { TestDefinition - Request string - Responses map[int]string - ResponseHeaders map[int]map[string]string - BeforeScript string - AfterRequestScript string - DbQuery string - DbResponse []string + Request string + Responses map[int]string + ResponseHeaders map[int]map[string]string + BeforeScript string + AfterRequestScript string + DbQuery string + DbResponse []string } func (t *Test) ToQuery() string { @@ -161,3 +161,7 @@ func (t *Test) SetResponses(val map[int]string) { func (t *Test) SetHeaders(val map[string]string) { t.HeadersVal = val } + +func (t *Test) SetDbQueryString(query string) { + t.DbQuery = query +} From 0e7c609f32354e6622ea1869200fd695845616b6 Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Mon, 6 Dec 2021 14:39:06 +0300 Subject: [PATCH 03/11] Preprocess response variables before running any checks --- runner/runner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 08651c8..08fc2e1 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -181,6 +181,10 @@ func (r *Runner) executeTest(v models.TestInterface, client *http.Client) (*mode result.Errors = append(result.Errors, errs...) } + if err := r.setVariablesFromResponse(v, result.ResponseContentType, bodyStr, resp.StatusCode); err != nil { + return nil, err + } + for _, c := range r.checkers { errs, err := c.Check(v, &result) if err != nil { @@ -189,10 +193,6 @@ func (r *Runner) executeTest(v models.TestInterface, client *http.Client) (*mode result.Errors = append(result.Errors, errs...) } - if err := r.setVariablesFromResponse(v, result.ResponseContentType, bodyStr, resp.StatusCode); err != nil { - return nil, err - } - return &result, nil } From 7a5a84a12187fa245cec7034d6a98baf12a3e673 Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Mon, 6 Dec 2021 16:18:59 +0300 Subject: [PATCH 04/11] Add new methods to test interface --- models/test.go | 1 + testloader/yaml_file/test.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/models/test.go b/models/test.go index d4659a6..eb7abf4 100644 --- a/models/test.go +++ b/models/test.go @@ -36,6 +36,7 @@ type TestInterface interface { SetResponses(map[int]string) SetHeaders(map[string]string) SetDbQueryString(string) + SetDbResponseJson([]string) // comparison properties NeedsCheckingValues() bool diff --git a/testloader/yaml_file/test.go b/testloader/yaml_file/test.go index 1f96020..3ad0ff5 100644 --- a/testloader/yaml_file/test.go +++ b/testloader/yaml_file/test.go @@ -165,3 +165,7 @@ func (t *Test) SetHeaders(val map[string]string) { func (t *Test) SetDbQueryString(query string) { t.DbQuery = query } + +func (t *Test) SetDbResponseJson(responses []string) { + t.DbResponse = responses +} From 181d10bb5743e0a7bfc81a4b0b7c5ff3032f9c0c Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Mon, 6 Dec 2021 16:19:21 +0300 Subject: [PATCH 05/11] Preprocess test variables before run checks --- runner/runner.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runner/runner.go b/runner/runner.go index 08fc2e1..04c754f 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -185,6 +185,9 @@ func (r *Runner) executeTest(v models.TestInterface, client *http.Client) (*mode return nil, err } + r.config.Variables.Load(v.GetVariables()) + v = r.config.Variables.Apply(v) + for _, c := range r.checkers { errs, err := c.Check(v, &result) if err != nil { From 8e4a4c2c2604fb2af82488161d62b3ee5d53f378 Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Mon, 6 Dec 2021 16:19:39 +0300 Subject: [PATCH 06/11] Preprocess database query and database response in variables --- variables/variables.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/variables/variables.go b/variables/variables.go index 9bbd351..9386699 100644 --- a/variables/variables.go +++ b/variables/variables.go @@ -48,6 +48,8 @@ func (vs *Variables) Apply(t models.TestInterface) models.TestInterface { newTest.SetMethod(vs.perform(newTest.GetMethod())) newTest.SetPath(vs.perform(newTest.Path())) newTest.SetRequest(vs.perform(newTest.GetRequest())) + newTest.SetDbQueryString(vs.perform(newTest.DbQueryString())) + newTest.SetDbResponseJson(vs.performDbResponses(newTest.DbResponseJson())) newTest.SetResponses(vs.performResponses(newTest.GetResponses())) newTest.SetHeaders(vs.performHeaders(newTest.Headers())) @@ -159,6 +161,20 @@ func (vs *Variables) performResponses(responses map[int]string) map[int]string { return res } +func (vs *Variables) performDbResponses(responses []string) []string { + if responses == nil { + return nil + } + + res := make([]string, len(responses)) + + for idx, v := range responses { + res[idx] = vs.perform(v) + } + + return res +} + func (vs *Variables) Add(v *Variable) *Variables { vs.variables[v.name] = v From 0e2b48be03b0ec9e41bc7be5f7dd3b483dba915f Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Mon, 6 Dec 2021 16:19:47 +0300 Subject: [PATCH 07/11] Fixing linter warnings --- runner/runner_upload_file_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/runner/runner_upload_file_test.go b/runner/runner_upload_file_test.go index d9115ea..8df87a3 100644 --- a/runner/runner_upload_file_test.go +++ b/runner/runner_upload_file_test.go @@ -48,7 +48,6 @@ func testServerUpload(t *testing.T) *httptest.Server { _, err = w.Write(respData) require.NoError(t, err) - return })) } From a3d9d4b219482fedb608b98ea225dcb73ed1c5d6 Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Mon, 6 Dec 2021 16:21:05 +0300 Subject: [PATCH 08/11] Add example for test db query and response pre-processing --- examples/with-db-example/Makefile | 8 ++ .../cases/database-with-vars.yaml | 21 +++++ examples/with-db-example/docker-compose.yaml | 38 +++++++++ examples/with-db-example/server.dockerfile | 4 + examples/with-db-example/server.py | 79 +++++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100644 examples/with-db-example/Makefile create mode 100644 examples/with-db-example/cases/database-with-vars.yaml create mode 100644 examples/with-db-example/docker-compose.yaml create mode 100644 examples/with-db-example/server.dockerfile create mode 100644 examples/with-db-example/server.py diff --git a/examples/with-db-example/Makefile b/examples/with-db-example/Makefile new file mode 100644 index 0000000..661c3bf --- /dev/null +++ b/examples/with-db-example/Makefile @@ -0,0 +1,8 @@ +.PHONY: setup +setup: + @docker-compose -f docker-compose.yaml up --build -d + @curl http://localhost:5000/info/10 + +.PHONY: test +test: setup + gonkey -db_dsn "postgresql://testing_user:testing_password@localhost:5432/testing_db?sslmode=disable" -debug -host http://localhost:5000 -tests ./cases/ diff --git a/examples/with-db-example/cases/database-with-vars.yaml b/examples/with-db-example/cases/database-with-vars.yaml new file mode 100644 index 0000000..bae240c --- /dev/null +++ b/examples/with-db-example/cases/database-with-vars.yaml @@ -0,0 +1,21 @@ +- name: Get random number + method: GET + path: /randint/ + response: + 200: '{ "num": {"generated": "$matchRegexp(\\d)" } }' + variables_to_set: + 200: + info_id: num.generated + +- name: Get info with database + method: GET + path: "/info/{{ $info_id }}" + response: + 200: '{"result_id": "{{ $info_id }}", "query_result": [[1, "golang"], [2, "gonkey"]]}' + variables_to_set: + 200: + golang_id: query_result.0.0 + dbQuery: > + SELECT id, name FROM testing WHERE id={{ $golang_id }} + dbResponse: + - '{"id": {{ $golang_id}}, "name": "golang"}' diff --git a/examples/with-db-example/docker-compose.yaml b/examples/with-db-example/docker-compose.yaml new file mode 100644 index 0000000..3c35560 --- /dev/null +++ b/examples/with-db-example/docker-compose.yaml @@ -0,0 +1,38 @@ +version: '3' + +services: + postgres: + image: postgres:10.3 + command: postgres -c 'max_connections=100' + volumes: + - postgres-db:/var/lib/postgresql/data + environment: + - POSTGRES_HOST=postgres + - POSTGRES_PORT=5432 + - POSTGRES_DB=testing_db + - POSTGRES_USER=testing_user + - POSTGRES_PASSWORD=testing_password + ports: + - 5432:5432 + healthcheck: + test: "pg_isready -U postgres" + + svc: + build: + context: . + dockerfile: server.dockerfile + command: python /app/server.py + ports: + - 5000:5000 + environment: + - APP_POSTGRES_HOST=postgres + - APP_POSTGRES_PORT=5432 + - APP_POSTGRES_USER=testing_user + - APP_POSTGRES_PASS=testing_password + - APP_POSTGRES_DB=testing_db + depends_on: + postgres: + condition: service_healthy + +volumes: + postgres-db: \ No newline at end of file diff --git a/examples/with-db-example/server.dockerfile b/examples/with-db-example/server.dockerfile new file mode 100644 index 0000000..4a9afdd --- /dev/null +++ b/examples/with-db-example/server.dockerfile @@ -0,0 +1,4 @@ +FROM python:3.7.9 + +RUN pip install -U psycopg2-binary --no-cache-dir +COPY server.py /app/server.py diff --git a/examples/with-db-example/server.py b/examples/with-db-example/server.py new file mode 100644 index 0000000..93b8cb3 --- /dev/null +++ b/examples/with-db-example/server.py @@ -0,0 +1,79 @@ +import http.server +import random +import json +import os +import socketserver +from http import HTTPStatus + +import psycopg2 + + +class Handler(http.server.SimpleHTTPRequestHandler): + + def get_response(self) -> dict: + if self.path.startswith('/info/'): + response = self.get_info() + elif self.path.startswith('/randint/'): + response = self.get_rand_num() + else: + response = {'non-existing': True} + + return response + + def get_info(self) -> dict: + info_id = self.path.split('/')[-1] + return { + 'result_id': info_id, + 'query_result': storage.get_sql_result('SELECT id, name FROM testing LIMIT 2'), + } + + def get_rand_num(self) -> dict: + return {'num': {'generated': str(random.randint(0, 100))}} + + def do_GET(self): + # заголовки ответа + self.send_response(HTTPStatus.OK) + self.send_header("Content-type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(self.get_response()).encode()) + + +class PostgresStorage: + def __init__(self): + # подключение к Postgres + params = { + "host": os.environ['APP_POSTGRES_HOST'], + "port": os.environ['APP_POSTGRES_PORT'], + "user": os.environ['APP_POSTGRES_USER'], + "password": os.environ['APP_POSTGRES_PASS'], + "database": os.environ['APP_POSTGRES_DB'], + } + self.conn = psycopg2.connect(**params) + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, self.conn) + self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + self.cursor = self.conn.cursor() + + def apply_migrations(self): + self.cursor.execute(""" + CREATE TABLE testing (id SERIAL PRIMARY KEY, name VARCHAR(200) NOT NULL); + """) + self.conn.commit() + self.cursor.executemany( + "INSERT INTO testing (name) VALUES (%(name)s);", + [{'name': 'golang'}, {'name': 'gonkey'}, {'name': 'testing'}], + ) + self.conn.commit() + + def get_sql_result(self, sql_str): + self.cursor.execute(sql_str) + query_data = list(self.cursor.fetchall()) + self.conn.commit() + return query_data + + +storage = PostgresStorage() +storage.apply_migrations() + +if __name__ == '__main__': + service = socketserver.TCPServer(('', 5000), Handler) + service.serve_forever() From f5681d7eaaf7a3ba5492d26c9405ad307dcd7f95 Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Tue, 7 Dec 2021 15:13:21 +0300 Subject: [PATCH 09/11] Fixing migration query inside example --- examples/with-db-example/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/with-db-example/server.py b/examples/with-db-example/server.py index 93b8cb3..116c123 100644 --- a/examples/with-db-example/server.py +++ b/examples/with-db-example/server.py @@ -40,7 +40,6 @@ def do_GET(self): class PostgresStorage: def __init__(self): - # подключение к Postgres params = { "host": os.environ['APP_POSTGRES_HOST'], "port": os.environ['APP_POSTGRES_PORT'], @@ -55,7 +54,7 @@ def __init__(self): def apply_migrations(self): self.cursor.execute(""" - CREATE TABLE testing (id SERIAL PRIMARY KEY, name VARCHAR(200) NOT NULL); + CREATE TABLE IF NOT EXISTS testing (id SERIAL PRIMARY KEY, name VARCHAR(200) NOT NULL); """) self.conn.commit() self.cursor.executemany( From 374a89785e19c7674b28e0e17e0163ac7b05f8a5 Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Thu, 9 Dec 2021 12:19:01 +0300 Subject: [PATCH 10/11] Fixing makefile and test case file inside example --- examples/with-db-example/Makefile | 5 +++++ examples/with-db-example/cases/database-with-vars.yaml | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/with-db-example/Makefile b/examples/with-db-example/Makefile index 661c3bf..500e606 100644 --- a/examples/with-db-example/Makefile +++ b/examples/with-db-example/Makefile @@ -3,6 +3,11 @@ setup: @docker-compose -f docker-compose.yaml up --build -d @curl http://localhost:5000/info/10 +.PHONY: teardown +teardown: + @docker-compose -f docker-compose.yaml down -v --remove-orphans + .PHONY: test test: setup gonkey -db_dsn "postgresql://testing_user:testing_password@localhost:5432/testing_db?sslmode=disable" -debug -host http://localhost:5000 -tests ./cases/ + make teardown diff --git a/examples/with-db-example/cases/database-with-vars.yaml b/examples/with-db-example/cases/database-with-vars.yaml index bae240c..8b4999f 100644 --- a/examples/with-db-example/cases/database-with-vars.yaml +++ b/examples/with-db-example/cases/database-with-vars.yaml @@ -10,12 +10,12 @@ - name: Get info with database method: GET path: "/info/{{ $info_id }}" - response: - 200: '{"result_id": "{{ $info_id }}", "query_result": [[1, "golang"], [2, "gonkey"]]}' variables_to_set: 200: golang_id: query_result.0.0 + response: + 200: '{"result_id": "{{ $info_id }}", "query_result": [[ {{ $golang_id }}, "golang"], [2, "gonkey"]]}' dbQuery: > SELECT id, name FROM testing WHERE id={{ $golang_id }} dbResponse: - - '{"id": {{ $golang_id}}, "name": "golang"}' + - '{"id": {{ $golang_id }}, "name": "golang"}' From fffd9e967af06e981dbc863d0c2b32f82fb50eda Mon Sep 17 00:00:00 2001 From: Nikita Tomchik Date: Thu, 9 Dec 2021 12:19:17 +0300 Subject: [PATCH 11/11] Add new functionality to readme's --- README-ru.md | 26 ++++++++++++++++++++++++++ README.md | 27 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/README-ru.md b/README-ru.md index 13c4f37..f28bcdd 100644 --- a/README-ru.md +++ b/README-ru.md @@ -212,6 +212,8 @@ responseHeaders: - headers - request - response +- dbQuery +- dbResponse - body для моков - headers для моков - requestConstraints для моков @@ -235,12 +237,17 @@ responseHeaders: "message": "{{ $mockParam }}" } statusCode: 200 + dbQuery: > + SELECT id, name FROM testing_tools WHERE id={{ $sqlQueryParam }} + dbResponse: + - '{"id": {{ $sqlResultParam }}, "name": "gonkey"}' ``` Присваивать значения переменным можно следующими способами: - в описании самого теста - из результатов предыдущего запроса +- из результата текущего запроса - в переменных окружения или в env-файле Приоритеты источников соответствуют порядку перечисления. @@ -294,6 +301,25 @@ responseHeaders: Глубина вложенности может быть любая. +##### Из результата текущего запроса + +Пример: + +```yaml +- name: Get info with database + method: GET + path: "/info/1" + variables_to_set: + 200: + golang_id: query_result.0.0 + response: + 200: '{"result_id": "1", "query_result": [[ {{ $golang_id }} , "golang"], [2, "gonkey"]]}' + dbQuery: > + SELECT id, name FROM testing_tools WHERE id={{ $golang_id }} + dbResponse: + - '{"id": {{ $golang_id}}, "name": "golang"}' +``` + ##### В переменных окружения или в env-файле Gonkey автоматически проверяет наличие указанной переменной среди переменных окружения (в таком же регистре) и берет значение оттуда, в случае наличия. diff --git a/README.md b/README.md index faa5dd1..869827f 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,8 @@ You can use variables in the description of the test, the following fields are s - headers - request - response +- dbQuery +- dbResponse - mocks body - mocks headers - mocks requestConstraints @@ -238,12 +240,17 @@ Example: "message": "{{ $mockParam }}" } statusCode: 200 + dbQuery: > + SELECT id, name FROM testing_tools WHERE id={{ $sqlQueryParam }} + dbResponse: + - '{"id": {{ $sqlResultParam }}, "name": "gonkey"}' ``` You can assign values to variables in the following ways (priorities are from top to bottom): - in the description of the test - from the response of the previous test +- from the response of currently running test - from environment variables or from env-file #### More detailed about assignment methods @@ -295,6 +302,26 @@ You can access nested fields like this: Any nesting levels are supported. +##### From the response of currently running test + +Example: + +```yaml +- name: Get info with database + method: GET + path: "/info/1" + variables_to_set: + 200: + golang_id: query_result.0.0 + response: + 200: '{"result_id": "1", "query_result": [[ {{ $golang_id }} , "golang"], [2, "gonkey"]]}' + dbQuery: > + SELECT id, name FROM testing_tools WHERE id={{ $golang_id }} + dbResponse: + - '{"id": {{ $golang_id}}, "name": "golang"}' +``` + + ##### From environment variables or from env-file Gonkey automatically checks if variable exists in the environment variables (case-sensitive) and loads a value from there, if it exists.