diff --git a/.travis.yml b/.travis.yml index 98bf14d9..cc19bc0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,8 +76,7 @@ jobs: install: - pip install -r tests/acceptance/requirements.txt script: - - pytest -vv --diff-type=split tests/acceptance/test_acceptance/ --host http://localhost:8080 -k "not test_batch_422" - - pytest -vv --diff-type=split tests/acceptance/test_acceptance/ --host http://localhost:8080 -k "test_batch_422" + - pytest -vv --diff-type=split tests/acceptance/test_acceptance/ --host http://localhost:8080 - stage: 'Trigger FSC Tests' if: (branch = master AND type = push) OR type = pull_request OR tag IS present diff --git a/tests/acceptance/test_acceptance/conftest.py b/tests/acceptance/test_acceptance/conftest.py index 895bf1d8..f6262344 100644 --- a/tests/acceptance/test_acceptance/conftest.py +++ b/tests/acceptance/test_acceptance/conftest.py @@ -39,12 +39,13 @@ def session_override_sdk_key(session_obj): return session_obj -@pytest.fixture(scope='session') +@pytest.fixture(scope='session', autouse=True) def agent_server(): """ Starts Agent server. Runs tests. Stops Agent server. """ host = os.getenv('host') + os.environ['OPTIMIZELY_SERVER_BATCHREQUESTS_OPERATIONSLIMIT'] = '3' if host == 'http://localhost:8080': # start server @@ -57,27 +58,6 @@ def agent_server(): yield -@pytest.fixture(scope='session') -def operations_limit(): - """ - Starts Agent server. Runs tests. Stops Agent server. - For custom operations limit setting. - """ - host = os.getenv('host') - - cmd = 'OPTIMIZELY_SERVER_BATCHREQUESTS_OPERATIONSLIMIT=2 make run' - - if host == 'http://localhost:8080': - # start server - subprocess.Popen(cmd, shell=True) - wait_for_agent_to_start() - yield - # Stop server - stop_server('optimizely') - else: - yield - - def stop_server(process): """ Kill all 'optimizely' processes diff --git a/tests/acceptance/test_acceptance/test_activate.py b/tests/acceptance/test_acceptance/test_activate.py index c0f94b3e..00164bc3 100644 --- a/tests/acceptance/test_acceptance/test_activate.py +++ b/tests/acceptance/test_acceptance/test_activate.py @@ -51,7 +51,7 @@ ("", expected_activate_ab_empty_experimentKey, 200), ("invalid exper key", expected_activate_ab_invalid_experimentKey, 200), ], ids=["valid case", "empty exper key", "invalid exper key"]) -def test_activate__experiment(agent_server, session_obj, experiment_key, expected_response, +def test_activate__experiment(session_obj, experiment_key, expected_response, expected_status_code): """ Test validates: @@ -125,7 +125,7 @@ def test_activate__experiment(agent_server, session_obj, experiment_key, expecte ("", expected_activate_feat_empty_featureKey, 200), ("invalid feat key", expected_activate_feat_invalid_featureKey, 200), ], ids=["valid case", "empty feat key", "invalid feat key"]) -def test_activate__feature(agent_server, session_obj, feature_key, expected_response, +def test_activate__feature(session_obj, feature_key, expected_response, expected_status_code): """ Test validates: @@ -230,7 +230,7 @@ def test_activate__feature(agent_server, session_obj, feature_key, expected_resp ("", {'error': 'type "" not supported'}, 400, True) ], ids=["experiment decision type", "feature decision type", "invalid decision type", "empty decision type"]) -def test_activate__type(agent_server, session_obj, decision_type, expected_response, +def test_activate__type(session_obj, decision_type, expected_response, expected_status_code, bypass_validation): """ Test cases: @@ -259,7 +259,7 @@ def test_activate__type(agent_server, session_obj, decision_type, expected_respo resp.raise_for_status() -def test_activate_403(agent_server, session_override_sdk_key): +def test_activate_403(session_override_sdk_key): """ Test that 403 Forbidden is returned. We use invalid SDK key to trigger 403. :param agent_server: starts agent server with default config @@ -289,7 +289,7 @@ def test_activate_403(agent_server, session_override_sdk_key): "feature test and decision_tr true", "feature test and decision_tr false", "empty disableTracking", "invalid disableTracking"]) -def test_activate__disable_tracking(agent_server, session_obj, experiment, disableTracking, +def test_activate__disable_tracking(session_obj, experiment, disableTracking, expected_status_code, bypass_validation): """ Setting to true will disable impression tracking for ab experiments and feature tests. @@ -428,7 +428,7 @@ def test_activate__disable_tracking(agent_server, session_obj, experiment, disab "enabled false, feature on", "enabled false, feature off", "empty value for enabled", "invalid value for enabled"]) -def test_activate__enabled(agent_server, session_obj, enabled, experimentKey, featureKey, +def test_activate__enabled(session_obj, enabled, experimentKey, featureKey, expected_response, expected_status_code, bypass_validation): """ Filter the activation response to return only enabled decisions. @@ -527,7 +527,7 @@ def test_activate__enabled(agent_server, session_obj, enabled, experimentKey, fe ]""" -def test_activate_with_config(agent_server, session_obj): +def test_activate_with_config(session_obj): """ Tests experimentKeys, featureKeys, variables and variations because it validates against the whole response body. diff --git a/tests/acceptance/test_acceptance/test_batch.py b/tests/acceptance/test_acceptance/test_batch.py index b5a50e7f..7e0c819f 100644 --- a/tests/acceptance/test_acceptance/test_batch.py +++ b/tests/acceptance/test_acceptance/test_batch.py @@ -6,7 +6,7 @@ from tests.acceptance.test_acceptance.conftest import sdk_key -def test_batch_valid_reponse(agent_server, session_obj): +def test_batch_valid_reponse(session_obj): # TODO - parameterize to feed in different values in the payload: valid SDK string, invalid sdk string, # empty string, integer, boolean, double """ @@ -81,7 +81,7 @@ def test_batch_valid_reponse(agent_server, session_obj): }""" -def test_batch_valid_response__multiple_operations(agent_server, session_obj): +def test_batch_valid_response__multiple_operations(session_obj): """ Verify that operations with different sdk keys can be sent in a batch. :param agent_server: starts agent server with default config @@ -155,7 +155,7 @@ def test_batch_valid_response__multiple_operations(agent_server, session_obj): resp.raise_for_status() -def test_batch_400(agent_server, session_obj): +def test_batch_400(session_obj): """ Invalid JSON, no SDK key in the operations' header. :param agent_server: starts agent server with default config @@ -189,10 +189,10 @@ def test_batch_400(agent_server, session_obj): resp.raise_for_status() -def test_batch_422(operations_limit, session_obj): +def test_batch_422(session_obj): """ - Set env variable OPTIMIZELY_SERVER_BATCHREQUESTS_OPERATIONSLIMIT to 2. - Then send 3 operaions, should fail with code 422 + Set env variable OPTIMIZELY_SERVER_BATCHREQUESTS_OPERATIONSLIMIT to 3 (already set to 3 for all tests) + Then send 4 operaions, should fail with code 422 :param operations_limit: starts agent server with custome set operations limit env var :param session_obj: session object """ @@ -233,7 +233,23 @@ def test_batch_422(operations_limit, session_obj): "operationID": "3", "url": "/v1/activate", "params": { - "type": "feature", + "type": "experiment", + "experimentKey": "ab_test1" + }, + "headers": { + "X-Optimizely-SDK-Key": "%s", + "Content-Type": "application/json" + } + }, + { + "body": { + "userId": "user2" + }, + "method": "POST", + "operationID": "4", + "url": "/v1/activate", + "params": { + "type": "experiment", "experimentKey": "ab_test1" }, "headers": { @@ -241,7 +257,7 @@ def test_batch_422(operations_limit, session_obj): "Content-Type": "application/json" } }] - }""" % (sdk_key, sdk_key) + }""" % (sdk_key, sdk_key, sdk_key) resp = create_and_validate_request_and_response(ENDPOINT_BATCH, 'post', session_obj, payload=payload, bypass_validation=False) diff --git a/tests/acceptance/test_acceptance/test_config.py b/tests/acceptance/test_acceptance/test_config.py index 92d1803c..324dd63e 100644 --- a/tests/acceptance/test_acceptance/test_config.py +++ b/tests/acceptance/test_acceptance/test_config.py @@ -127,7 +127,7 @@ }""" -def test_config(agent_server, session_obj): +def test_config(session_obj): """ Test validates all returned available experiment and features definitions for this environment. @@ -144,7 +144,7 @@ def test_config(agent_server, session_obj): assert json.loads(expected_config) == resp.json() -def test_config_403(agent_server, session_override_sdk_key): +def test_config_403(session_override_sdk_key): """ Test that 403 Forbidden is returned. We use invalid SDK key to trigger 403. :param agent_server: starts agent server with default config diff --git a/tests/acceptance/test_acceptance/test_overrides.py b/tests/acceptance/test_acceptance/test_overrides.py index efc4c416..d8f90825 100644 --- a/tests/acceptance/test_acceptance/test_overrides.py +++ b/tests/acceptance/test_acceptance/test_overrides.py @@ -12,7 +12,7 @@ BASE_URL = os.getenv('host') -def test_overrides(agent_server, session_obj): +def test_overrides(session_obj): """ Override an experiment decision for a user @@ -108,7 +108,7 @@ def test_overrides(agent_server, session_obj): ("matjaz", "ab_test1", "invalid_variation", 200, expected_invalid_variation_key, False), ], ids=["empty_userId", "empty_experiment_key", "empty_variationKey", "invalid_userId", "invalid_experimentKey", "invalid_variationKey"]) -def test_overrides__invalid_arguments(agent_server, session_obj, userId, experimentKey, variationKey, +def test_overrides__invalid_arguments(session_obj, userId, experimentKey, variationKey, expected_status_code, expected_response, bypass_validation): """ :param agent_server: starts agent server with default config @@ -123,7 +123,7 @@ def test_overrides__invalid_arguments(agent_server, session_obj, userId, experim assert resp.text == expected_response -def test_overrides_403(agent_server, session_override_sdk_key): +def test_overrides_403(session_override_sdk_key): """ Test that 403 Forbidden is returned. We use invalid SDK key to trigger 403. :param agent_server: starts agent server with default config diff --git a/tests/acceptance/test_acceptance/test_track.py b/tests/acceptance/test_acceptance/test_track.py index dcb4c487..0b18230c 100644 --- a/tests/acceptance/test_acceptance/test_track.py +++ b/tests/acceptance/test_acceptance/test_track.py @@ -15,7 +15,7 @@ ("", 400, True), ("invalid_event_key", 200, False) ], ids=["Valid event key", "Empty event key", "Invalid event key"]) -def test_track(agent_server, session_obj, event_key, status_code,bypass_validation): +def test_track(session_obj, event_key, status_code,bypass_validation): """ Track event for the given user. Track sends event and user details to Optimizely’s analytics backend