fix() optimize to only one call to acceptance tests in travis.yaml (#287

)

.travis.yml

@@ -76,8 +76,7 @@ jobs:
       install:
         - pip install -r tests/acceptance/requirements.txt
       script:
-        - pytest -vv --diff-type=split tests/acceptance/test_acceptance/ --host http://localhost:8080 -k "not test_batch_422"
-        - pytest -vv --diff-type=split tests/acceptance/test_acceptance/ --host http://localhost:8080 -k "test_batch_422"
+        - pytest -vv --diff-type=split tests/acceptance/test_acceptance/ --host http://localhost:8080
 
     - stage: 'Trigger FSC Tests'
       if: (branch = master AND type = push) OR type = pull_request OR tag IS present

tests/acceptance/test_acceptance/conftest.py

@@ -39,12 +39,13 @@ def session_override_sdk_key(session_obj):
     return session_obj
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope='session', autouse=True)
 def agent_server():
     """
     Starts Agent server. Runs tests. Stops Agent server.
     """
     host = os.getenv('host')
+    os.environ['OPTIMIZELY_SERVER_BATCHREQUESTS_OPERATIONSLIMIT'] = '3'
 
     if host == 'http://localhost:8080':
         # start server
@@ -57,27 +58,6 @@ def agent_server():
         yield
 
 
-@pytest.fixture(scope='session')
-def operations_limit():
-    """
-    Starts Agent server. Runs tests. Stops Agent server.
-    For custom operations limit setting.
-    """
-    host = os.getenv('host')
-
-    cmd = 'OPTIMIZELY_SERVER_BATCHREQUESTS_OPERATIONSLIMIT=2 make run'
-
-    if host == 'http://localhost:8080':
-        # start server
-        subprocess.Popen(cmd, shell=True)
-        wait_for_agent_to_start()
-        yield
-        # Stop server
-        stop_server('optimizely')
-    else:
-        yield
-
-
 def stop_server(process):
     """
     Kill all 'optimizely' processes

tests/acceptance/test_acceptance/test_activate.py

@@ -51,7 +51,7 @@
     ("", expected_activate_ab_empty_experimentKey, 200),
     ("invalid exper key", expected_activate_ab_invalid_experimentKey, 200),
 ], ids=["valid case", "empty exper key", "invalid exper key"])
-def test_activate__experiment(agent_server, session_obj, experiment_key, expected_response,
+def test_activate__experiment(session_obj, experiment_key, expected_response,
                               expected_status_code):
     """
     Test validates:
@@ -125,7 +125,7 @@ def test_activate__experiment(agent_server, session_obj, experiment_key, expecte
     ("", expected_activate_feat_empty_featureKey, 200),
     ("invalid feat key", expected_activate_feat_invalid_featureKey, 200),
 ], ids=["valid case", "empty feat key", "invalid feat key"])
-def test_activate__feature(agent_server, session_obj, feature_key, expected_response,
+def test_activate__feature(session_obj, feature_key, expected_response,
                            expected_status_code):
     """
     Test validates:
@@ -230,7 +230,7 @@ def test_activate__feature(agent_server, session_obj, feature_key, expected_resp
     ("", {'error': 'type "" not supported'}, 400, True)
 ], ids=["experiment decision type", "feature decision type", "invalid decision type",
         "empty decision type"])
-def test_activate__type(agent_server, session_obj, decision_type, expected_response,
+def test_activate__type(session_obj, decision_type, expected_response,
                         expected_status_code, bypass_validation):
     """
     Test cases:
@@ -259,7 +259,7 @@ def test_activate__type(agent_server, session_obj, decision_type, expected_respo
             resp.raise_for_status()
 
 
-def test_activate_403(agent_server, session_override_sdk_key):
+def test_activate_403(session_override_sdk_key):
     """
     Test that 403 Forbidden is returned. We use invalid SDK key to trigger 403.
     :param agent_server: starts agent server with default config
@@ -289,7 +289,7 @@ def test_activate_403(agent_server, session_override_sdk_key):
         "feature test and decision_tr true",
         "feature test and decision_tr false", "empty disableTracking",
         "invalid disableTracking"])
-def test_activate__disable_tracking(agent_server, session_obj, experiment, disableTracking,
+def test_activate__disable_tracking(session_obj, experiment, disableTracking,
                                     expected_status_code, bypass_validation):
     """
     Setting to true will disable impression tracking for ab experiments and feature tests.
@@ -428,7 +428,7 @@ def test_activate__disable_tracking(agent_server, session_obj, experiment, disab
             "enabled false, feature on",
             "enabled false, feature off", "empty value for enabled",
             "invalid value for enabled"])
-def test_activate__enabled(agent_server, session_obj, enabled, experimentKey, featureKey,
+def test_activate__enabled(session_obj, enabled, experimentKey, featureKey,
                            expected_response, expected_status_code, bypass_validation):
     """
     Filter the activation response to return only enabled decisions.
@@ -527,7 +527,7 @@ def test_activate__enabled(agent_server, session_obj, enabled, experimentKey, fe
 ]"""
 
 
-def test_activate_with_config(agent_server, session_obj):
+def test_activate_with_config(session_obj):
     """
     Tests experimentKeys, featureKeys, variables and variations because it
     validates against the whole response body.

tests/acceptance/test_acceptance/test_batch.py

@@ -6,7 +6,7 @@
 from tests.acceptance.test_acceptance.conftest import sdk_key
 
 
-def test_batch_valid_reponse(agent_server, session_obj):
+def test_batch_valid_reponse(session_obj):
     # TODO - parameterize to feed in different values in the payload: valid SDK string, invalid sdk string,
     #  empty string, integer, boolean, double
     """
@@ -81,7 +81,7 @@ def test_batch_valid_reponse(agent_server, session_obj):
 }"""
 
 
-def test_batch_valid_response__multiple_operations(agent_server, session_obj):
+def test_batch_valid_response__multiple_operations(session_obj):
     """
     Verify that operations with different sdk keys can be sent in a batch.
     :param agent_server: starts agent server with default config
@@ -155,7 +155,7 @@ def test_batch_valid_response__multiple_operations(agent_server, session_obj):
     resp.raise_for_status()
 
 
-def test_batch_400(agent_server, session_obj):
+def test_batch_400(session_obj):
     """
     Invalid JSON, no SDK key in the operations' header.
     :param agent_server: starts agent server with default config
@@ -189,10 +189,10 @@ def test_batch_400(agent_server, session_obj):
     resp.raise_for_status()
 
 
-def test_batch_422(operations_limit, session_obj):
+def test_batch_422(session_obj):
     """
-    Set env variable OPTIMIZELY_SERVER_BATCHREQUESTS_OPERATIONSLIMIT to 2.
-        Then send 3 operaions, should fail with code 422
+    Set env variable OPTIMIZELY_SERVER_BATCHREQUESTS_OPERATIONSLIMIT to 3 (already set to 3 for all tests)
+    Then send 4 operaions, should fail with code 422
     :param operations_limit: starts agent server with custome set operations limit env var
     :param session_obj: session object
     """
@@ -233,15 +233,31 @@ def test_batch_422(operations_limit, session_obj):
           "operationID": "3",
           "url": "/v1/activate",
           "params": {
-            "type": "feature",
+            "type": "experiment",
+            "experimentKey": "ab_test1"
+          },
+          "headers": {
+            "X-Optimizely-SDK-Key": "%s",
+            "Content-Type": "application/json"
+            }
+          },
+          {
+          "body": {
+            "userId": "user2"
+          },
+          "method": "POST",
+          "operationID": "4",
+          "url": "/v1/activate",
+          "params": {
+            "type": "experiment",
             "experimentKey": "ab_test1"
           },
           "headers": {
             "X-Optimizely-SDK-Key": "%s",
             "Content-Type": "application/json"
           }
         }]
-        }""" % (sdk_key, sdk_key)
+        }""" % (sdk_key, sdk_key, sdk_key)
 
     resp = create_and_validate_request_and_response(ENDPOINT_BATCH, 'post', session_obj, payload=payload,
                                                     bypass_validation=False)

tests/acceptance/test_acceptance/test_config.py

@@ -127,7 +127,7 @@
 }"""
 
 
-def test_config(agent_server, session_obj):
+def test_config(session_obj):
     """
     Test validates all returned available experiment and features definitions
     for this environment.
@@ -144,7 +144,7 @@ def test_config(agent_server, session_obj):
     assert json.loads(expected_config) == resp.json()
 
 
-def test_config_403(agent_server, session_override_sdk_key):
+def test_config_403(session_override_sdk_key):
     """
     Test that 403 Forbidden is returned. We use invalid SDK key to trigger 403.
     :param agent_server: starts agent server with default config

tests/acceptance/test_acceptance/test_overrides.py

@@ -12,7 +12,7 @@
 BASE_URL = os.getenv('host')
 
 
-def test_overrides(agent_server, session_obj):
+def test_overrides(session_obj):
     """
     Override an experiment decision for a user
 
@@ -108,7 +108,7 @@ def test_overrides(agent_server, session_obj):
         ("matjaz", "ab_test1", "invalid_variation", 200, expected_invalid_variation_key, False),
     ], ids=["empty_userId", "empty_experiment_key", "empty_variationKey",
             "invalid_userId", "invalid_experimentKey", "invalid_variationKey"])
-def test_overrides__invalid_arguments(agent_server, session_obj, userId, experimentKey, variationKey,
+def test_overrides__invalid_arguments(session_obj, userId, experimentKey, variationKey,
                                       expected_status_code, expected_response, bypass_validation):
     """
     :param agent_server: starts agent server with default config
@@ -123,7 +123,7 @@ def test_overrides__invalid_arguments(agent_server, session_obj, userId, experim
     assert resp.text == expected_response
 
 
-def test_overrides_403(agent_server, session_override_sdk_key):
+def test_overrides_403(session_override_sdk_key):
     """
     Test that 403 Forbidden is returned. We use invalid SDK key to trigger 403.
     :param agent_server: starts agent server with default config

tests/acceptance/test_acceptance/test_track.py

@@ -15,7 +15,7 @@
     ("", 400, True),
     ("invalid_event_key", 200, False)
 ], ids=["Valid event key", "Empty event key", "Invalid event key"])
-def test_track(agent_server, session_obj, event_key, status_code,bypass_validation):
+def test_track(session_obj, event_key, status_code,bypass_validation):
     """
     Track event for the given user.
     Track sends event and user details to Optimizely’s analytics backend