Skip to content

Commit 872c852

Browse files
committed
Remove now-unnecessary test on negative samples, fix mde greater than 1 test
1 parent eba061f commit 872c852

File tree

2 files changed

+5
-25
lines changed

2 files changed

+5
-25
lines changed

basemath/basemath.py

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -212,9 +212,3 @@ def __init__(
212212
(self.required_samples, self.intercept) = self._calculate_sample_size(
213213
self.var_H0, self.mean_H1, self.var_H1, self.alpha, self.beta
214214
)
215-
216-
if self.required_samples < 0:
217-
raise AnalysisException(
218-
"The provided alpha and beta values result in a negative number of required"
219-
"samples -- please reconsider your values."
220-
)

tests/basemath/test_basemath.py

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -118,11 +118,11 @@ def test_mde_greater_than_one():
118118
It could easily be greater than or equal to 1, like 2 for a 200% uplift. That's
119119
not a very realistic experiment, but it should still work with our approach.
120120
"""
121-
basemath = BaseMathsTest(0.1, 3, 0.05, 0.2, seed="test-experiment")
122-
assert basemath.required_samples == 47
123-
assert basemath.evaluate_experiment(0, 12, 0, 15) == 0
124-
assert basemath.evaluate_experiment(12, 20, 15, 30) == 0
125-
assert basemath.evaluate_experiment(32, 8, 45, 10) == 1
121+
basemath = BaseMathsTest(0.01, 3, 0.05, 0.2, seed="test-experiment")
122+
assert basemath.required_samples == 261
123+
assert basemath.evaluate_experiment(0, 120, 0, 150) == 0
124+
assert basemath.evaluate_experiment(120, 50, 150, 80) == 0
125+
assert basemath.evaluate_experiment(170, 26, 230, 40) == 1
126126

127127

128128
def test_first_call_concludes_experiment_failure():
@@ -217,20 +217,6 @@ def test_negative_number_of_samples():
217217
assert str(exception_context_manager_previous.value) == expected_exception
218218

219219

220-
def test_experiment_with_negative_required_samples():
221-
"""
222-
Due to some instability in the root-finding method we use, some alpha/beta values
223-
can result in a negative number of samples. We have some basic handling for this case.
224-
"""
225-
with pytest.raises(AnalysisException) as exception_context_manager:
226-
BaseMathsTest(0.3, 0.9, 0.9, 0.01, seed="test-experiment")
227-
expected_exception_text = (
228-
"The provided alpha and beta values result in a negative number of required"
229-
"samples -- please reconsider your values."
230-
)
231-
assert str(exception_context_manager.value) == expected_exception_text
232-
233-
234220
def test_guarantee_crossing_bound():
235221
"""
236222
If the 'line' is below the bound, then we should always fail the experiment,

0 commit comments

Comments
 (0)