Skip to content

Commit

Permalink
fix(tests): Fix flaky test_readiness_not_enough_memory_bytes (#4115)
Browse files Browse the repository at this point in the history
Unflake `test_readiness_not_enough_memory` and
`test_readiness_not_enough_bytes`. The relay errors expected by these
tests did not always occur in order, and not always within 1 second.

I confirmed by brute force that these tests no longer fail, even with
100 repetitions:
https://github.com/getsentry/relay/actions/runs/11235435151/job/31233375132?pr=4115#step:8:735
  • Loading branch information
jjbayer authored Oct 8, 2024
1 parent 89a453f commit d1e950f
Showing 1 changed file with 23 additions and 16 deletions.
39 changes: 23 additions & 16 deletions tests/integration/test_healthchecks.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
Test the health check endpoints
"""

import queue
import time
import tempfile
import os
Expand Down Expand Up @@ -80,21 +81,34 @@ def test_readiness_proxy(mini_sentry, relay):
assert response.status_code == 200


def assert_not_enough_memory(relay, mini_sentry, expected_comparison):
response = wait_get(relay, "/api/relay/healthcheck/ready/")
assert response.status_code == 503
errors = ""
for _ in range(100):
try:
errors += f"{mini_sentry.test_failures.get(timeout=10)}\n"
except queue.Empty:
break
if (
"Not enough memory" in errors
and expected_comparison in errors
and "Health check probe 'system memory'" in errors
and "Health check probe 'spool health'" in errors
):
return

assert False, f"Not all errors represented: {errors}"


def test_readiness_not_enough_memory_bytes(mini_sentry, relay):
relay = relay(
mini_sentry,
{"relay": {"mode": "proxy"}, "health": {"max_memory_bytes": 42}},
wait_health_check=False,
)

response = wait_get(relay, "/api/relay/healthcheck/ready/")
error = str(mini_sentry.test_failures.get(timeout=2))
assert "Not enough memory" in error and ">= 42" in error
error = str(mini_sentry.test_failures.get(timeout=1))
assert "Health check probe 'system memory'" in error
error = str(mini_sentry.test_failures.get(timeout=1))
assert "Health check probe 'spool health'" in error
assert response.status_code == 503
assert_not_enough_memory(relay, mini_sentry, ">= 42")


def test_readiness_not_enough_memory_percent(mini_sentry, relay):
Expand All @@ -103,15 +117,8 @@ def test_readiness_not_enough_memory_percent(mini_sentry, relay):
{"relay": {"mode": "proxy"}, "health": {"max_memory_percent": 0.01}},
wait_health_check=False,
)
response = wait_get(relay, "/api/relay/healthcheck/ready/")

error = str(mini_sentry.test_failures.get(timeout=2))
assert "Not enough memory" in error and ">= 1.00%" in error
error = str(mini_sentry.test_failures.get(timeout=1))
assert "Health check probe 'system memory'" in error
error = str(mini_sentry.test_failures.get(timeout=1))
assert "Health check probe 'spool health'" in error
assert response.status_code == 503
assert_not_enough_memory(relay, mini_sentry, ">= 1.00%")


def test_readiness_depends_on_aggregator_being_full(mini_sentry, relay):
Expand Down

0 comments on commit d1e950f

Please sign in to comment.