Skip to content

Commit 152b24c

Browse files
AnkushMalakerAnkushMalaker
committed
Refactor Redis job queue cleanup process for improved success tracking
- Replaced total job count with separate counters for successful and failed jobs during Redis queue cleanup. - Enhanced logging to provide detailed feedback on the number of jobs cleared and any failures encountered. - Improved error handling to ensure job counts are accurately reflected even when exceptions occur.
1 parent 4f49de6 commit 152b24c

File tree

4 files changed

+50
-25
lines changed

4 files changed

+50
-25
lines changed

‎.github/workflows/full-tests-with-api.yml‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -116,15 +116,20 @@ jobs:
116116
echo "test_exit_code=$TEST_EXIT_CODE" >> $GITHUB_ENV
117117
exit 0 # Don't fail here, we'll fail at the end after uploading artifacts
118118
119-
- name: Show service logs
119+
- name: Save service logs to files
120120
if: always()
121121
working-directory: backends/advanced
122122
run: |
123-
echo "=== Backend Logs (last 50 lines) ==="
124-
docker compose -f docker-compose-test.yml logs --tail=50 chronicle-backend-test
125-
echo ""
126-
echo "=== Worker Logs (last 50 lines) ==="
127-
docker compose -f docker-compose-test.yml logs --tail=50 workers-test
123+
echo "Saving service logs to files..."
124+
mkdir -p logs
125+
docker compose -f docker-compose-test.yml logs --no-log-prefix chronicle-backend-test > logs/backend.log 2>&1 || true
126+
docker compose -f docker-compose-test.yml logs --no-log-prefix workers-test > logs/workers.log 2>&1 || true
127+
docker compose -f docker-compose-test.yml logs --no-log-prefix mongo-test > logs/mongo.log 2>&1 || true
128+
docker compose -f docker-compose-test.yml logs --no-log-prefix redis-test > logs/redis.log 2>&1 || true
129+
docker compose -f docker-compose-test.yml logs --no-log-prefix qdrant-test > logs/qdrant.log 2>&1 || true
130+
docker compose -f docker-compose-test.yml logs --no-log-prefix speaker-service-test > logs/speaker.log 2>&1 || true
131+
echo "✓ Logs saved to backends/advanced/logs/"
132+
ls -lh logs/
128133
129134
- name: Check if test results exist
130135
if: always()
@@ -200,6 +205,7 @@ jobs:
200205
with:
201206
name: robot-test-logs-full
202207
path: |
208+
backends/advanced/logs/*.log
203209
backends/advanced/.env
204210
tests/setup/.env.test
205211
retention-days: 7

‎.github/workflows/pr-tests-with-api.yml‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -110,15 +110,20 @@ jobs:
110110
echo "test_exit_code=$TEST_EXIT_CODE" >> $GITHUB_ENV
111111
exit 0 # Don't fail here, we'll fail at the end after uploading artifacts
112112
113-
- name: Show service logs
113+
- name: Save service logs to files
114114
if: always()
115115
working-directory: backends/advanced
116116
run: |
117-
echo "=== Backend Logs (last 50 lines) ==="
118-
docker compose -f docker-compose-test.yml logs --tail=50 chronicle-backend-test
119-
echo ""
120-
echo "=== Worker Logs (last 50 lines) ==="
121-
docker compose -f docker-compose-test.yml logs --tail=50 workers-test
117+
echo "Saving service logs to files..."
118+
mkdir -p logs
119+
docker compose -f docker-compose-test.yml logs --no-log-prefix chronicle-backend-test > logs/backend.log 2>&1 || true
120+
docker compose -f docker-compose-test.yml logs --no-log-prefix workers-test > logs/workers.log 2>&1 || true
121+
docker compose -f docker-compose-test.yml logs --no-log-prefix mongo-test > logs/mongo.log 2>&1 || true
122+
docker compose -f docker-compose-test.yml logs --no-log-prefix redis-test > logs/redis.log 2>&1 || true
123+
docker compose -f docker-compose-test.yml logs --no-log-prefix qdrant-test > logs/qdrant.log 2>&1 || true
124+
docker compose -f docker-compose-test.yml logs --no-log-prefix speaker-service-test > logs/speaker.log 2>&1 || true
125+
echo "✓ Logs saved to backends/advanced/logs/"
126+
ls -lh logs/
122127
123128
- name: Check if test results exist
124129
if: always()
@@ -242,6 +247,7 @@ jobs:
242247
with:
243248
name: robot-test-logs-pr-labeled
244249
path: |
250+
backends/advanced/logs/*.log
245251
backends/advanced/.env
246252
tests/setup/.env.test
247253
retention-days: 7

‎.github/workflows/robot-tests.yml‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -85,15 +85,20 @@ jobs:
8585
echo "test_exit_code=$TEST_EXIT_CODE" >> $GITHUB_ENV
8686
exit 0 # Don't fail here, we'll fail at the end after uploading artifacts
8787
88-
- name: Show service logs
88+
- name: Save service logs to files
8989
if: always()
9090
working-directory: backends/advanced
9191
run: |
92-
echo "=== Backend Logs (last 50 lines) ==="
93-
docker compose -f docker-compose-test.yml logs --tail=50 chronicle-backend-test
94-
echo ""
95-
echo "=== Worker Logs (last 50 lines) ==="
96-
docker compose -f docker-compose-test.yml logs --tail=50 workers-test
92+
echo "Saving service logs to files..."
93+
mkdir -p logs
94+
docker compose -f docker-compose-test.yml logs --no-log-prefix chronicle-backend-test > logs/backend.log 2>&1 || true
95+
docker compose -f docker-compose-test.yml logs --no-log-prefix workers-test > logs/workers.log 2>&1 || true
96+
docker compose -f docker-compose-test.yml logs --no-log-prefix mongo-test > logs/mongo.log 2>&1 || true
97+
docker compose -f docker-compose-test.yml logs --no-log-prefix redis-test > logs/redis.log 2>&1 || true
98+
docker compose -f docker-compose-test.yml logs --no-log-prefix qdrant-test > logs/qdrant.log 2>&1 || true
99+
docker compose -f docker-compose-test.yml logs --no-log-prefix speaker-service-test > logs/speaker.log 2>&1 || true
100+
echo "✓ Logs saved to backends/advanced/logs/"
101+
ls -lh logs/
97102
98103
- name: Check if test results exist
99104
if: always()
@@ -215,8 +220,9 @@ jobs:
215220
if: failure()
216221
uses: actions/upload-artifact@v4
217222
with:
218-
name: robot-test-logs
223+
name: robot-test-logs-no-api
219224
path: |
225+
backends/advanced/logs/*.log
220226
backends/advanced/.env
221227
tests/setup/.env.test
222228
retention-days: 7

‎backends/advanced/src/scripts/cleanup_state.py‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -564,10 +564,12 @@ def _cleanup_redis(self, stats: CleanupStats):
564564
logger.info("Cleaning Redis job queues...")
565565

566566
queue_names = ["transcription", "memory", "audio", "default"]
567-
total_jobs = 0
567+
successful_jobs = 0
568+
failed_jobs = 0
568569
failed_queues = []
569570

570571
for queue_name in queue_names:
572+
job_count = 0 # Initialize to 0 in case counting fails
571573
try:
572574
queue = Queue(queue_name, connection=self.redis_conn)
573575

@@ -581,7 +583,6 @@ def _cleanup_redis(self, stats: CleanupStats):
581583
len(queue.deferred_job_registry) +
582584
len(queue.scheduled_job_registry)
583585
)
584-
total_jobs += job_count
585586

586587
# Clear queue and registries
587588
queue.empty()
@@ -601,18 +602,24 @@ def _cleanup_redis(self, stats: CleanupStats):
601602
for job_id in queue.scheduled_job_registry.get_job_ids():
602603
queue.scheduled_job_registry.remove(job_id)
603604

605+
# Only count as successful if cleanup completed without exception
606+
successful_jobs += job_count
604607
logger.info(f"Cleared {queue_name} queue ({job_count} jobs)")
605608

606609
except Exception as e:
607610
logger.error(f"Failed to clean {queue_name} queue: {e}", exc_info=True)
611+
# job_count might be 0 if counting failed, or partial count if cleanup failed
612+
failed_jobs += job_count
608613
failed_queues.append(queue_name)
609614
# Continue processing remaining queues
610615

611-
stats.redis_jobs_count = total_jobs
612-
logger.info(f"Cleared total of {total_jobs} Redis jobs")
613-
616+
stats.redis_jobs_count = successful_jobs
614617
if failed_queues:
615-
logger.warning(f"Failed to clean queues: {', '.join(failed_queues)}")
618+
logger.warning(
619+
f"Cleared {successful_jobs} Redis jobs, failed to clear {failed_jobs} jobs from queues: {', '.join(failed_queues)}"
620+
)
621+
else:
622+
logger.info(f"Cleared total of {successful_jobs} Redis jobs")
616623

617624
def _cleanup_legacy_wav(self, stats: CleanupStats):
618625
"""Clean legacy WAV files"""

0 commit comments

Comments
 (0)