Skip to content

Commit

Permalink
fix exception handling in rebalane thread
Browse files Browse the repository at this point in the history
Signed-off-by: Leonid Chernin <leonidc@il.ibm.com>
  • Loading branch information
Leonid Chernin committed Dec 18, 2024
1 parent 2539ac4 commit c4d72fa
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions control/rebalance.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import time
import json
import re
import os
import signal
from .utils import GatewayLogger
from .config import GatewayConfig
from .proto import gateway_pb2 as pb2
Expand Down Expand Up @@ -40,8 +42,10 @@ def auto_rebalance_task(self ):
self.logger.debug(f"Nothing found for rebalance, break at {i} iteration")
break
except Exception:
self.logger.exception(f"Exception in auto rebalance")
raise
self.logger.exception(f"Exception in auto rebalance")
parent_pid = os.getppid()
os.kill(parent_pid, signal.SIGTERM)
raise SystemExit(f"auto rebalance failed, quitting gateway")
time.sleep(0.01) #release lock for 10ms after rebalancing each 1 NS
time.sleep(self.rebalance_period_sec)

Expand Down

0 comments on commit c4d72fa

Please sign in to comment.