diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/Serverable.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/Serverable.scala
index 67021026499..8c52fd45e22 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/Serverable.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/Serverable.scala
@@ -47,13 +47,18 @@ abstract class Serverable(name: String) extends CompositeService(name) {
 
   override def stop(): Unit = synchronized {
     try {
-      stopServer()
+      if (started.getAndSet(false)) {
+        super.stop()
+      }
     } catch {
       case t: Throwable =>
-        warn(s"Error stopping spark ${t.getMessage}", t)
+        warn(s"Error stopping $name ${t.getMessage}", t)
     } finally {
-      if (started.getAndSet(false)) {
-        super.stop()
+      try {
+        stopServer()
+      } catch {
+        case t: Throwable =>
+          warn(s"Error stopping spark ${t.getMessage}", t)
       }
     }
   }
diff --git a/kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala b/kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
index 5f6b2ed3c6b..4ecac80993c 100644
--- a/kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
+++ b/kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
@@ -36,7 +36,6 @@ object KyuubiServer extends Logging {
     if (!ServiceDiscovery.supportServiceDiscovery(conf)) {
       zkServer.initialize(conf)
       zkServer.start()
-      sys.addShutdownHook(zkServer.stop())
       conf.set(HA_ZK_QUORUM, zkServer.getConnectString)
       conf.set(HA_ZK_ACL_ENABLED, false)
     }
@@ -82,7 +81,7 @@ class KyuubiServer(name: String) extends Serverable(name) {
   override private[kyuubi] val backendService: AbstractBackendService = new KyuubiBackendService()
   private val discoveryService = new ServiceDiscovery(this)
 
-  override def initialize(conf: KyuubiConf): Unit = {
+  override def initialize(conf: KyuubiConf): Unit = synchronized {
     val kinit = new KinitAuxiliaryService()
     addService(kinit)
     super.initialize(conf)