Crash when Zookeeper connection fails to establish

Previously, Metronome would become a zombie indefinitely if the initial
connection attempt to Zookeeper failed. Now, we crash if any exception is
thrown while initializing the Metronome JobApplicationLoader.

Additionally, we promote several more components to become eagerly initialized,
as opposed to lazy, in order to make it so our exception catch-all can catch
errors.

JIRA Issues: DCOS_OSS-4239

jobs/src/main/scala/dcos/metronome/JobsModule.scala

@@ -25,11 +25,11 @@ class JobsModule(
   private[this] lazy val pluginModule = new PluginModule(config.scallopConf, crashStrategy)
   def pluginManger: PluginManager = pluginModule.pluginManager
 
-  lazy val repositoryModule = new RepositoryModule(config)
+  val repositoryModule = new RepositoryModule(config)
 
-  lazy val actorsModule = new ActorsModule(actorSystem)
+  val actorsModule = new ActorsModule(actorSystem)
 
-  lazy val schedulerRepositoriesModule = new SchedulerRepositoriesModule(metricsModule.metrics, config, repositoryModule, lifecycleState, actorsModule, actorSystem)
+  val schedulerRepositoriesModule = new SchedulerRepositoriesModule(metricsModule.metrics, config, repositoryModule, lifecycleState, actorsModule, actorSystem)
 
   val schedulerModule: SchedulerModule = new SchedulerModule(
     metricsModule.metrics,

src/main/scala/dcos/metronome/JobApplicationLoader.scala

@@ -8,17 +8,17 @@ import controllers.AssetsComponents
 import com.softwaremill.macwire._
 import com.typesafe.scalalogging.StrictLogging
 import dcos.metronome.api.v1.LeaderProxyFilter
-import dcos.metronome.api.{ApiModule, ErrorHandler}
+import dcos.metronome.api.{ ApiModule, ErrorHandler }
 import mesosphere.marathon.MetricsModule
 import mesosphere.marathon.core.async.ExecutionContexts
 import mesosphere.marathon.core.base.JvmExitsCrashStrategy
 import org.slf4j.LoggerFactory
-import play.shaded.ahc.org.asynchttpclient.{AsyncHttpClientConfig, DefaultAsyncHttpClient}
+import play.shaded.ahc.org.asynchttpclient.{ AsyncHttpClientConfig, DefaultAsyncHttpClient }
 import play.api.ApplicationLoader.Context
 import play.api._
 import play.api.i18n._
-import play.api.libs.ws.ahc.{AhcConfigBuilder, AhcWSClient, AhcWSClientConfig, StandaloneAhcWSClient}
-import play.api.libs.ws.{WSClient, WSConfigParser}
+import play.api.libs.ws.ahc.{ AhcConfigBuilder, AhcWSClient, AhcWSClientConfig, StandaloneAhcWSClient }
+import play.api.libs.ws.{ WSClient, WSConfigParser }
 import play.api.mvc.EssentialFilter
 import play.api.routing.Router
 
@@ -31,7 +31,7 @@ import scala.util.Failure
 class JobApplicationLoader extends ApplicationLoader with StrictLogging {
   private[this] val log = LoggerFactory.getLogger(getClass)
 
-  def load(context: Context): Application = {
+  def load(context: Context): Application = try {
     val jobComponents = new JobComponents(context)
 
     jobComponents.metricsModule.start(jobComponents.actorSystem)
@@ -44,6 +44,12 @@ class JobApplicationLoader extends ApplicationLoader with StrictLogging {
     })(ExecutionContexts.callerThread)
 
     jobComponents.application
+  } catch {
+    case ex: Throwable =>
+      // something awful
+      logger.error(s"Exception occurred while trying to initialize Metronome. Shutting down", ex)
+      JvmExitsCrashStrategy.crash()
+      throw ex
   }
 }
 
@@ -52,7 +58,7 @@ class JobComponents(context: Context) extends BuiltInComponentsFromContext(conte
   LoggerConfigurator(context.environment.classLoader).foreach {
     _.configure(context.environment)
   }
-  lazy val clock: Clock = Clock.systemUTC()
+  val clock: Clock = Clock.systemUTC()
 
   override lazy val httpErrorHandler = new ErrorHandler
 
@@ -71,7 +77,7 @@ class JobComponents(context: Context) extends BuiltInComponentsFromContext(conte
     jobsModule.actorsModule,
     metricsModule)
 
-  lazy val wsClient: WSClient = {
+  val wsClient: WSClient = {
     val parser = new WSConfigParser(configuration.underlying, environment.classLoader)
     val config = AhcWSClientConfig(wsClientConfig = parser.parse())
     val builder = new AhcConfigBuilder(config)
@@ -87,7 +93,7 @@ class JobComponents(context: Context) extends BuiltInComponentsFromContext(conte
     new AhcWSClient(new StandaloneAhcWSClient(asyncHttpClient)(jobsModule.actorsModule.materializer))
   }
 
-  override lazy val httpFilters: Seq[EssentialFilter] = Seq(
+  override val httpFilters: Seq[EssentialFilter] = Seq(
     new LeaderProxyFilter(wsClient, jobsModule.schedulerModule.electionService, config))
 
   override def router: Router = apiModule.router