diff --git a/api/src/main/resources/public/api/v0/schema/jobspec_v0.schema.json b/api/src/main/resources/public/api/v0/schema/jobspec_v0.schema.json index a7403261..2111a0e5 100644 --- a/api/src/main/resources/public/api/v0/schema/jobspec_v0.schema.json +++ b/api/src/main/resources/public/api/v0/schema/jobspec_v0.schema.json @@ -99,7 +99,17 @@ "patternProperties": { ".*": { "oneOf": [ - { "type": "string" } + { "type": "string" }, + { + "type": "object", + "description": "An environment variable set to a secret", + "properties": { + "secret": { + "type": "string", + "documentation": "The name of the secret to refer to. At runtime, the value of the secret will be injected into the value of the variable." + } + } + } ] } } @@ -191,6 +201,21 @@ }, "required": ["containerPath", "hostPath", "mode"] } + }, + "secrets": { + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "description": "An environment variable set to a secret", + "properties": { + "source": { + "type": "string", + "documentation": "The name of the secret to refer to. At runtime, the value of the secret will be injected into the value of the variable." + } + } + } + } } }, "required": ["cpus", "mem", "disk"] diff --git a/api/src/main/resources/public/api/v1/schema/jobspec.schema.json b/api/src/main/resources/public/api/v1/schema/jobspec.schema.json index 99c87dab..7860a24b 100644 --- a/api/src/main/resources/public/api/v1/schema/jobspec.schema.json +++ b/api/src/main/resources/public/api/v1/schema/jobspec.schema.json @@ -100,7 +100,18 @@ "patternProperties": { ".*": { "oneOf": [ - { "type": "string" } + { "type": "string" }, + { + "type": "object", + "description": "An environment variable set to a secret", + "properties": { + "secret": { + "type": "string", + "documentation": "The name of the secret to refer to. At runtime, the value of the secret will be injected into the value of the variable." + } + } + } + ] } } @@ -197,6 +208,21 @@ }, "required": ["containerPath", "hostPath", "mode"] } + }, + "secrets": { + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "description": "An environment variable set to a secret", + "properties": { + "source": { + "type": "string", + "documentation": "The name of the secret to refer to. At runtime, the value of the secret will be injected into the value of the variable." + } + } + } + } } }, "required": ["cpus", "mem", "disk"] diff --git a/api/src/main/scala/dcos/metronome/api/v1/models/package.scala b/api/src/main/scala/dcos/metronome/api/v1/models/package.scala index b3df5321..28d86a40 100644 --- a/api/src/main/scala/dcos/metronome/api/v1/models/package.scala +++ b/api/src/main/scala/dcos/metronome/api/v1/models/package.scala @@ -153,14 +153,15 @@ package object models { (__ \ "cmd").formatNullable[String] ~ (__ \ "args").formatNullable[Seq[String]] ~ (__ \ "user").formatNullable[String] ~ - (__ \ "env").formatNullable[Map[String, String]].withDefault(JobRunSpec.DefaultEnv) ~ + (__ \ "env").formatNullable[Map[String, EnvVarValueOrSecret]].withDefault(JobRunSpec.DefaultEnv) ~ (__ \ "placement").formatNullable[PlacementSpec].withDefault(JobRunSpec.DefaultPlacement) ~ (__ \ "artifacts").formatNullable[Seq[Artifact]].withDefault(JobRunSpec.DefaultArtifacts) ~ (__ \ "maxLaunchDelay").formatNullable[Duration].withDefault(JobRunSpec.DefaultMaxLaunchDelay) ~ (__ \ "docker").formatNullable[DockerSpec] ~ (__ \ "volumes").formatNullable[Seq[Volume]].withDefault(JobRunSpec.DefaultVolumes) ~ (__ \ "restart").formatNullable[RestartSpec].withDefault(JobRunSpec.DefaultRestartSpec) ~ - (__ \ "taskKillGracePeriodSeconds").formatNullable[FiniteDuration])(JobRunSpec.apply, unlift(JobRunSpec.unapply)) + (__ \ "taskKillGracePeriodSeconds").formatNullable[FiniteDuration] ~ + (__ \ "secrets").formatNullable[Map[String, SecretDef]].withDefault(JobRunSpec.DefaultSecrets))(JobRunSpec.apply, unlift(JobRunSpec.unapply)) implicit lazy val JobSpecFormat: Format[JobSpec] = ( (__ \ "id").format[JobId] ~ diff --git a/api/src/test/scala/dcos/metronome/api/v1/controllers/JobSpecControllerTest.scala b/api/src/test/scala/dcos/metronome/api/v1/controllers/JobSpecControllerTest.scala index 3bae6a15..faaed598 100644 --- a/api/src/test/scala/dcos/metronome/api/v1/controllers/JobSpecControllerTest.scala +++ b/api/src/test/scala/dcos/metronome/api/v1/controllers/JobSpecControllerTest.scala @@ -141,6 +141,42 @@ class JobSpecControllerTest extends PlaySpec with OneAppPerTestWithComponents[Mo Then("a forbidden response is send") status(unauthorized) mustBe FORBIDDEN } + + "create a job with secrets" in { + Given("No job") + + When("A job is created") + val response = route(app, FakeRequest(POST, s"/v1/jobs").withJsonBody(jobSpecWithSecretsJson)).get + + Then("The job is created") + status(response) mustBe CREATED + contentType(response) mustBe Some("application/json") + contentAsJson(response) mustBe jobSpecWithSecretsJson + } + + "indicate a problem when creating a job without a secret definition" in { + Given("No job") + + When("A job is created") + val response = route(app, FakeRequest(POST, s"/v1/jobs").withJsonBody(jobSpecWithSecretVarsOnlyJson)).get + + Then("A validation error is returned") + status(response) mustBe UNPROCESSABLE_ENTITY + contentType(response) mustBe Some("application/json") + // contentAsJson(response) \ "message" mustBe JsDefined(JsString("Object is not valid")) + } + + "indicate a problem when creating a job without a secret name" in { + Given("No job") + + When("A job is created") + val response = route(app, FakeRequest(POST, s"/v1/jobs").withJsonBody(jobSpecWithSecretDefsOnlyJson)).get + + Then("A validation error is returned") + status(response) mustBe UNPROCESSABLE_ENTITY + contentType(response) mustBe Some("application/json") + // contentAsJson(response) \ "message" mustBe JsDefined(JsString("Object is not valid")) + } } "GET /jobs" should { @@ -380,6 +416,21 @@ class JobSpecControllerTest extends PlaySpec with OneAppPerTestWithComponents[Mo val jobSpec1Json = Json.toJson(jobSpec1) val jobSpec2 = spec("spec2") val jobSpec2Json = Json.toJson(jobSpec2) + val jobSpecWithSecrets = { + val jobSpec = spec("spec-with-secrets") + jobSpec.copy(run = jobSpec.run.copy(env = Map("secretVar" -> EnvVarSecret("secretId")), secrets = Map("secretId" -> SecretDef("source")))) + } + val jobSpecWithSecretVarsOnly = { + val jobSpec = spec("spec-with-secret-vars-only") + jobSpec.copy(run = jobSpec.run.copy(env = Map("secretVar" -> EnvVarSecret("secretId")))) + } + val jobSpecWithSecretDefsOnly = { + val jobSpec = spec("spec-with-secret-defs-only") + jobSpec.copy(run = jobSpec.run.copy(secrets = Map("secretId" -> SecretDef("source")))) + } + val jobSpecWithSecretsJson = Json.toJson(jobSpecWithSecrets) + val jobSpecWithSecretVarsOnlyJson = Json.toJson(jobSpecWithSecretVarsOnly) + val jobSpecWithSecretDefsOnlyJson = Json.toJson(jobSpecWithSecretDefsOnly) val auth = new TestAuthFixture before { diff --git a/jobs/src/main/protobuf/metronome.proto b/jobs/src/main/protobuf/metronome.proto index a3c49790..7db0c916 100644 --- a/jobs/src/main/protobuf/metronome.proto +++ b/jobs/src/main/protobuf/metronome.proto @@ -190,6 +190,21 @@ message JobSpec { // https://github.com/mesosphere/marathon/blob/releases/1.3/docs/docs/rest-api/public/api/v2/schema/AppDefinition.json // int64 in mesos and int32 in marathon... going with int64 optional int64 task_kill_grace_period_seconds = 14; + + // key value pair used to store secrets path + message EnvironmentVariableSecret { + optional string name = 1; + optional string secretId = 2; + } + repeated EnvironmentVariableSecret environmentSecrets = 15; + + // key value pair used to store secrets path + message Secret { + optional string id = 1; + optional string source = 2; + } + repeated Secret secrets = 16; + } optional RunSpec run = 6; } diff --git a/jobs/src/main/scala/dcos/metronome/model/EnvVarValueOrSecret.scala b/jobs/src/main/scala/dcos/metronome/model/EnvVarValueOrSecret.scala new file mode 100644 index 00000000..82be3271 --- /dev/null +++ b/jobs/src/main/scala/dcos/metronome/model/EnvVarValueOrSecret.scala @@ -0,0 +1,41 @@ +package dcos.metronome.model + +trait EnvVarValueOrSecret + +case class EnvVarValue(value: String) extends EnvVarValueOrSecret + +object EnvVarValue { + implicit object playJsonFormat extends play.api.libs.json.Format[EnvVarValue] { + def reads(json: play.api.libs.json.JsValue): play.api.libs.json.JsResult[EnvVarValue] = { + json.validate[String].map(EnvVarValue.apply) + } + def writes(envVarValue: EnvVarValue): play.api.libs.json.JsValue = { + play.api.libs.json.JsString(envVarValue.value) + } + } +} + +/** + * An environment variable set to a secret + * @param secret The name of the secret to refer to. At runtime, the value of the + * secret will be injected into the value of the variable. + */ +case class EnvVarSecret(secret: String) extends EnvVarValueOrSecret + +object EnvVarSecret { + implicit val playJsonFormat = play.api.libs.json.Json.format[EnvVarSecret] +} + +object EnvVarValueOrSecret { + implicit object playJsonFormat extends play.api.libs.json.Format[EnvVarValueOrSecret] { + def reads(json: play.api.libs.json.JsValue): play.api.libs.json.JsResult[EnvVarValueOrSecret] = { + json.validate[EnvVarValue].orElse(json.validate[EnvVarSecret]) + } + def writes(envOrSecret: EnvVarValueOrSecret): play.api.libs.json.JsValue = { + envOrSecret match { + case envVarValue: EnvVarValue => play.api.libs.json.Json.toJson(envVarValue)(EnvVarValue.playJsonFormat) + case envVarSecret: EnvVarSecret => play.api.libs.json.Json.toJson(envVarSecret)(EnvVarSecret.playJsonFormat) + } + } + } +} \ No newline at end of file diff --git a/jobs/src/main/scala/dcos/metronome/model/JobRunSpec.scala b/jobs/src/main/scala/dcos/metronome/model/JobRunSpec.scala index 0bb7f180..6f66f521 100644 --- a/jobs/src/main/scala/dcos/metronome/model/JobRunSpec.scala +++ b/jobs/src/main/scala/dcos/metronome/model/JobRunSpec.scala @@ -12,20 +12,21 @@ import scala.collection.immutable._ case class Artifact(uri: String, extract: Boolean = true, executable: Boolean = false, cache: Boolean = false) case class JobRunSpec( - cpus: Double = JobRunSpec.DefaultCpus, - mem: Double = JobRunSpec.DefaultMem, - disk: Double = JobRunSpec.DefaultDisk, - cmd: Option[String] = JobRunSpec.DefaultCmd, - args: Option[Seq[String]] = JobRunSpec.DefaultArgs, - user: Option[String] = JobRunSpec.DefaultUser, - env: Map[String, String] = JobRunSpec.DefaultEnv, - placement: PlacementSpec = JobRunSpec.DefaultPlacement, - artifacts: Seq[Artifact] = JobRunSpec.DefaultArtifacts, - maxLaunchDelay: Duration = JobRunSpec.DefaultMaxLaunchDelay, - docker: Option[DockerSpec] = JobRunSpec.DefaultDocker, - volumes: Seq[Volume] = JobRunSpec.DefaultVolumes, - restart: RestartSpec = JobRunSpec.DefaultRestartSpec, - taskKillGracePeriodSeconds: Option[FiniteDuration] = JobRunSpec.DefaultTaskKillGracePeriodSeconds) + cpus: Double = JobRunSpec.DefaultCpus, + mem: Double = JobRunSpec.DefaultMem, + disk: Double = JobRunSpec.DefaultDisk, + cmd: Option[String] = JobRunSpec.DefaultCmd, + args: Option[Seq[String]] = JobRunSpec.DefaultArgs, + user: Option[String] = JobRunSpec.DefaultUser, + env: Map[String, EnvVarValueOrSecret] = JobRunSpec.DefaultEnv, + placement: PlacementSpec = JobRunSpec.DefaultPlacement, + artifacts: Seq[Artifact] = JobRunSpec.DefaultArtifacts, + maxLaunchDelay: Duration = JobRunSpec.DefaultMaxLaunchDelay, + docker: Option[DockerSpec] = JobRunSpec.DefaultDocker, + volumes: Seq[Volume] = JobRunSpec.DefaultVolumes, + restart: RestartSpec = JobRunSpec.DefaultRestartSpec, + taskKillGracePeriodSeconds: Option[FiniteDuration] = JobRunSpec.DefaultTaskKillGracePeriodSeconds, + secrets: Map[String, SecretDef] = JobRunSpec.DefaultSecrets) object JobRunSpec { val DefaultCpus: Double = 1.0 @@ -36,26 +37,41 @@ object JobRunSpec { val DefaultCmd = None val DefaultArgs = None val DefaultUser = None - val DefaultEnv = Map.empty[String, String] + val DefaultEnv = Map.empty[String, EnvVarValueOrSecret] val DefaultArtifacts = Seq.empty[Artifact] val DefaultDocker = None val DefaultVolumes = Seq.empty[Volume] val DefaultRestartSpec = RestartSpec() val DefaultTaskKillGracePeriodSeconds = None + val DefaultSecrets = Map.empty[String, SecretDef] implicit lazy val validJobRunSpec: Validator[JobRunSpec] = new Validator[JobRunSpec] { import com.wix.accord._ import ViolationBuilder._ override def apply(jobRunSpec: JobRunSpec): Result = { - if (jobRunSpec.cmd.isDefined || jobRunSpec.docker.exists(d => d.image.nonEmpty)) - Success - else - RuleViolation(jobRunSpec, JobRunSpecMessages.cmdOrDockerValidation, None) + var violations = Set.empty[Result] + + def check(test: Boolean, errorMessage: String) = { + if (!test) { + violations += RuleViolation(jobRunSpec, errorMessage, None) + } + } + val envVarDefinedSecretNames = jobRunSpec.env.values.collect { case EnvVarSecret(secretName) => secretName }.toSet + val providedSecretNames = jobRunSpec.secrets.keySet + + check(jobRunSpec.cmd.isDefined || jobRunSpec.docker.exists(d => d.image.nonEmpty), JobRunSpecMessages.cmdOrDockerValidation) + check(envVarDefinedSecretNames == providedSecretNames, JobRunSpecMessages.secretsValidation(envVarDefinedSecretNames, providedSecretNames)) + + violations.headOption.getOrElse(Success) } + } } object JobRunSpecMessages { val cmdOrDockerValidation = "Cmd or docker image must be specified" + def secretsValidation(envVarSecretsName: Set[String], providedSecretsNames: Set[String]) = { + s"Secret names are different from provided secrets. Defined: ${envVarSecretsName.mkString(", ")}, Provided: ${providedSecretsNames.mkString(", ")}" + } } diff --git a/jobs/src/main/scala/dcos/metronome/model/JobSpec.scala b/jobs/src/main/scala/dcos/metronome/model/JobSpec.scala index a3e2c096..83ebedfb 100644 --- a/jobs/src/main/scala/dcos/metronome/model/JobSpec.scala +++ b/jobs/src/main/scala/dcos/metronome/model/JobSpec.scala @@ -4,10 +4,13 @@ package model import com.wix.accord.dsl._ import com.wix.accord.Validator import mesosphere.marathon.api.v2.Validation._ -import mesosphere.marathon.plugin.{ Secret, EnvVarValue, RunSpec } +import mesosphere.marathon.plugin.{ Secret, RunSpec } import scala.collection.immutable._ -import JobRunSpec._ +import dcos.metronome.model.JobRunSpec._ +import dcos.metronome.utils.glue.MarathonConversions +import mesosphere.marathon.api.v2.Validation._ +import mesosphere.marathon case class JobSpec( id: JobId, @@ -17,10 +20,10 @@ case class JobSpec( run: JobRunSpec = JobSpec.DefaultRunSpec) extends RunSpec { def schedule(id: String): Option[ScheduleSpec] = schedules.find(_.id == id) - override def user: Option[String] = run.user - override def acceptedResourceRoles: Option[Predef.Set[String]] = None - override def secrets: Map[String, Secret] = Map.empty - override def env: Map[String, EnvVarValue] = mesosphere.marathon.state.EnvVarValue(run.env) + override val user: Option[String] = run.user + override val acceptedResourceRoles: Option[Predef.Set[String]] = None + override val secrets: Map[String, Secret] = MarathonConversions.secretsToMarathon(run.secrets) + override val env: Map[String, marathon.state.EnvVarValue] = MarathonConversions.envVarToMarathon(run.env) } object JobSpec { diff --git a/jobs/src/main/scala/dcos/metronome/model/QueuedJobRunInfo.scala b/jobs/src/main/scala/dcos/metronome/model/QueuedJobRunInfo.scala index bf1a72a9..849611bc 100644 --- a/jobs/src/main/scala/dcos/metronome/model/QueuedJobRunInfo.scala +++ b/jobs/src/main/scala/dcos/metronome/model/QueuedJobRunInfo.scala @@ -1,7 +1,9 @@ package dcos.metronome package model -import mesosphere.marathon.plugin.{ EnvVarValue, PathId, RunSpec, Secret } +import dcos.metronome.utils.glue.MarathonConversions +import mesosphere.marathon.plugin.{ PathId, Secret, RunSpec } +import mesosphere.marathon.plugin import mesosphere.marathon.state.Timestamp import scala.collection.immutable.Map @@ -17,6 +19,6 @@ case class QueuedJobRunInfo( lazy val runId: String = id.path.last override def user: Option[String] = run.user override def secrets: Map[String, Secret] = Map.empty - override def env: Map[String, EnvVarValue] = mesosphere.marathon.state.EnvVarValue(run.env) - override def labels = Map.empty[String, String] + override val env: Map[String, plugin.EnvVarValue] = MarathonConversions.envVarToMarathon(run.env) + override val labels = Map.empty[String, String] } diff --git a/jobs/src/main/scala/dcos/metronome/model/SecretDef.scala b/jobs/src/main/scala/dcos/metronome/model/SecretDef.scala new file mode 100644 index 00000000..3f7e557b --- /dev/null +++ b/jobs/src/main/scala/dcos/metronome/model/SecretDef.scala @@ -0,0 +1,26 @@ +package dcos.metronome.model + +/** + * A secret declaration + * @param source reference to a secret which will be injected with a value from the secret store. + */ +case class SecretDef(source: String) + +object SecretDef { + import play.api.libs.json.Reads._ + implicit object playJsonFormat extends play.api.libs.json.Format[SecretDef] { + def reads(json: play.api.libs.json.JsValue): play.api.libs.json.JsResult[SecretDef] = { + val source = json.\("source").validate[String](play.api.libs.json.JsPath.read[String](minLength[String](ConstraintSourceMinLength))) + val _errors = Seq(("source", source)).collect({ + case (field, e: play.api.libs.json.JsError) => e.repath(play.api.libs.json.JsPath.\(field)).asInstanceOf[play.api.libs.json.JsError] + }) + if (_errors.nonEmpty) _errors.reduceOption[play.api.libs.json.JsError](_.++(_)).getOrElse(_errors.head) + else play.api.libs.json.JsSuccess(SecretDef(source = source.get)) + } + def writes(secret: SecretDef): play.api.libs.json.JsValue = { + val source = play.api.libs.json.Json.toJson(secret.source) + play.api.libs.json.JsObject(Seq(("source", source)).filter(_._2 != play.api.libs.json.JsNull).++(Seq.empty)) + } + } + val ConstraintSourceMinLength = 1 +} \ No newline at end of file diff --git a/jobs/src/main/scala/dcos/metronome/repository/impl/kv/marshaller/JobSpecMarshaller.scala b/jobs/src/main/scala/dcos/metronome/repository/impl/kv/marshaller/JobSpecMarshaller.scala index 4e3f7e6b..22c7d559 100644 --- a/jobs/src/main/scala/dcos/metronome/repository/impl/kv/marshaller/JobSpecMarshaller.scala +++ b/jobs/src/main/scala/dcos/metronome/repository/impl/kv/marshaller/JobSpecMarshaller.scala @@ -125,9 +125,11 @@ object RunSpecConversions { .setMaxLaunchDelay(runSpec.maxLaunchDelay.toSeconds) .setPlacement(runSpec.placement.toProto) .setRestart(runSpec.restart.toProto) - .addAllEnvironment(runSpec.env.toProto.asJava) + .addAllEnvironment(runSpec.env.toEnvProto.asJava) + .addAllEnvironmentSecrets(runSpec.env.toEnvSecretProto.asJava) .addAllArtifacts(runSpec.artifacts.toProto.asJava) .addAllVolumes(runSpec.volumes.toProto.asJava) + .addAllSecrets(runSpec.secrets.toProto.asJava) runSpec.cmd.foreach(builder.setCmd) runSpec.args.foreach { args => builder.addAllArguments(args.asJava) } @@ -156,14 +158,15 @@ object RunSpecConversions { maxLaunchDelay = runSpec.getMaxLaunchDelay.seconds, placement = runSpec.getPlacement.toModel, restart = runSpec.getRestart.toModel, - env = runSpec.getEnvironmentList.asScala.toModel, + env = runSpec.getEnvironmentList.asScala.toModel ++ runSpec.getEnvironmentSecretsList.asScala.toModel, artifacts = runSpec.getArtifactsList.asScala.toModel, volumes = runSpec.getVolumesList.asScala.toModel, cmd = cmd, args = args, user = user, docker = docker, - taskKillGracePeriodSeconds = taskKillGracePeriodSeconds) + taskKillGracePeriodSeconds = taskKillGracePeriodSeconds, + secrets = runSpec.getSecretsList.asScala.toModel) } } @@ -277,19 +280,48 @@ object RunSpecConversions { def toModel: DockerSpec = DockerSpec(image = dockerSpec.getImage, forcePullImage = dockerSpec.getForcePullImage) } - implicit class EnvironmentToProto(val environment: Map[String, String]) extends AnyVal { - def toProto: Iterable[Protos.JobSpec.RunSpec.EnvironmentVariable] = environment.map { - case (key, value) => + implicit class EnvironmentToProto(val environment: Map[String, EnvVarValueOrSecret]) extends AnyVal { + def toEnvProto: Iterable[Protos.JobSpec.RunSpec.EnvironmentVariable] = environment.collect { + case (key, EnvVarValue(value)) => Protos.JobSpec.RunSpec.EnvironmentVariable.newBuilder() .setKey(key) .setValue(value) .build } + def toEnvSecretProto: Iterable[Protos.JobSpec.RunSpec.EnvironmentVariableSecret] = environment.collect { + case (name, EnvVarSecret(secretId)) => + Protos.JobSpec.RunSpec.EnvironmentVariableSecret.newBuilder() + .setName(name) + .setSecretId(secretId) + .build + } + } + + implicit class SecretsToProto(val secrets: Map[String, SecretDef]) extends AnyVal { + def toProto: Iterable[Protos.JobSpec.RunSpec.Secret] = secrets.map { + case (secretId, SecretDef(source)) => + Protos.JobSpec.RunSpec.Secret.newBuilder() + .setId(secretId) + .setSource(source) + .build() + } } implicit class ProtosToEnvironment(val environmentVariables: mutable.Buffer[Protos.JobSpec.RunSpec.EnvironmentVariable]) extends AnyVal { - def toModel: Map[String, String] = environmentVariables.map { environmentVariable => - environmentVariable.getKey -> environmentVariable.getValue + def toModel: Map[String, EnvVarValueOrSecret] = environmentVariables.map { environmentVariable => + environmentVariable.getKey -> EnvVarValue(environmentVariable.getValue) + }.toMap + } + + implicit class ProtosToEnvironmentSecrets(val environmentSecrets: mutable.Buffer[Protos.JobSpec.RunSpec.EnvironmentVariableSecret]) extends AnyVal { + def toModel: Map[String, EnvVarValueOrSecret] = environmentSecrets.map { environmentSecret => + environmentSecret.getName -> EnvVarSecret(environmentSecret.getSecretId) + }.toMap + } + + implicit class ProtosToSecrets(val secrets: mutable.Buffer[Protos.JobSpec.RunSpec.Secret]) extends AnyVal { + def toModel: Map[String, SecretDef] = secrets.map { secret => + secret.getId -> SecretDef(secret.getSource) }.toMap } diff --git a/jobs/src/main/scala/dcos/metronome/utils/glue/MarathonConversions.scala b/jobs/src/main/scala/dcos/metronome/utils/glue/MarathonConversions.scala new file mode 100644 index 00000000..03eebbb5 --- /dev/null +++ b/jobs/src/main/scala/dcos/metronome/utils/glue/MarathonConversions.scala @@ -0,0 +1,19 @@ +package dcos.metronome.utils.glue + +import dcos.metronome.model.{ EnvVarSecret, EnvVarValue, EnvVarValueOrSecret, SecretDef } +import mesosphere.marathon + +object MarathonConversions { + + def envVarToMarathon(envVars: Map[String, EnvVarValueOrSecret]): Map[String, marathon.state.EnvVarValue] = { + envVars.mapValues { + case EnvVarValue(value) => marathon.state.EnvVarString(value) + case EnvVarSecret(secret) => marathon.state.EnvVarSecretRef(secret) + } + } + + def secretsToMarathon(secrets: Map[String, SecretDef]): Map[String, marathon.state.Secret] = { + secrets.map { case (name, value) => name -> marathon.state.Secret(value.source) } + } + +} diff --git a/jobs/src/main/scala/dcos/metronome/utils/glue/MarathonImplicits.scala b/jobs/src/main/scala/dcos/metronome/utils/glue/MarathonImplicits.scala index 20558bc2..d7e6841f 100644 --- a/jobs/src/main/scala/dcos/metronome/utils/glue/MarathonImplicits.scala +++ b/jobs/src/main/scala/dcos/metronome/utils/glue/MarathonImplicits.scala @@ -4,11 +4,10 @@ package utils.glue import java.util.concurrent.TimeUnit import dcos.metronome.model._ -import dcos.metronome.scheduler.SchedulerConfig import mesosphere.marathon import mesosphere.marathon.core.health.HealthCheck import mesosphere.marathon.core.readiness.ReadinessCheck -import mesosphere.marathon.state.{ AppDefinition, Container, EnvVarValue, FetchUri, PathId, PortDefinition, RunSpec, Secret, UpgradeStrategy } +import mesosphere.marathon.state.{ AppDefinition, Container, FetchUri, PathId, PortDefinition, RunSpec, Secret, UpgradeStrategy } import scala.collection.immutable.Seq import scala.concurrent.duration._ @@ -87,7 +86,7 @@ object MarathonImplicits { cmd = jobSpec.run.cmd, args = jobSpec.run.args, user = jobSpec.run.user, - env = EnvVarValue(jobSpec.run.env), + env = MarathonConversions.envVarToMarathon(jobSpec.run.env), instances = 1, cpus = jobSpec.run.cpus, mem = jobSpec.run.mem, @@ -112,7 +111,7 @@ object MarathonImplicits { ipAddress = None, versionInfo = AppDefinition.VersionInfo.NoVersion, residency = None, - secrets = Map.empty[String, Secret]) + secrets = MarathonConversions.secretsToMarathon(jobSpec.run.secrets)) } } } diff --git a/jobs/src/test/scala/dcos/metronome/repository/impl/kv/marshaller/JobSpecMarshallerTest.scala b/jobs/src/test/scala/dcos/metronome/repository/impl/kv/marshaller/JobSpecMarshallerTest.scala index 87001e51..7f152793 100644 --- a/jobs/src/test/scala/dcos/metronome/repository/impl/kv/marshaller/JobSpecMarshallerTest.scala +++ b/jobs/src/test/scala/dcos/metronome/repository/impl/kv/marshaller/JobSpecMarshallerTest.scala @@ -36,7 +36,7 @@ class JobSpecMarshallerTest extends FunSuite with Matchers { cmd = Some("sleep 500"), args = None, user = Some("root"), - env = Map("key" -> "value"), + env = Map("key" -> EnvVarValue("value")), placement = PlacementSpec(constraints = Seq(ConstraintSpec("hostname", Operator.Eq, Some("localhost")))), artifacts = Seq(Artifact("http://www.foo.bar/file.tar.gz", extract = false, executable = true, cache = true)), maxLaunchDelay = 24.hours, @@ -58,4 +58,4 @@ class JobSpecMarshallerTest extends FunSuite with Matchers { enabled = true)), run = runSpec) } -} +} \ No newline at end of file diff --git a/project/Build.scala b/project/Build.scala index 50486548..2956b5c3 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -54,6 +54,7 @@ object Build extends sbt.Build { Dependency.metrics, Dependency.jsonValidate, Dependency.Test.scalatest, + Dependency.Test.scalaCheck, Dependency.Test.scalatestPlay ) ) @@ -75,7 +76,9 @@ object Build extends sbt.Build { Dependency.akka, Dependency.metrics, Dependency.Test.akkaTestKit, - Dependency.Test.mockito + Dependency.Test.mockito, + Dependency.Test.scalatest, + Dependency.Test.scalaCheck ) ) ) @@ -135,6 +138,7 @@ object Build extends sbt.Build { // Test deps versions val AsyncAwait = "0.9.7" val ScalaTest = "2.2.6" + val ScalaCheck = "1.13.4" val MacWire = "2.2.2" val Marathon = "1.3.13" val Play = "2.5.18" @@ -166,6 +170,7 @@ object Build extends sbt.Build { object Test { val scalatest = "org.scalatest" %% "scalatest" % V.ScalaTest % "test" val scalatestPlay = "org.scalatestplus.play" %% "scalatestplus-play" % "1.5.1" % "test" + val scalaCheck = "org.scalacheck" %% "scalacheck" % V.ScalaCheck % "test" val akkaTestKit = "com.typesafe.akka" %% "akka-testkit" % V.Akka % "test" val mockito = "org.mockito" % "mockito-core" % V.Mockito % "test" } diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 828c75b5..f62a370e 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -365,7 +365,7 @@ metronome { # "vips" (Currently n/a) can be used to enable the networking VIP integration UI. # "task_killing" can be used to enable the TASK_KILLING state in Mesos (0.28 or later) # "external_volumes" (Currently n/a) can be used if the cluster is configured to use external volumes. - # Example: --enable_features vips,task_killing,external_volumes + # Example: --enable_features vips,task_killing,external_volumes,secrets features.enable = ${?METRONOME_FEATURES_ENABLE} # (Optional. Default: "metronome") Framework name used to register with mesos. diff --git a/version.sbt b/version.sbt index 9e6291c5..f78cd177 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.4.1" +version in ThisBuild := "0.4.2"