Ama cli docker

ama-cli/build.gradle

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+buildscript {
+    ext.kotlin_version = '1.3.21'
+
+    repositories {
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+        classpath 'org.junit.platform:junit-platform-gradle-plugin:1.1.0'
+    }
+}
+
+plugins {
+    id "org.nosphere.apache.rat" version "0.3.1"
+    id "org.jetbrains.kotlin.jvm" version "1.3.0"
+    id "distribution"
+}
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+
+repositories {
+    maven { url "https://plugins.gradle.org/m2/" }
+    maven { url 'http://repository.jetbrains.com/all' }
+    maven { url "https://jetbrains.jfrog.io/jetbrains/spek-snapshots" }
+    maven { url "http://dl.bintray.com/jetbrains/spek" }
+    maven { url "http://oss.jfrog.org/artifactory/oss-snapshot-local" }
+
+    mavenCentral()
+    jcenter()
+}
+
+dependencies {
+    compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
+    compile "org.jetbrains.kotlin:kotlin-reflect"
+    compile group: 'org.slf4j', name: 'slf4j-api', version: '2.0.0-alpha0'
+    compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '2.0.0-alpha0'
+// https://mvnrepository.com/artifact/io.github.microutils/kotlin-logging
+    compile group: 'io.github.microutils', name: 'kotlin-logging', version: '1.6.26'
+
+
+    compile group: 'org.jetbrains.kotlinx', name: 'kotlinx-coroutines-core', version: '1.1.1'
+    compile group: 'com.github.docker-java', name: 'docker-java', version: '3.1.0-rc-5'
+    testCompile 'org.jetbrains.spek:spek-api:1.1.5'
+    testCompile "org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version"
+    testCompile 'org.apache.curator:curator-test:2.13.0'
+    testRuntime 'org.jetbrains.spek:spek-junit-platform-engine:1.1.5'
+    compile group: 'com.xenomachina', name: 'kotlin-argparser', version: '2.0.7'
+
+    // Spek requires kotlin-reflect, can be omitted if already in the classpath
+    testRuntimeOnly "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
+}
+
+
+compileKotlin{
+    kotlinOptions.jvmTarget = "1.8"
+}
+
+compileTestKotlin {
+    kotlinOptions.jvmTarget = "1.8"
+}

leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/AppMasterArgsParser.kt → ama-cli/settings.gradle

@@ -13,25 +13,4 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- */
-package org.apache.amaterasu.leader.yarn
-
-import org.apache.amaterasu.leader.common.launcher.AmaOpts
-import org.apache.amaterasu.leader.common.launcher.ArgsParser
-import org.apache.amaterasu.leader.common.utilities.MessagingClientUtil
-
-class AppMasterArgsParser: ArgsParser() {
-
-    override fun run() {
-
-        var opts = AmaOpts(repo, branch, env, name, jobId, newJobId, report, home, userName, password)
-
-        val appMaster = ApplicationMaster()
-        appMaster.address = MessagingClientUtil.borkerAddress
-        println("broker address is ${appMaster.address}")
-        appMaster.broker.addConnector(appMaster.address)
-        appMaster.broker.start()
-
-        appMaster.execute(opts)
-    }
-}
+ */

ama-cli/src/main/kotlin/org/apache/amaterasu/ama/cli/container/ContainerHandler.kt

@@ -0,0 +1,81 @@
+package org.apache.amaterasu.ama.cli.container
+
+import com.github.dockerjava.api.DockerClient
+import com.github.dockerjava.api.model.BuildResponseItem
+import com.github.dockerjava.core.DefaultDockerClientConfig
+import com.github.dockerjava.core.DockerClientBuilder
+import com.github.dockerjava.core.command.BuildImageResultCallback
+import com.github.dockerjava.core.command.PushImageResultCallback
+import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory
+import mu.KLogging
+import java.io.File
+
+
+class ContainerHandler(val action : String, val imageName : String = "") {
+  companion object: KLogging()
+
+  private val dockerHost = System.getenv("DOCKER_HOST") ?: "unix:///var/run/docker.sock"
+  private val dockerRegistry = System.getenv("DOCKER_REGISTRY") ?: "127.0.0.1:5000"
+
+  private val actionHandlers = mapOf("BUILD" to ::buildDockerImage,
+                    "PUSH" to ::pushDockerImageToRegistry)
+
+  init {
+    logger.info("Got $action for docker as action")
+    val dockerExec =  buildDockerServerExecutor()
+    val dockerClient = getDockerClient(defaultContainerHandler(), dockerExec)
+    actionHandlers[action]?.invoke(dockerClient)
+
+  }
+
+
+  private fun pushDockerImageToRegistry(dockerClient: DockerClient){
+    dockerClient.pushImageCmd("$dockerRegistry/$imageName").exec(PushImageResultCallback()).awaitCompletion()
+  }
+
+  private fun tagDockerImage(dockerClient: DockerClient, imageId : String) {
+    dockerClient.tagImageCmd(imageId,"$dockerRegistry/$imageName", imageName).exec()
+  }
+
+  private fun buildDockerImage(dockerClient: DockerClient) {
+    val baseDir = File("./")
+    val imageId = dockerClient.buildImageCmd(baseDir).exec(BuildImageCallback()).awaitImageId()
+    tagDockerImage(dockerClient,imageId)
+  }
+
+  private fun buildDockerServerExecutor() : JerseyDockerCmdExecFactory {
+    return JerseyDockerCmdExecFactory()
+        .withReadTimeout(1000)
+        .withConnectTimeout(1000)
+        .withMaxTotalConnections(100)
+        .withMaxPerRouteConnections(10)
+  }
+
+  private fun defaultContainerHandler(): DefaultDockerClientConfig {
+    return DefaultDockerClientConfig.createDefaultConfigBuilder()
+        .withDockerHost(dockerHost)
+        .withDockerTlsVerify(false)
+        .withRegistryUrl(dockerRegistry)
+        .build()
+  }
+
+  private fun getDockerClient(config: DefaultDockerClientConfig, dockerCmdExecFactory: JerseyDockerCmdExecFactory?): DockerClient {
+    return DockerClientBuilder.getInstance(config)
+        .withDockerCmdExecFactory(dockerCmdExecFactory)
+        .build()
+  }
+
+
+
+
+  private class BuildImageCallback : BuildImageResultCallback() {
+    override fun onNext(item: BuildResponseItem) {
+      super.onNext(item)
+    }
+  }
+
+
+
+
+
+}

ama-cli/src/main/kotlin/org/apache/amaterasu/ama/cli/container/DockerImageFile.kt

@@ -0,0 +1,36 @@
+package org.apache.amaterasu.ama.cli.container
+
+import java.io.File
+
+
+fun dockerFile(initilizer: DockerImageFile.() -> Unit) : DockerImageFile {
+  return DockerImageFile().apply(initilizer)
+}
+
+class DockerImageFile {
+  val DOCKER_FILE_NAME = "Dockerfile"
+  var baseImage : String = ""
+  var commandsList : MutableList<String> = mutableListOf()
+  var entrypoint  = mutableListOf<String>()
+  fun from(baseImage : String) {
+    this.baseImage = baseImage
+  }
+
+  fun commands(vararg commands : String){
+    commandsList.addAll(commands)
+  }
+
+  fun entrypoint(vararg command : String) {
+    entrypoint.addAll(command)
+  }
+
+  fun createImageFile(dockerFileName : String = DOCKER_FILE_NAME) {
+    File(dockerFileName).writeText(compileImageString())
+  }
+
+  private fun compileImageString() : String {
+    return "FROM " + baseImage + "\n" +
+        commandsList.joinToString (separator = "\n") + "\n" +
+      "ENTRYPOINT " + entrypoint
+  }
+}

ama-cli/src/main/kotlin/org/apache/amaterasu/ama/cli/main/AmaCliMain.kt

@@ -0,0 +1,60 @@
+package org.apache.amaterasu.ama.cli.main
+
+import com.xenomachina.argparser.ArgParser
+import com.xenomachina.argparser.default
+import com.xenomachina.argparser.mainBody
+import org.apache.amaterasu.ama.cli.container.ContainerHandler
+import org.apache.amaterasu.ama.cli.container.dockerFile
+import java.lang.System.exit
+
+
+fun main( args : Array<String> ): Unit = mainBody("ama-cli") {
+  if (shouldPrintUsage(args)){
+    ArgParser(arrayOf("--help")).parseInto( ::AmaCLIArgs )
+  }
+
+  val parsedArgs = ArgParser(args).parseInto(::AmaCLIArgs)
+  when(parsedArgs.agentAction){
+    "CREATE" -> createContainerImage(parsedArgs)
+    "BUILD", "PUSH"  -> handleDockerCommands(parsedArgs)
+    else -> exit(1)
+  }
+
+
+
+}
+
+private fun createContainerImage(parsedArgs: AmaCLIArgs) {
+  dockerFile {
+    from(parsedArgs.baseimage)
+    commands(parsedArgs.commands)
+    entrypoint(parsedArgs.entrypoint)
+  }.createImageFile()
+}
+
+private fun handleDockerCommands(parsedArgs: AmaCLIArgs){
+  ContainerHandler(parsedArgs.agentAction, parsedArgs.imageName)
+}
+
+
+private fun shouldPrintUsage(args: Array<String>) = args.size < AmaCLIArgs.REQUIRED_ARGS
+
+class AmaCLIArgs(parser: ArgParser) {
+
+  companion object {
+    const val REQUIRED_ARGS = 4
+  }
+
+  val v by parser.flagging("enable verbose mode")
+
+  val baseimage by parser.storing("base docker image")
+
+  val commands by parser.storing("commands to run")
+
+  val entrypoint by parser.storing("entrypoint to the container")
+
+  val agentAction by parser.storing("action to preform [CREATE,BUILD,TAG,PUSH,RUN,COLLECT]")
+
+  val imageName by parser.storing("when building a docker image given a specific name to tag").default("AMA_CLI_PRODUCT")
+}
+

ama-cli/src/main/resources/log4j.properties

@@ -0,0 +1,17 @@
+log4j.rootLogger=debug, stdout, R
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+# Pattern to output the caller's file name and line number.
+log4j.appender.stdout.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n
+
+log4j.appender.R=org.apache.log4j.RollingFileAppender
+log4j.appender.R.File=example.log
+
+log4j.appender.R.MaxFileSize=100KB
+# Keep one backup file
+log4j.appender.R.MaxBackupIndex=1
+
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n

ama-cli/src/test/kotlin/DockerImageFileTest.kt

@@ -0,0 +1,33 @@
+
+import org.apache.amaterasu.ama.cli.container.dockerFile
+import org.junit.Test
+import java.io.File
+import kotlin.test.assertEquals
+
+class DockerImageFileTest {
+
+
+  @Test
+  fun buildBasicImageTest() {
+    //given
+    val expectedDockerFile = """FROM busybox
+        COPY A to B
+        Volume A/B
+        ENTRYPOINT [java, -jar, somejar.jar, -H]""".trimIndent()
+
+    val dockerFile = dockerFile {
+      from("busybox")
+      commands("COPY A to B", "Volume A/B")
+      entrypoint("java", "-jar", "somejar.jar", "-H")
+    }
+
+    //when
+    dockerFile.createImageFile()
+
+    //then
+    assertEquals(expectedDockerFile, File("Dockerfile").readLines().joinToString(separator = "\n"))
+
+    File("Dockerfile").delete()
+  }
+
+}

build.gradle

@@ -92,6 +92,8 @@ repositories {
 }
 
 dependencies {
+    compile "ch.qos.logback:logback-classic:1.2.3"
+
     compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
 
 

leader-yarn/build.gradle

@@ -72,7 +72,7 @@ dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
     compile "org.jetbrains.kotlin:kotlin-reflect"
     compile group: 'org.jetbrains.kotlinx', name: 'kotlinx-coroutines-core', version: '1.1.1'
-
+    compile group: 'com.github.docker-java', name: 'docker-java', version: '3.1.0-rc-5'
     testCompile 'org.jetbrains.spek:spek-api:1.1.5'
     testCompile "org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version"
     testCompile 'org.apache.curator:curator-test:2.13.0'