Add some tools to use docker from gradle (deephaven#559)

.github/workflows/check-ci.yml

@@ -40,4 +40,4 @@ jobs:
         if: always()
         with:
           name: check-ci-results
-          path: '**/build/test-results/test/TEST-*.xml'
+          path: '**/build/test-results/*/TEST-*.xml'

.github/workflows/long-check-ci.yml

@@ -39,4 +39,4 @@ jobs:
         if: always()
         with:
           name: long-check-ci-results
-          path: '**/build/test-results/test/TEST-*.xml'
+          path: '**/build/test-results/*/TEST-*.xml'

.github/workflows/nightly-check-ci.yml

@@ -44,4 +44,4 @@ jobs:
         with:
           check_name: nightly-check-ci test report
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          report_paths: '**/build/test-results/test/TEST-*.xml'
+          report_paths: '**/build/test-results/*/TEST-*.xml'

Generators/Generators.gradle

@@ -1,8 +1,14 @@
+plugins {
+    id 'com.bmuschko.docker-remote-api'
+}
+
 configurations {
     compile.extendsFrom fishConfig, jdom
     testCompile.extendsFrom fishBaseTest
 }
 
+evaluationDependsOn(':deephaven-jpy')
+
 dependencies {
 
     compile project(':DB'),
@@ -151,6 +157,70 @@ if (PyEnv.pythonEnabled(project)) {
     venv.pythonExec(project, "generatePyDoc", ["javadocExtraction.py", devRoot, "false"], [":allJavadoc", venv.taskNameCreateVenv], "${project.projectDir}/python")
     venv.pythonExec(project, "generateManPyDocAssert", ["manualDocstrings.py", devRoot, "true"], venv.taskNameCreateVenv, "${project.projectDir}/python")
     venv.pythonExec(project, "generateManPyDoc", ["manualDocstrings.py", devRoot, "false"], venv.taskNameCreateVenv, "${project.projectDir}/python")
+} else {
+    TaskProvider<Task> generatePyDoc = Docker.registerDockerTask(project, "generatePyDoc") {
+        copyIn {
+            from('python') {
+                into 'python'
+            }
+
+            // TODO move these to a later sync task, so that we detect changes and minimally build images
+            // Copy existing generated JSON to where the image can get it - remove if we stop checking in JSON
+            from("$devRoot/Integrations/python/deephaven") {
+                // allow javadocExtraction.py to verify existing json
+                include 'doc'
+                // allow manualDocstrings.py to verify existing json
+                include 'docCustom'
+                into 'out'
+            }
+            // Depend on generated javadoc so we can generate JSON from it
+            dependsOn ':allJavadoc'
+            from("$devRoot/build/docs/javadoc") {
+                // input for javadocExtraction.py
+                into 'javadoc'
+            }
+        }
+
+        dockerfile {
+            from 'docker.io/library/python:3.7.10'
+
+            runCommand 'pip3 install beautifulsoup4==4.9.3 lxml==4.6.3'
+
+            copyFile('.', '.')
+
+            runCommand '''set -eux; \\
+                          python3 /python/javadocExtraction.py / false; \\
+                          python3 /python/manualDocstrings.py / false
+                          '''
+        }
+
+        parentContainers = [project(':deephaven-jpy').tasks.findByName("buildDockerForRuntime")]// deephaven/java-and-python
+        imageName = 'deephaven/pydocs'
+        copyOut {
+            into("$devRoot/Integrations/python/deephaven")
+            preserve {// only overwrite doc and docCustom directories
+                include '**/*'
+                exclude 'doc'
+                exclude 'docCustom'
+            }
+        }
+    }
+
+    tasks.register("generateManPyDoc") {
+        // TODO core#430 remove this task when BUILD_PY disappears
+        // for now, this task just makes sure that the main task ran
+        dependsOn generatePyDoc
+    }
+    tasks.register("generateManPyDocAssert") {
+        doLast {
+            logger.warn "This task no longer checks output, we're expecting to always regenerate now"
+        }
+    }
+    tasks.register("generatePyDocAssert") {
+        doLast {
+            logger.warn "This task no longer checks output, we're expecting to always regenerate now"
+        }
+    }
 }
 
 
@@ -188,9 +258,7 @@ task generatePythonIntegrationStaticMethods(type: JavaExec, dependsOn: 'classes'
     systemProperty 'workspace', workspace
     systemProperty 'devroot', devRoot
     systemProperty 'Configuration.rootFile', 'dh-defaults.prop'
-    if (PyEnv.pythonEnabled(project)) {
-        dependsOn ':Generators:generatePyDoc', ':Generators:generateManPyDoc'
-    }
+    dependsOn ':Generators:generatePyDoc', ':Generators:generateManPyDoc'
 }
 
 task generatePythonIntegrationStaticMethodsAssert(type: JavaExec, dependsOn: 'classes') {
@@ -217,9 +285,7 @@ task generatePythonFigureWrapper(type: JavaExec, dependsOn: 'classes') {
     systemProperty 'workspace', workspace
     systemProperty 'devroot', devRoot
     systemProperty 'Configuration.rootFile', 'dh-defaults.prop'
-    if (PyEnv.pythonEnabled(project)) {
-        dependsOn ':Generators:generatePyDoc', ':Generators:generateManPyDoc', ':Generators:generateFigureImmutable'
-    }
+    dependsOn ':Generators:generatePyDoc', ':Generators:generateFigureImmutable'
 }
 
 task generatePythonFigureWrapperAssert(type: JavaExec, dependsOn: 'classes') {

Generators/python/javadocExtraction.py

@@ -707,8 +707,8 @@ def methodDigest(methodDetailList, details, requiredModifiers={'public'}, maxCou
         if assert_t in ['true', 't', '1']:
             assertNoChange = True
 
-    docRoot = os.path.join(devRoot, 'build', 'docs', 'javadoc')
-    outDir = os.path.join(devRoot, 'Integrations', 'python', 'deephaven', 'doc')
+    docRoot = os.path.join(devRoot, 'javadoc')
+    outDir = os.path.join(devRoot, 'out', 'doc')
 
     # junk any contents of outDir, if it exists - it's easier than trying to sync somehow
     if (not assertNoChange) and os.path.exists(outDir):
@@ -718,7 +718,7 @@ def methodDigest(methodDetailList, details, requiredModifiers={'public'}, maxCou
     currentDocs = populateCurrentDocs(outDir)
 
     # walk down the com directory of docRoot, and find all the html files
-    for root, dirs, files in os.walk(os.path.join(docRoot, 'com')):
+    for root, dirs, files in os.walk(os.path.join(docRoot, 'io')):
         for fil in files:
             fstem, fext = os.path.splitext(fil)
             if (fstem[0] == '.') or (fext != '.html') or (fstem.startswith('package-')):

Generators/python/manualDocstrings.py

@@ -161,8 +161,8 @@
               "           **NO ACTUAL GENERATION HERE**")
 
     # Parse the arguments
-    devRoot = sys.argv[1]  # devRoot = '/Users/thomasmccullough/workspace/iris'
-    outDir = os.path.join(devRoot, 'Integrations', 'python', 'deephaven', 'docCustom')
+    devRoot = sys.argv[1]
+    outDir = os.path.join(devRoot, 'out', 'docCustom')
 
     assertNoChange = False
     if len(sys.argv) > 2:

Integrations/build.gradle

@@ -1,4 +1,4 @@
-import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+import com.bmuschko.gradle.docker.tasks.image.Dockerfile
 
 plugins {
     id 'com.bmuschko.docker-remote-api'
@@ -14,16 +14,11 @@ configurations {
 dependencies {
 
     compile project(':DB'), project(':DbTypes')
-
+    testRuntimeOnly project(':log-to-slf4j')
     // add configs, and some runtime dependencies to test classpaths
     testRuntime project(':configs'), project(':test-configs'),
             // need these projects so session default imports / generated type wrappers resolve correctly
             project(':Numerics'), project(':Plot')
-
-            // it's not clear why this isn't automatically hooked up; there _is_ no test java sources,
-            // but the main java source output _should_ be automatically applied.
-            sourceSets.main.output
-
 }
 
 sourceSets {
@@ -69,6 +64,47 @@ idea {
         }
 }
 
+task prepareDockerForWheel(type: Sync) {
+    from 'python'
+    into layout.buildDirectory.dir('docker')
+}
+def buildDockerForWheel = Docker.registerDockerImage(project, 'buildDockerForWheel') {
+    dependsOn prepareDockerForWheel
+    buildArgs.put('DEEPHAVEN_VERSION', "${project.version}")
+    images.add('deephaven/deephaven-wheel')
+    target.set('build')
+}
+
+/**
+ * Produce a docker image with the prereqs to run and use deephaven python
+ */
+task deephavenPythonDockerfile(type: Dockerfile) {
+    destFile.set layout.buildDirectory.file('deephaven-python-docker/Dockerfile')
+    from new Dockerfile.From('deephaven/deephaven-jpy-wheel').withStage('deephaven-jpy-wheel')
+    from new Dockerfile.From('deephaven/deephaven-wheel').withStage('deephaven-wheel')
+    from new Dockerfile.From('deephaven/java-and-python')
+
+    copyFile(new Dockerfile.CopyFile('/usr/src/app/dist', '.').withStage('deephaven-jpy-wheel'))
+    copyFile(new Dockerfile.CopyFile('/usr/src/app/dist', '.').withStage('deephaven-wheel'))
+
+    runCommand('''set -eux;\\
+                  pip3 install setuptools wheel; \\
+                  pip3 install *.whl; \\
+                  rm *.whl
+                  ''')
+}
+Docker.registerDockerImage(project, 'buildDeephavenPython') {
+    def dhJpyTasks = project(':deephaven-jpy').tasks
+    inputs.files dhJpyTasks.getByName('buildDockerForRuntime').outputs.files,
+            dhJpyTasks.getByName('buildDockerForWheel').outputs.files,
+            buildDockerForWheel.get().outputs.files,
+            deephavenPythonDockerfile.outputs.files
+
+    inputDir.set layout.buildDirectory.dir('deephaven-python-docker')
+
+    images.add('deephaven/runtime-base')
+}
+
 /**
  * We are not yet enabling the building of jpy on all environments yet.
  * It requires the ability of the running machine to build/install python:
@@ -91,16 +127,57 @@ if (PyEnv.pythonEnabled(project)) {
         venvTest.pythonTest(project, "test-${pv.name}")
     }
 
-}
-
-task prepareDockerForWheel(type: Sync) {
-    from 'python'
-    into "${buildDir}/docker"
-}
+} else {
+    JavaPluginConvention java = project.convention.plugins.get('java') as JavaPluginConvention
+    SourceSet test = java.sourceSets.maybeCreate('test')
+
+    def pyTest = Docker.registerDockerTask(project, "test-py-37") {
+        copyIn {
+            from('python') {
+                into 'python'
+            }
+            from(test.runtimeClasspath) {
+                into 'classpath'
+            }
+
+            // Unpack the config contents for now, since we don't seem to read the configs from inside a jar.
+            // This does not add a task dependency, but we already put :configs in the testRuntime classpath,
+            // so it is part of the previous statement
+            from(zipTree(project(':configs').tasks.getByName('jar').outputs.files.singleFile)) {
+                into 'python/configs'
+            }
+        }
+        parentContainers = [tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+
+        dockerfile {
+            // set up the container, env vars - things that aren't likely to change
+            from 'deephaven/runtime-base'
+            runCommand '''set -eux; \\
+                          pip3 install unittest-xml-reporting==3.0.4;\\
+                          mkdir -p /out/report;\\
+                          mkdir -p /workspace/cache/classes'''
+            environmentVariable 'DEEPHAVEN_CLASSPATH', '/classpath/*:/classpath'
+            environmentVariable 'DEEPHAVEN_WORKSPACE', '/workspace'
+            environmentVariable 'DEEPHAVEN_DEVROOT', '/python'
+            environmentVariable 'DEEPHAVEN_PROPFILE', 'dh-defaults.prop'
+            environmentVariable 'JDK_HOME', '/usr/lib/jvm/zulu8/jre/'
+            environmentVariable 'JAVA_VERSION', '1.8'
+            environmentVariable 'DEEPHAVEN_VERSION', project.version
+
+            workingDir '/python'
+
+            // copy in the contents that we do expect to change as the project updates
+            copyFile 'python', '/python'
+            copyFile 'classpath', '/classpath'
+        }
+        entrypoint = ['python3', '-m', 'xmlrunner', 'discover', '-v', '-o', '/out/report']
 
-task buildDockerForWheel(type: DockerBuildImage) {
-    dependsOn prepareDockerForWheel
-    buildArgs.put('DEEPHAVEN_VERSION', "${project.version}")
-    images.add('deephaven/deephaven-wheel')
-    target.set('build')
+        copyOut {
+            into layout.buildDirectory.dir('test-results')
+        }
+    }
+    pyTest.configure({
+        onlyIf { TestTools.shouldRunTests(project) }
+    })
+    tasks.getByName('test').dependsOn(pyTest)
 }