From 6e1b6c02c23fc5f3415f4577eb7d66471ab0ba9e Mon Sep 17 00:00:00 2001
From: Ryanne Dolan <ryannedolan@gmail.com>
Date: Mon, 28 Aug 2023 13:06:57 -0500
Subject: [PATCH] Allow multiple additive templates (#49)

* Allow multiple additive templates

* Drop fat application jars

* Bug fix: actually skip when not a KafkaTopic Acl
---
 Dockerfile                                    |  8 ++-
 bin/hoptimator                                |  2 +-
 deploy/hoptimator-operator-deployment.yaml    |  2 +-
 deploy/hoptimator-pod.yaml                    |  2 +-
 .../linkedin/hoptimator/catalog/Resource.java | 52 ++++++++++++-------
 hoptimator-cli-integration/build.gradle       | 37 +++++++++++++
 hoptimator-cli/build.gradle                   | 17 ------
 hoptimator-cli/run.sh                         | 10 ----
 .../linkedin/hoptimator/HoptimatorCliApp.java |  2 +-
 .../kafka/KafkaTopicAclReconciler.java        |  1 +
 hoptimator-operator-integration/build.gradle  | 36 +++++++++++++
 hoptimator-operator/build.gradle              | 30 -----------
 hoptimator-operator/run.sh                    |  9 ----
 .../subscription/SubscriptionReconciler.java  |  9 ++--
 .../linkedin/hoptimator/planner/Pipeline.java | 13 ++---
 settings.gradle                               |  2 +
 16 files changed, 128 insertions(+), 104 deletions(-)
 create mode 100644 hoptimator-cli-integration/build.gradle
 delete mode 100755 hoptimator-cli/run.sh
 create mode 100644 hoptimator-operator-integration/build.gradle
 delete mode 100755 hoptimator-operator/run.sh

diff --git a/Dockerfile b/Dockerfile
index 3fe0c96..41331ef 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,10 +1,8 @@
 FROM eclipse-temurin:18
 WORKDIR /home/
-ADD ./hoptimator-cli/run.sh ./hoptimator
-ADD ./hoptimator-operator/run.sh ./hoptimator-operator
-ADD ./hoptimator-cli/build/libs/hoptimator-cli-all.jar ./hoptimator-cli-all.jar
-ADD ./hoptimator-operator/build/libs/hoptimator-operator-all.jar ./hoptimator-operator-all.jar
+ADD ./hoptimator-cli-integration/build/distributions/hoptimator-cli-integration.tar ./
+ADD ./hoptimator-operator-integration/build/distributions/hoptimator-operator-integration.tar ./
 ADD ./etc/* ./
 ENTRYPOINT ["/bin/sh", "-c"]
-CMD ["./hoptimator -n '' -p '' -u jdbc:calcite:model=model.yaml"]
+CMD ["./hoptimator-cli-integration/bin/hoptimator-cli-integration -n '' -p '' -u jdbc:calcite:model=model.yaml"]
 
diff --git a/bin/hoptimator b/bin/hoptimator
index 46d4602..7fecf4b 100755
--- a/bin/hoptimator
+++ b/bin/hoptimator
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-kubectl exec -it hoptimator -c hoptimator -- ./hoptimator -n "" -p "" -u "jdbc:calcite:model=/etc/config/model.yaml" "$@"
+kubectl exec -it hoptimator -c hoptimator -- ./hoptimator-cli-integration/bin/hoptimator-cli-integration -n "" -p "" -u "jdbc:calcite:model=/etc/config/model.yaml" "$@"
diff --git a/deploy/hoptimator-operator-deployment.yaml b/deploy/hoptimator-operator-deployment.yaml
index 679bf58..8487f4f 100644
--- a/deploy/hoptimator-operator-deployment.yaml
+++ b/deploy/hoptimator-operator-deployment.yaml
@@ -19,7 +19,7 @@ spec:
       - name: hoptimator-operator
         image: docker.io/library/hoptimator
         imagePullPolicy: Never
-        command: ["./hoptimator-operator", "/etc/config/model.yaml"]
+        command: ["./hoptimator-operator-integration/bin/hoptimator-operator-integration", "/etc/config/model.yaml"]
         volumeMounts:
         - name: config-volume
           mountPath: /etc/config
diff --git a/deploy/hoptimator-pod.yaml b/deploy/hoptimator-pod.yaml
index aeb3685..fc6d8fb 100644
--- a/deploy/hoptimator-pod.yaml
+++ b/deploy/hoptimator-pod.yaml
@@ -13,7 +13,7 @@ spec:
       mountPath: /etc/config
     readinessProbe:
       exec:
-        command: ["./hoptimator", "--run=./readiness-probe.sql"]
+        command: ["./hoptimator-cli-integration/bin/hoptimator-cli-integration", "--run=./readiness-probe.sql"]
       timeoutSeconds: 30
   volumes:
   - name: config-volume
diff --git a/hoptimator-catalog/src/main/java/com/linkedin/hoptimator/catalog/Resource.java b/hoptimator-catalog/src/main/java/com/linkedin/hoptimator/catalog/Resource.java
index b09a3cf..fb5c3af 100644
--- a/hoptimator-catalog/src/main/java/com/linkedin/hoptimator/catalog/Resource.java
+++ b/hoptimator-catalog/src/main/java/com/linkedin/hoptimator/catalog/Resource.java
@@ -1,21 +1,24 @@
 package com.linkedin.hoptimator.catalog;
 
+import java.io.InputStream;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.Scanner;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.UUID;
+import java.util.function.Supplier;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
-import java.util.Scanner;
-import java.util.function.Supplier;
-import java.io.InputStream;
 import java.util.stream.Collectors;
+import java.net.URL;
 
 /**
  * Represents something required by a Table.
@@ -96,9 +99,13 @@ public Set<String> keys() {
   }
 
   /** Render this Resource using the given TemplateFactory */
-  public String render(TemplateFactory templateFactory) {
+  public Collection<String> render(TemplateFactory templateFactory) {
     try {
-      return templateFactory.get(this).render(this);
+      List<String> res = new ArrayList<>();
+      for (Template template : templateFactory.find(this)) {
+        res.add(template.render(this));
+      }
+      return res;
     } catch (Exception e) {
       throw new RuntimeException("Error rendering " + template, e);
     }
@@ -306,10 +313,15 @@ public String render(Resource resource) {
 
   /** Locates a Template for a given Resource */
   public interface TemplateFactory {
-    Template get(Resource resource);
+    Collection<Template> find(Resource resource) throws IOException;
+
+    default Collection<String> render(Resource resource) throws IOException {
+      return find(resource).stream().map(x -> x.render(resource))
+        .collect(Collectors.toList());
+    }
   }
  
-  /** Finds a Template for a given Resource by looking for resource files in the classpath. */ 
+  /** Finds Templates for a given Resource by looking for resource files in the classpath. */ 
   public static class SimpleTemplateFactory implements TemplateFactory {
     private final Environment env;
 
@@ -318,20 +330,22 @@ public SimpleTemplateFactory(Environment env) {
     }
 
     @Override
-    public Template get(Resource resource) {
+    public Collection<Template> find(Resource resource) throws IOException {
       String template = resource.template();
-      InputStream in = getClass().getClassLoader().getResourceAsStream(template + ".yaml.template");
-      if (in == null) {
-        throw new IllegalArgumentException("No template '" + template + "' found in jar resources");
-      }
-      StringBuilder sb = new StringBuilder();
-      Scanner scanner = new Scanner(in);
-      scanner.useDelimiter("\n");
-      while (scanner.hasNext()) {
-        sb.append(scanner.next());
-        sb.append("\n");
+      List<Template> res = new ArrayList<>();
+      for (Enumeration<URL> e = getClass().getClassLoader().getResources(template + ".yaml.template");
+          e.hasMoreElements();) {
+        InputStream in = e.nextElement().openStream();
+        StringBuilder sb = new StringBuilder();
+        Scanner scanner = new Scanner(in);
+        scanner.useDelimiter("\n");
+        while (scanner.hasNext()) {
+          sb.append(scanner.next());
+          sb.append("\n");
+        }
+        res.add(new SimpleTemplate(env, sb.toString()));
       }
-      return new SimpleTemplate(env, sb.toString());
+      return res;
     }
   }
 }
diff --git a/hoptimator-cli-integration/build.gradle b/hoptimator-cli-integration/build.gradle
new file mode 100644
index 0000000..2c5ce46
--- /dev/null
+++ b/hoptimator-cli-integration/build.gradle
@@ -0,0 +1,37 @@
+plugins {
+  id 'java'
+  id 'application'
+  id 'idea'
+}
+
+dependencies {
+  implementation project(path: ':hoptimator-cli', configuration: 'shadow')
+
+  // include adapters for integration tests
+  implementation project(':hoptimator-kafka-adapter')
+  implementation project(':hoptimator-mysql-adapter')
+  implementation project(':hoptimator-flink-adapter')
+  implementation libs.flinkCsv
+  implementation libs.flinkConnectorKafka
+  implementation libs.slf4jSimple
+}
+
+application {
+  mainClassName  = 'com.linkedin.hoptimator.HoptimatorCliApp'
+  applicationDefaultJvmArgs = [
+    "--add-opens", "java.base/java.lang=ALL-UNNAMED",
+    "--add-opens", "java.base/java.util=ALL-UNNAMED",
+    "--add-opens", "java.base/java.time=ALL-UNNAMED" ]
+}
+
+java {
+  withJavadocJar()
+  withSourcesJar()
+}
+
+idea {
+  module {
+    downloadJavadoc = true
+    downloadSources = true
+  }
+}
diff --git a/hoptimator-cli/build.gradle b/hoptimator-cli/build.gradle
index 23017d8..cc4a692 100644
--- a/hoptimator-cli/build.gradle
+++ b/hoptimator-cli/build.gradle
@@ -1,27 +1,14 @@
 plugins {
   id 'com.github.johnrengelman.shadow' version '8.1.1'
   id 'java'
-  id 'application'
   id 'idea'
   id 'maven-publish'
 }
 
-configurations {
-  integration.extendsFrom implementation
-}
-
 dependencies {
   implementation project(':hoptimator-planner')
   implementation project(':hoptimator-catalog')
 
-  // include adapters for integration tests
-  integration project(':hoptimator-kafka-adapter')
-  integration project(':hoptimator-mysql-adapter')
-  integration project(':hoptimator-flink-adapter')
-  integration libs.flinkCsv
-  integration libs.flinkConnectorKafka
-  integration libs.slf4jSimple
-
   implementation libs.avro 
   implementation libs.sqlline
   implementation libs.slf4jApi
@@ -87,15 +74,11 @@ java {
 }
 
 shadowJar {
-  configurations = [project.configurations.integration]
-
   // This is required for Flink and Avatica to play nicely
   relocate 'com.google', 'org.apache.flink.calcite.shaded.com.google'
 
   exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA'
   zip64 true
-  manifest.attributes 'Main-Class': 'com.linkedin.hoptimator.HoptimatorCliApp'
-  mainClassName  = 'com.linkedin.hoptimator.HoptimatorCliApp'
   mergeServiceFiles()
 }
 
diff --git a/hoptimator-cli/run.sh b/hoptimator-cli/run.sh
deleted file mode 100755
index a056a78..0000000
--- a/hoptimator-cli/run.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-java \
-  --add-opens java.base/java.lang=ALL-UNNAMED \
-  --add-opens java.base/java.util=ALL-UNNAMED \
-  --add-opens java.base/java.time=ALL-UNNAMED \
-  -classpath "/opt/plugins/*/lib/*:./hoptimator-cli-all.jar" \
-  -Dorg.slf4j.simpleLogger.defaultLogLevel=error \
-  $JAVA_OPTS \
-  com.linkedin.hoptimator.HoptimatorCliApp --verbose=true -nn hoptimator --isolation=TRANSACTION_NONE "$@"
diff --git a/hoptimator-cli/src/main/java/com/linkedin/hoptimator/HoptimatorCliApp.java b/hoptimator-cli/src/main/java/com/linkedin/hoptimator/HoptimatorCliApp.java
index e7585b3..b0702e4 100644
--- a/hoptimator-cli/src/main/java/com/linkedin/hoptimator/HoptimatorCliApp.java
+++ b/hoptimator-cli/src/main/java/com/linkedin/hoptimator/HoptimatorCliApp.java
@@ -214,7 +214,7 @@ public void execute(String line, DispatchCallback dispatchCallback) {
         // TODO provide generated avro schema to environment
         Resource.TemplateFactory templateFactory = new Resource.SimpleTemplateFactory(
           new Resource.SimpleEnvironment(properties).orIgnore());
-        sqlline.output(pipeline.render(templateFactory));
+        pipeline.render(templateFactory).stream().forEach(x -> sqlline.output(x));
         dispatchCallback.setToSuccess();
       } catch (Exception e) {
         sqlline.error(e.toString());
diff --git a/hoptimator-kafka-adapter/src/main/java/com/linkedin/hoptimator/operator/kafka/KafkaTopicAclReconciler.java b/hoptimator-kafka-adapter/src/main/java/com/linkedin/hoptimator/operator/kafka/KafkaTopicAclReconciler.java
index aa643ad..0e8d562 100644
--- a/hoptimator-kafka-adapter/src/main/java/com/linkedin/hoptimator/operator/kafka/KafkaTopicAclReconciler.java
+++ b/hoptimator-kafka-adapter/src/main/java/com/linkedin/hoptimator/operator/kafka/KafkaTopicAclReconciler.java
@@ -62,6 +62,7 @@ public Result reconcile(Request request) {
 
       if (!targetKind.equals("KafkaTopic")) {
         log.info("Not a KafkaTopic Acl. Skipping.");
+        return new Result(false);
       }
 
       V1alpha1AclSpec.MethodEnum method = object.getSpec().getMethod();
diff --git a/hoptimator-operator-integration/build.gradle b/hoptimator-operator-integration/build.gradle
new file mode 100644
index 0000000..5a5435c
--- /dev/null
+++ b/hoptimator-operator-integration/build.gradle
@@ -0,0 +1,36 @@
+plugins {
+  id 'java'
+  id 'application'
+  id 'idea'
+}
+
+dependencies {
+  implementation project(':hoptimator-operator')
+ 
+  // include adapters in integration tests 
+  implementation project(':hoptimator-kafka-adapter')
+  implementation project(':hoptimator-mysql-adapter')
+  implementation project(':hoptimator-flink-adapter')
+  implementation libs.slf4jSimple
+
+  implementation libs.calciteCore
+  implementation libs.kubernetesClient
+  implementation libs.kubernetesExtendedClient
+  implementation libs.slf4jApi
+  implementation libs.commonsCli
+  implementation libs.avro
+  
+  testImplementation libs.junit
+  testImplementation libs.assertj
+}
+
+application {
+  mainClassName  = 'com.linkedin.hoptimator.operator.HoptimatorOperatorApp'
+}
+
+idea {
+  module {
+    downloadJavadoc = true
+    downloadSources = true
+  }
+}
diff --git a/hoptimator-operator/build.gradle b/hoptimator-operator/build.gradle
index 320ff64..0cf2be5 100644
--- a/hoptimator-operator/build.gradle
+++ b/hoptimator-operator/build.gradle
@@ -1,26 +1,14 @@
 plugins {
   id 'java'
-  id 'com.github.johnrengelman.shadow' version '8.1.1'
-  id 'application'
   id 'maven-publish'
   id 'idea'
 }
 
-configurations {
-  integration.extendsFrom implementation
-}
-
 dependencies {
   implementation project(':hoptimator-planner')
   implementation project(':hoptimator-catalog')
   implementation project(':hoptimator-models')
  
-  // include adapters in integration tests 
-  integration project(':hoptimator-kafka-adapter')
-  integration project(':hoptimator-mysql-adapter')
-  integration project(':hoptimator-flink-adapter')
-  integration libs.slf4jSimple
-
   implementation libs.calciteCore
   implementation libs.kubernetesClient
   implementation libs.kubernetesExtendedClient
@@ -32,10 +20,6 @@ dependencies {
   testImplementation libs.assertj
 }
 
-application {
-  mainClassName  = 'com.linkedin.hoptimator.operator.HoptimatorOperatorApp'
-}
-
 publishing {
   repositories {
     maven {
@@ -81,20 +65,6 @@ publishing {
   }
 }
 
-
-shadowJar {
-  configurations = [project.configurations.integration]
-
-  // This is required for Flink and Avatica to play nicely
-  relocate 'com.google', 'org.apache.flink.calcite.shaded.com.google'
-
-  exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA'
-  zip64 true
-  manifest.attributes 'Main-Class': 'com.linkedin.hoptimator.operator.HoptimatorOperatorApp'
-  mainClassName  = 'com.linkedin.hoptimator.operator.HoptimatorOperatorApp'
-  mergeServiceFiles()
-}
-
 idea {
   module {
     downloadJavadoc = true
diff --git a/hoptimator-operator/run.sh b/hoptimator-operator/run.sh
deleted file mode 100755
index 63d51ad..0000000
--- a/hoptimator-operator/run.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-java \
-  --add-opens java.base/java.lang=ALL-UNNAMED \
-  --add-opens java.base/java.util=ALL-UNNAMED \
-  --add-opens java.base/java.time=ALL-UNNAMED \
-  -classpath "/opt/plugins/*/lib/*:./hoptimator-operator-all.jar" \
-  $JAVA_OPTS \
-  com.linkedin.hoptimator.operator.HoptimatorOperatorApp "$@"
diff --git a/hoptimator-operator/src/main/java/com/linkedin/hoptimator/operator/subscription/SubscriptionReconciler.java b/hoptimator-operator/src/main/java/com/linkedin/hoptimator/operator/subscription/SubscriptionReconciler.java
index fc2e772..4ac3e18 100644
--- a/hoptimator-operator/src/main/java/com/linkedin/hoptimator/operator/subscription/SubscriptionReconciler.java
+++ b/hoptimator-operator/src/main/java/com/linkedin/hoptimator/operator/subscription/SubscriptionReconciler.java
@@ -27,6 +27,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -98,21 +99,21 @@ public Result reconcile(Request request) {
    
           // Render resources related to all source tables.
           List<String> upstreamResources = pipeline.upstreamResources().stream()
-            .map(x -> x.render(templateFactory))
+            .flatMap(x -> x.render(templateFactory).stream())
             .collect(Collectors.toList());
 
           // Render the SQL job
-          String sqlJob = pipeline.sqlJob().render(templateFactory);
+          Collection<String> sqlJob = pipeline.sqlJob().render(templateFactory);
 
           // Render resources related to the sink table. For these resources, we pass along any
           // "hints" as part of the environment.
           List<String> downstreamResources = pipeline.downstreamResources().stream()
-            .map(x -> x.render(sinkTemplateFactory))
+            .flatMap(x -> x.render(sinkTemplateFactory).stream())
             .collect(Collectors.toList());
 
           List<String> combined = new ArrayList<>();
           combined.addAll(upstreamResources);
-          combined.add(sqlJob);
+          combined.addAll(sqlJob);
           combined.addAll(downstreamResources);
 
           status.setResources(combined);
diff --git a/hoptimator-planner/src/main/java/com/linkedin/hoptimator/planner/Pipeline.java b/hoptimator-planner/src/main/java/com/linkedin/hoptimator/planner/Pipeline.java
index 9082b4b..f4ff9d0 100644
--- a/hoptimator-planner/src/main/java/com/linkedin/hoptimator/planner/Pipeline.java
+++ b/hoptimator-planner/src/main/java/com/linkedin/hoptimator/planner/Pipeline.java
@@ -5,6 +5,8 @@
 import com.linkedin.hoptimator.catalog.Resource;
 import com.linkedin.hoptimator.catalog.ResourceProvider;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -58,14 +60,13 @@ public Collection<Resource> resources() {
     return resourceProvider.resources(null);
   }
 
-  /** Render all resources as one big YAML stream */
-  public String render(Resource.TemplateFactory templateFactory) {
-    StringBuilder sb = new StringBuilder();
+  /** Render all resources */
+  public Collection<String> render(Resource.TemplateFactory templateFactory) throws IOException {
+    List<String> res = new ArrayList<>();
     for (Resource resource : resources()) {
-      sb.append(templateFactory.get(resource).render(resource));
-      sb.append("\n---\n"); // yaml resource separator
+      res.addAll(templateFactory.render(resource));
     }
-    return sb.toString();
+    return res;
   }
 
   /** Render a graph of resources in mermaid format */
diff --git a/settings.gradle b/settings.gradle
index b4be71c..ce091ca 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -3,12 +3,14 @@ rootProject.name='hoptimator'
 
 include 'hoptimator-catalog'
 include 'hoptimator-cli'
+include 'hoptimator-cli-integration'
 include 'hoptimator-flink-adapter'
 include 'hoptimator-flink-runner'
 include 'hoptimator-kafka-adapter'
 include 'hoptimator-models'
 include 'hoptimator-mysql-adapter'
 include 'hoptimator-operator'
+include 'hoptimator-operator-integration'
 include 'hoptimator-planner'
 
 dependencyResolutionManagement {