fix(config): remove spark config defaults

datatok · Nov 16, 2022 · 325cb78 · 325cb78
1 parent 5b32bde
commit 325cb78
Show file tree

Hide file tree

Showing 4 changed files with 2 additions and 52 deletions.
diff --git a/dev/default.conf b/dev/default.conf
@@ -111,7 +111,6 @@ djobi {
             driver {
               allowMultipleContexts = false
               memory = "256M"
-              host = "localhost"
             }
 
             es {

diff --git a/djobi-cli/src/main/release/defaults.conf b/djobi-cli/src/main/release/defaults.conf
@@ -76,40 +76,14 @@ djobi {
 
     executors {
       spark {
-        master = "local"
-
-        webHistoryUrlForJob = "http://localhost:4040"
 
         data {
 
         }
 
         conf {
-          spark {
-            driver {
-              allowMultipleContexts = false
-              memory = "256M"
-              host = "localhost"
-            }
-
-            es {
-              scroll.size = 100
-              nodes.wan.only= true
-              mapping.date.rich = false
-              net {
-                #ssl = true
-                ssl.cert.allow.self.signed = true
-              }
-            }
 
-            sql {
-              shuffle.partitions = 10
-            }
-
-            eventLog.enabled = false
-            ui.enabled = false
-          }
         }
       }
     }
-}
+}
diff --git a/djobi-core/src/main/resources/reference.conf b/djobi-core/src/main/resources/reference.conf
@@ -82,30 +82,7 @@ djobi {
         }
 
         conf {
-          spark {
-            driver {
-              allowMultipleContexts = false
-              memory = "256M"
-              host = "localhost"
-            }
-
-            es {
-              scroll.size = 100
-              nodes.wan.only= true
-              mapping.date.rich = false
-              net {
-                #ssl = true
-                ssl.cert.allow.self.signed = true
-              }
-            }
 
-            sql {
-              shuffle.partitions = 10
-            }
-
-            eventLog.enabled = false
-            ui.enabled = false
-          }
         }
       }
     }

diff --git a/.../src/test/java/io/datatok/djobi/spark/executor/config/SparkExecutorConfigFactoryTest.java b/.../src/test/java/io/datatok/djobi/spark/executor/config/SparkExecutorConfigFactoryTest.java
@@ -86,7 +86,7 @@ void shouldConfigureSparkContext() throws IOException {
 
         final SparkContext context = executor.getSparkContext();
 
-        Assertions.assertEquals("djobi-app", context.appName());
+        Assertions.assertEquals("djobi", context.appName());
 
         Assertions.assertEquals("true", context.getConf().get("spark.es.nodes.wan.only"));
     }