Refactor Spark configuration keys in create_spark function

- Updated the configuration keys in the create_spark function within startup.py to use a more concise naming convention, changing 'spark.executor.memory', 'spark.executor.cores', and 'spark.executor.instances' to 'executor_memory', 'executor_cores', and 'executor_instances' respectively.
- Adjusted the corresponding references in the Spark session creation logic to align with the new key names, improving consistency and readability of the configuration handling.

Author: xuwenyihust

docker/notebook/startup.py

@@ -116,9 +116,9 @@ def create_spark(notebook_path=None):
     except Exception as e:
         logger.error(f"Error loading config: {str(e)}. Using defaults.")
         config_json = {
-            'spark.executor.memory': '1g',
-            'spark.executor.cores': 1,
-            'spark.executor.instances': 1
+            'executor_memory': '1g',
+            'executor_cores': 1,
+            'executor_instances': 1
         }
 
     spark = PawMarkSparkSession(
@@ -133,9 +133,9 @@ def create_spark(notebook_path=None):
             .config("spark.eventLog.dir", "/opt/data/spark-events") \
             .config("spark.history.fs.logDirectory", "/opt/data/spark-events") \
             .config("spark.sql.warehouse.dir", "/opt/data/spark-warehouse") \
-            .config("executor.memory", config_json['executor.memory']) \
-            .config("executor.cores", config_json['executor.cores']) \
-            .config("spark.executor.instances", config_json['spark.executor.instances']) \
+            .config("executor.memory", config_json['executor_memory']) \
+            .config("executor.cores", config_json['executor_cores']) \
+            .config("spark.executor.instances", config_json['executor_instances']) \
             .getOrCreate()
         )