From b39e3de8eccbda6182ff35c0a2a21700e98846d8 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Mon, 23 Sep 2024 11:12:32 +0900 Subject: [PATCH 1/2] init order --- .../main/scala/org/apache/spark/sql/delta/DeltaConfig.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala index cc0206e486..31fb22379e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala @@ -535,7 +535,10 @@ trait DeltaConfigsBase extends DeltaLogging { * (e.g., in append and OPTIMIZE) as well as data skipping (e.g., the column stats beyond this * number will be ignored even when they exist). */ - val DATA_SKIPPING_NUM_INDEXED_COLS = buildConfig[Int]( + lazy val DATA_SKIPPING_NUM_INDEXED_COLS = buildConfig[Int]( + // This is lazy because of the initialization order. + // DeltaConfig -> DataSkippingReader -> Column.apply -> SQLConf.get + // -> SparkSession -> SqlGatewayConfig -> DeltaConfig "dataSkippingNumIndexedCols", DataSkippingReader.DATA_SKIPPING_NUM_INDEXED_COLS_DEFAULT_VALUE.toString, _.toInt, From c8a7324c5dd47b8fa394439fb1ce35414508a30c Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 25 Sep 2024 12:51:21 +0900 Subject: [PATCH 2/2] Update spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala --- .../main/scala/org/apache/spark/sql/delta/DeltaConfig.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala index 31fb22379e..6e45d18791 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala @@ -536,9 +536,8 @@ trait DeltaConfigsBase extends DeltaLogging { * number will be ignored even when they exist). */ lazy val DATA_SKIPPING_NUM_INDEXED_COLS = buildConfig[Int]( - // This is lazy because of the initialization order. - // DeltaConfig -> DataSkippingReader -> Column.apply -> SQLConf.get - // -> SparkSession -> SqlGatewayConfig -> DeltaConfig + // It involves loading additional context in DataSkippingReader so + // delay its initialization as much as possible "dataSkippingNumIndexedCols", DataSkippingReader.DATA_SKIPPING_NUM_INDEXED_COLS_DEFAULT_VALUE.toString, _.toInt,