[Spark] Fix delta-spark test log4j (#3700)

#### Which Delta project/connector is this regarding?

- [X] Spark
- [ ] Standalone
- [ ] Flink
- [ ] Kernel
- [ ] Other (fill in here)

## Description

#3146 added support for spark
master structured logging, but broke the test logging for delta-spark
against spark 3.5. This PR fixes that.

The broken logging looked like this
(https://github.com/delta-io/delta/actions/runs/10856009436/job/30129811815):

```
ERROR StatusConsoleListener Unable to locate plugin type for JsonTemplateLayout
ERROR StatusConsoleListener Unable to locate plugin for JsonTemplateLayout
ERROR StatusConsoleListener Could not create plugin of type class org.apache.logging.log4j.core.appender.FileAppender for element File: java.lang.NullPointerException
 java.lang.NullPointerException
	at org.apache.logging.log4j.core.config.plugins.visitors.PluginElementVisitor.findNamedNode(PluginElementVisitor.java:104)
```

## How was this patch tested?

GitHub CI tests.

## Does this PR introduce _any_ user-facing changes?

No.

build.sbt

@@ -176,6 +176,7 @@ def crossSparkSettings(): Seq[Setting[_]] = getSparkVersion() match {
     Compile / unmanagedSourceDirectories += (Compile / baseDirectory).value / "src" / "main" / "scala-spark-3.5",
     Test / unmanagedSourceDirectories += (Test / baseDirectory).value / "src" / "test" / "scala-spark-3.5",
     Antlr4 / antlr4Version := "4.9.3",
+    Test / javaOptions ++= Seq("-Dlog4j.configurationFile=log4j2.properties"),
 
     // Java-/Scala-/Uni-Doc Settings
     scalacOptions ++= Seq(
@@ -204,8 +205,9 @@ def crossSparkSettings(): Seq[Setting[_]] = getSparkVersion() match {
       "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
       "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
       "--add-opens=java.base/sun.security.action=ALL-UNNAMED",
-      "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
-    )
+      "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
+      "-Dlog4j.configurationFile=log4j2_spark_master.properties"
+    ),
 
     // Java-/Scala-/Uni-Doc Settings
     // This isn't working yet against Spark Master.

spark/src/test/resources/log4j2.properties

@@ -38,26 +38,13 @@ appender.file.append = true
 appender.file.layout.type = PatternLayout
 appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
 
-# Structured Logging Appender
-appender.structured.type = File
-appender.structured.name = structured
-appender.structured.fileName = target/structured.log
-appender.structured.layout.type = JsonTemplateLayout
-appender.structured.layout.eventTemplateUri = classpath:org/apache/spark/SparkLayout.json
-
 # Pattern Logging Appender
 appender.pattern.type = File
 appender.pattern.name = pattern
 appender.pattern.fileName = target/pattern.log
 appender.pattern.layout.type = PatternLayout
 appender.pattern.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n%ex
 
-# Custom logger for testing structured logging with Spark master
-logger.structured_logging.name = org.apache.spark.sql.delta.logging.DeltaStructuredLoggingSuite
-logger.structured_logging.level = trace
-logger.structured_logging.appenderRefs = structured
-logger.structured_logging.appenderRef.structured.ref = structured
-
 # Custom logger for testing structured logging with Spark 3.5 shims
 logger.pattern_logging.name = org.apache.spark.sql.delta.logging.DeltaPatternLoggingSuite
 logger.pattern_logging.level = trace
@@ -76,4 +63,3 @@ appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
 # Ignore messages below warning level from Jetty, because it's a bit verbose
 logger.jetty.name = org.sparkproject.jetty
 logger.jetty.level = warn
-

spark/src/test/resources/log4j2_spark_master.properties

@@ -0,0 +1,65 @@
+#
+#  Copyright (2021) The Delta Lake Project Authors.
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.append = true
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+# Structured Logging Appender
+appender.structured.type = File
+appender.structured.name = structured
+appender.structured.fileName = target/structured.log
+appender.structured.layout.type = JsonTemplateLayout
+appender.structured.layout.eventTemplateUri = classpath:org/apache/spark/SparkLayout.json
+
+# Custom logger for testing structured logging with Spark master
+logger.structured_logging.name = org.apache.spark.sql.delta.logging.DeltaStructuredLoggingSuite
+logger.structured_logging.level = trace
+logger.structured_logging.appenderRefs = structured
+logger.structured_logging.appenderRef.structured.ref = structured
+
+# Tests that launch java subprocesses can set the "test.appender" system property to
+# "console" to avoid having the child process's logs overwrite the unit test's
+# log file.
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+# Ignore messages below warning level from Jetty, because it's a bit verbose
+logger.jetty.name = org.sparkproject.jetty
+logger.jetty.level = warn