diff --git a/bin/common.sh b/bin/common.sh index 138e23de967..8d2b6a75677 100755 --- a/bin/common.sh +++ b/bin/common.sh @@ -16,9 +16,12 @@ # limitations under the License. # -export SMART_SERVER_LOG_FILE_NAME=smartserver-master-$(hostname)-$(whoami).log -export SMART_STANDBY_LOG_FILE_NAME=smartserver-standby-$(hostname)-$(whoami).log -export SMART_AGENT_LOG_FILE_NAME=smartagent-$(hostname)-$(whoami).log +export SMART_SERVER_LOG_FILE_NAME=smartserver-master-$(hostname)-$(whoami) +export SMART_STANDBY_LOG_FILE_NAME=smartserver-standby-$(hostname)-$(whoami) +export SMART_AGENT_LOG_FILE_NAME=smartagent-$(hostname)-$(whoami) +export LOG_FILE_EXTENSION=.log +export STDOUT_FILE_EXTENSION=.out +export STDERR_FILE_EXTENSION=.err export SMART_LOG_FILE_NAME=${SMART_SERVER_LOG_FILE_NAME} if [ -L ${BASH_SOURCE-$0} ]; then @@ -40,7 +43,6 @@ fi if [[ -z "${SMART_LOG_DIR}" ]]; then export SMART_LOG_DIR="/var/log/ssm" fi -export SMART_LOG_FILE=${SMART_LOG_DIR}/${SMART_LOG_FILE_NAME} if [[ -z "$SMART_PID_DIR" ]]; then export SMART_PID_DIR="${SMART_HOME}/run" @@ -137,7 +139,6 @@ fi # JAVA_OPTS+=" ${SMART_JAVA_OPTS} -Dfile.encoding=${SMART_ENCODING} ${SMART_MEM}" # JAVA_OPTS+=" -Dlog4j.configuration=file://${SMART_CONF_DIR}/log4j.properties" -JAVA_OPTS+=" -Dlog4j2.configurationFile=file://${SMART_CONF_DIR}/log4j.properties" export JAVA_OPTS JAVA_INTP_OPTS="${SMART_INTP_JAVA_OPTS} -Dfile.encoding=${SMART_ENCODING}" @@ -195,7 +196,9 @@ function smart_start_daemon() { rm -f "${pidfile}" >/dev/null 2>&1 fi - start_daemon "${pidfile}" >>${SMART_LOG_FILE} 2>&1 < /dev/null & + SMART_OUT_FILE=${SMART_LOG_DIR}/${SMART_LOG_FILE_NAME}${STDOUT_FILE_EXTENSION} + SMART_ERR_FILE=${SMART_LOG_DIR}/${SMART_LOG_FILE_NAME}${STDERR_FILE_EXTENSION} + start_daemon "${pidfile}" >>${SMART_OUT_FILE} 2>>${SMART_ERR_FILE} < /dev/null & daemon_pid=$! (( counter=0 )) @@ -296,10 +299,11 @@ function init_command() { SMART_CLASSNAME=org.smartdata.server.SmartDaemon SMART_PID_FILE=/tmp/SmartServer.pid ALLOW_DAEMON_OPT=true + export SMART_LOG_FILE_NAME=${SMART_SERVER_LOG_FILE_NAME} if [ $SSM_DEBUG_ENABLED == "true" ]; then JAVA_OPTS+=" -Xdebug -Xrunjdwp:transport=dt_socket,address=8008,server=y,suspend=y" fi - JAVA_OPTS+=" -Dsmart.log.file="${SMART_LOG_FILE_NAME} + JAVA_OPTS+=" -Dsmart.log.file="${SMART_LOG_FILE_NAME}${LOG_FILE_EXTENSION} JAVA_OPTS+=" ${SSM_JAVA_OPT} ${SSM_SERVER_JAVA_OPT}" SMART_VARGS+=" -D smart.agent.master.address="${SSM_EXEC_HOST} reorder_lib @@ -309,11 +313,10 @@ function init_command() { SMART_PID_FILE=/tmp/SmartAgent.pid ALLOW_DAEMON_OPT=true export SMART_LOG_FILE_NAME=${SMART_AGENT_LOG_FILE_NAME} - export SMART_LOG_FILE=${SMART_LOG_DIR}/${SMART_LOG_FILE_NAME} if [ $SSM_DEBUG_ENABLED == "true" ]; then JAVA_OPTS+=" -Xdebug -Xrunjdwp:transport=dt_socket,address=8008,server=y,suspend=y" fi - JAVA_OPTS+=" -Dsmart.log.file="${SMART_LOG_FILE_NAME} + JAVA_OPTS+=" -Dsmart.log.file="${SMART_LOG_FILE_NAME}${LOG_FILE_EXTENSION} JAVA_OPTS+=" ${SSM_JAVA_OPT} ${SSM_AGENT_JAVA_OPT}" SMART_VARGS+=" -D smart.agent.address="${SSM_EXEC_HOST} ;; @@ -322,11 +325,10 @@ function init_command() { SMART_PID_FILE=/tmp/SmartServer.pid ALLOW_DAEMON_OPT=true export SMART_LOG_FILE_NAME=${SMART_STANDBY_LOG_FILE_NAME} - export SMART_LOG_FILE=${SMART_LOG_DIR}/${SMART_LOG_FILE_NAME} if [ $SSM_DEBUG_ENABLED == "true" ]; then JAVA_OPTS+=" -Xdebug -Xrunjdwp:transport=dt_socket,address=8008,server=y,suspend=y" fi - JAVA_OPTS+=" -Dsmart.log.file="${SMART_LOG_FILE_NAME} + JAVA_OPTS+=" -Dsmart.log.file="${SMART_LOG_FILE_NAME}${LOG_FILE_EXTENSION} JAVA_OPTS+=" ${SSM_JAVA_OPT} ${SSM_SERVER_JAVA_OPT}" SMART_VARGS+=" -D smart.agent.master.address="${SSM_EXEC_HOST} reorder_lib diff --git a/conf/log4j.properties b/conf/log4j2.properties similarity index 73% rename from conf/log4j.properties rename to conf/log4j2.properties index 46cc03d4fcb..9d6136a6f86 100644 --- a/conf/log4j.properties +++ b/conf/log4j2.properties @@ -12,13 +12,13 @@ # log4j configuration used during build and unit tests rootLogger.level = INFO -property.filename = ${env:smart.log.dir}/${env:smart.log.file} +property.filename = ${sys:smart.log.dir}/${sys:smart.log.file} appenders = R, console appender.console.type = Console appender.console.name = STDOUT appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d %5p [%t] (%F:%L) - %m%n +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n appender.R.type = RollingFile appender.R.name = File @@ -35,8 +35,15 @@ rootLogger.appenderRefs = R, console rootLogger.appenderRef.console.ref = STDOUT rootLogger.appenderRef.R.ref = File -# The below configurations are supposed to be workable when log4j 1.2.17 is used. -log4j.logger.org.apache.hadoop.ipc.Server=ERROR +logger.ipc.name = org.apache.hadoop.ipc.Server +logger.ipc.level = ERROR + +logger.securedIpc.name = SecurityLogger +logger.securedIpc.level = WARN + # Downgrade Hadoop Retry Exception, please remove this line during debug -log4j.logger.org.apache.hadoop.io.retry.RetryInvocationHandler=ERROR -log4j.logger.org.apache.hadoop.hdfs.DataStreamer=ERROR \ No newline at end of file +logger.hadoopRetry.name = org.apache.hadoop.io.retry.RetryInvocationHandler +logger.hadoopRetry.level = ERROR + +logger.dataStreamer.name = org.apache.hadoop.hdfs.DataStreamer +logger.dataStreamer.level = ERROR diff --git a/smart-hadoop-support/smart-hadoop/src/test/resources/log4j.properties b/smart-action/src/test/resources/log4j2.properties similarity index 70% rename from smart-hadoop-support/smart-hadoop/src/test/resources/log4j.properties rename to smart-action/src/test/resources/log4j2.properties index 1a6baaec65a..9eaa484e2a2 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/resources/log4j.properties +++ b/smart-action/src/test/resources/log4j2.properties @@ -11,8 +11,9 @@ # limitations under the License. # log4j configuration used during build and unit tests -log4j.rootLogger=info,stdout -log4j.threshhold=ALL -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n +rootLogger = INFO, STDOUT + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/smart-agent/src/test/resources/log4j.properties b/smart-agent/src/test/resources/log4j2.properties similarity index 66% rename from smart-agent/src/test/resources/log4j.properties rename to smart-agent/src/test/resources/log4j2.properties index 4227aa5a843..be629616c4a 100644 --- a/smart-agent/src/test/resources/log4j.properties +++ b/smart-agent/src/test/resources/log4j2.properties @@ -16,16 +16,9 @@ # limitations under the License. # -# Define the root logger -log4j.rootLogger=INFO,console +rootLogger = INFO, STDOUT -# Logging Threshold -log4j.threshhold=ALL - -# -# console appender -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=[%p] [%d{MM/dd/yyyy HH:mm:ss.SSS}] [%c{1}] %m%n +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/smart-engine/src/test/resources/log4j.properties b/smart-engine/src/test/resources/log4j.properties deleted file mode 100644 index 9dc270fcc9e..00000000000 --- a/smart-engine/src/test/resources/log4j.properties +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# log4j configuration used during build and unit tests - -#log4j.rootLogger=info,stdout -#log4j.threshhold=ALL -#log4j.appender.stdout=org.apache.log4j.ConsoleAppender -#log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n -log4j.rootLogger=info,stdout diff --git a/smart-action/src/test/resources/log4j.properties b/smart-engine/src/test/resources/log4j2.properties similarity index 70% rename from smart-action/src/test/resources/log4j.properties rename to smart-engine/src/test/resources/log4j2.properties index 1a6baaec65a..9eaa484e2a2 100644 --- a/smart-action/src/test/resources/log4j.properties +++ b/smart-engine/src/test/resources/log4j2.properties @@ -11,8 +11,9 @@ # limitations under the License. # log4j configuration used during build and unit tests -log4j.rootLogger=info,stdout -log4j.threshhold=ALL -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n +rootLogger = INFO, STDOUT + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/smart-hadoop-support/smart-hadoop/src/test/resources/log4j2.properties b/smart-hadoop-support/smart-hadoop/src/test/resources/log4j2.properties index 25dfb7b6b48..9eaa484e2a2 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/resources/log4j2.properties +++ b/smart-hadoop-support/smart-hadoop/src/test/resources/log4j2.properties @@ -11,9 +11,9 @@ # limitations under the License. # log4j configuration used during build and unit tests -rootLogger=info,stdout +rootLogger = INFO, STDOUT -appender.console.name=stdout -appender.console.type=Console +appender.console.type = Console +appender.console.name = STDOUT appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n \ No newline at end of file +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/smart-integration/src/test/resources/log4j.properties b/smart-integration/src/test/resources/log4j2.properties similarity index 70% rename from smart-integration/src/test/resources/log4j.properties rename to smart-integration/src/test/resources/log4j2.properties index 1a6baaec65a..9eaa484e2a2 100644 --- a/smart-integration/src/test/resources/log4j.properties +++ b/smart-integration/src/test/resources/log4j2.properties @@ -11,8 +11,9 @@ # limitations under the License. # log4j configuration used during build and unit tests -log4j.rootLogger=info,stdout -log4j.threshhold=ALL -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n +rootLogger = INFO, STDOUT + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/smart-metastore/src/test/resources/log4j.properties b/smart-metastore/src/test/resources/log4j2.properties similarity index 70% rename from smart-metastore/src/test/resources/log4j.properties rename to smart-metastore/src/test/resources/log4j2.properties index 9c23d8c1fdf..9eaa484e2a2 100644 --- a/smart-metastore/src/test/resources/log4j.properties +++ b/smart-metastore/src/test/resources/log4j2.properties @@ -11,8 +11,9 @@ # limitations under the License. # log4j configuration used during build and unit tests -#log4j.rootLogger=info,stdout -#log4j.threshhold=ALL -#log4j.appender.stdout=org.apache.log4j.ConsoleAppender -#log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n +rootLogger = INFO, STDOUT + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/smart-server/src/test/resources/log4j.properties b/smart-server/src/test/resources/log4j.properties deleted file mode 100644 index 336cf347a8b..00000000000 --- a/smart-server/src/test/resources/log4j.properties +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# log4j configuration used during build and unit tests - -log4j.rootLogger=info,stdout -log4j.threshhold=ALL -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n -#log4j.rootLogger=info,stdout diff --git a/smart-server/src/test/resources/log4j2.properties b/smart-server/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..9eaa484e2a2 --- /dev/null +++ b/smart-server/src/test/resources/log4j2.properties @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# log4j configuration used during build and unit tests + +rootLogger = INFO, STDOUT + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/smart-web-server/pom.xml b/smart-web-server/pom.xml index fc0b1bef65f..a8f6ea30a73 100644 --- a/smart-web-server/pom.xml +++ b/smart-web-server/pom.xml @@ -120,10 +120,6 @@ - - org.springframework.boot - spring-boot-starter-log4j2 - org.projectlombok diff --git a/smart-web-server/src/main/java/org/smartdata/server/SmartRestServer.java b/smart-web-server/src/main/java/org/smartdata/server/SmartRestServer.java index 4398d8b4789..c890f33df57 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/SmartRestServer.java +++ b/smart-web-server/src/main/java/org/smartdata/server/SmartRestServer.java @@ -30,6 +30,10 @@ import static org.smartdata.conf.SmartConfKeys.SMART_CONF_KEYS_PREFIX; public class SmartRestServer { + private static final String SPRING_LOGGING_PROPERTY = + "org.springframework.boot.logging.LoggingSystem"; + private static final String SPRING_DISABLED_LOGGER = "none"; + private final SpringApplication springApplication; private volatile ConfigurableApplicationContext applicationContext; @@ -38,13 +42,14 @@ public SmartRestServer(SmartConf ssmConfig, SmartEngine smartEngine) { this.springApplication = new SpringApplication(RestServerApplication.class); injectToSpringProperties(ssmConfig); - SsmContextInitializer contextInitializer = new SsmContextInitializer(smartEngine, ssmConfig); springApplication.addInitializers(contextInitializer); } public void start() { + // disable repeated log4j loggers global configuration by Spring + System.setProperty(SPRING_LOGGING_PROPERTY, SPRING_DISABLED_LOGGER); applicationContext = springApplication.run(); } diff --git a/supports/tools/docker/multihost/conf/log4j2.properties b/supports/tools/docker/multihost/conf/log4j2.properties new file mode 100644 index 00000000000..9d6136a6f86 --- /dev/null +++ b/supports/tools/docker/multihost/conf/log4j2.properties @@ -0,0 +1,49 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# log4j configuration used during build and unit tests + +rootLogger.level = INFO +property.filename = ${sys:smart.log.dir}/${sys:smart.log.file} +appenders = R, console + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +appender.R.type = RollingFile +appender.R.name = File +appender.R.fileName = ${filename} +appender.R.filePattern = ${filename}.%d{yyyy-MM-dd} +appender.R.layout.type = PatternLayout +appender.R.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n +appender.R.policies.type = Policies +appender.R.policies.time.type = TimeBasedTriggeringPolicy +appender.R.policies.time.interval = 1 + +rootLogger.appenderRefs = R, console + +rootLogger.appenderRef.console.ref = STDOUT +rootLogger.appenderRef.R.ref = File + +logger.ipc.name = org.apache.hadoop.ipc.Server +logger.ipc.level = ERROR + +logger.securedIpc.name = SecurityLogger +logger.securedIpc.level = WARN + +# Downgrade Hadoop Retry Exception, please remove this line during debug +logger.hadoopRetry.name = org.apache.hadoop.io.retry.RetryInvocationHandler +logger.hadoopRetry.level = ERROR + +logger.dataStreamer.name = org.apache.hadoop.hdfs.DataStreamer +logger.dataStreamer.level = ERROR diff --git a/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh b/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh index 8004047c25f..49f46ad72d9 100644 --- a/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh +++ b/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh @@ -7,6 +7,8 @@ cp /root/.ssh/id_rsa.pub /tmp/shared/id_rsa.pub service ssh start ssh-keyscan "$HOSTNAME" >> /root/.ssh/known_hosts echo "export JAVA_HOME=${JAVA_HOME}" >> /root/.bashrc +echo "export SMART_HOME=${SSM_HOME}" >> /root/.bashrc +echo "export SMART_CONF_DIR=${SSM_HOME}/conf/" >> /root/.bashrc # Starting Smart Storage Manager cd $SSM_HOME || exit @@ -15,7 +17,7 @@ echo "---------------------------" echo "Starting SSM server locally" echo "---------------------------" -source bin/start-ssm.sh --config conf/ & +source bin/start-ssm.sh --config ${SSM_HOME}/conf/ & wait_for_it $(hostname -f):8081 echo "-------------------"