Spark security options constraints (#1892)

* Resolved httpclient dependencies (#1887)

* added security constraints

Author: compae

CHANGELOG.md

@@ -9,6 +9,7 @@
 * Info service
 * Security options configurable
 * Https with marathon-lb
+* Frontend performance improvements
 * Bugfix: Postgres library added
 
 ## 1.4.0 (May 17, 2017)

dist/src/main/resources/marathon-app-template.json

@@ -32,20 +32,21 @@
     "SPARK_HOME": "???",
     "MESOS_NATIVE_JAVA_LIBRARY": "???",
     "LD_LIBRARY_PATH": "???",
-    "HDFS_USER_NAME": "???",
+    "HADOOP_USER_NAME": "???",
     "SPARK_USER": "???",
-    "HDFS_CONF_FROM_URI": "???",
+    "HADOOP_CONF_FROM_URI": "???",
     "CORE_SITE_FROM_URI": "???",
-    "HDFS_CONF_FROM_DFS": "???",
-    "HDFS_CONF_FROM_DFS_NOT_SECURED": "???",
+    "HDFS_CONF_URI": "???",
+    "HADOOP_CONF_FROM_DFS": "???",
+    "HADOOP_CONF_FROM_DFS_NOT_SECURED": "???",
     "HADOOP_FS_DEFAULT_NAME": "???",
     "HADOOP_CONF_URI": "???",
     "HADOOP_SECURITY_AUTH": "???",
     "HADOOP_RPC_PROTECTION": "???",
     "HADOOP_DFS_ENCRYPT_DATA_TRANSFER": "???",
     "HADOOP_SECURITY_TOKEN_USE_IP": "???",
-    "HADOOP_NAMENODE_KERBEROS_PRINCIPAL": "???",
-    "HADOOP_NAMENODE_KERBEROS_PRINCIPAL_PATTERN": "???",
+    "HADOOP_NAMENODE_KRB_PRINCIPAL": "???",
+    "HADOOP_NAMENODE_KRB_PRINCIPAL_PATTERN": "???",
     "HADOOP_DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES": "???",
     "HADOOP_DFS_ENCRYPT_DATA_CIPHER_KEY_BITLENGTH": "???",
     "HADOOP_CONF_DIR": "???",
@@ -56,7 +57,7 @@
     "ZOOKEEPER_LOG_LEVEL": "???",
     "SECURITY_TLS_ENABLE": "???",
     "SECURITY_TRUSTSTORE_ENABLE": "???",
-    "SECURITY_KERBEROS_ENABLE": "???",
+    "SECURITY_KRB_ENABLE": "???",
     "SECURITY_OAUTH2_ENABLE": "???",
     "SECURITY_MESOS_ENABLE": "???",
     "SPARK_SECURITY_KAFKA_ENABLE": "???"

docker/hdfs_utils.sh

@@ -23,6 +23,8 @@ function generate_core-site-from-uri() {
     echo "[CORE-SITE] HADOOP $HADOOP_CONF_DIR/core-site.xml was NOT configured"
     exit 1
   fi
+  echo "" >> ${VARIABLES}
+  echo "export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" >> ${VARIABLES}
   echo "" >> ${SYSTEM_VARIABLES}
   echo "export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" >> ${SYSTEM_VARIABLES}
 }
@@ -40,6 +42,8 @@ function generate_hdfs-conf-from-uri() {
 
   if [[ $? == 0 ]]; then
     echo "[HADOOP-CONF] HADOOP $CORE_SITE and $HDFS_SITE configured succesfully"
+    echo "" >> ${VARIABLES}
+    echo "export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" >> ${VARIABLES}
     echo "" >> ${SYSTEM_VARIABLES}
     echo "export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" >> ${SYSTEM_VARIABLES}
   else
@@ -129,13 +133,13 @@ cat > "${HADOOP_CONF_DIR}/hdfs-site.xml" <<EOF
         </configuration>
 EOF
 
-sed -i "s#__<KERBEROS_PRINCIPAL>__#$HADOOP_NAMENODE_KERBEROS_PRINCIPAL#" "${HADOOP_CONF_DIR}/hdfs-site.xml" \
+sed -i "s#__<KERBEROS_PRINCIPAL>__#$HADOOP_NAMENODE_KRB_PRINCIPAL#" "${HADOOP_CONF_DIR}/hdfs-site.xml" \
 && echo "[hdfs-site.xml] dfs.namenode.kerberos.principal in hdfs-site.xml" \
-|| echo "[hdfs-site.xml-ERROR] Something went wrong when HADOOP_NAMENODE_KERBEROS_PRINCIPAL was configured in hdfs-site.xml"
+|| echo "[hdfs-site.xml-ERROR] Something went wrong when HADOOP_NAMENODE_KRB_PRINCIPAL was configured in hdfs-site.xml"
 
-sed -i "s#__<KERBEROS_PRINCIPAL_PATTERN>__#$HADOOP_NAMENODE_KERBEROS_PRINCIPAL_PATTERN#" "${HADOOP_CONF_DIR}/hdfs-site.xml" \
+sed -i "s#__<KERBEROS_PRINCIPAL_PATTERN>__#$HADOOP_NAMENODE_KRB_PRINCIPAL_PATTERN#" "${HADOOP_CONF_DIR}/hdfs-site.xml" \
 && echo "[hdfs-site.xml] dfs.namenode.kerberos.principal.pattern in hdfs-site.xml" \
-|| echo "[hdfs-site.xml-ERROR] Something went wrong when HADOOP_NAMENODE_KERBEROS_PRINCIPAL_PATTERN was configured in hdfs-site.xml"
+|| echo "[hdfs-site.xml-ERROR] Something went wrong when HADOOP_NAMENODE_KRB_PRINCIPAL_PATTERN was configured in hdfs-site.xml"
 
 sed -i "s#__<ENCRYPT_DATA_TRANSFER>__#$HADOOP_DFS_ENCRYPT_DATA_TRANSFER#" "${HADOOP_CONF_DIR}/hdfs-site.xml" \
 && echo "[hdfs-site.xml] dfs.encrypt.data.transfer in hdfs-site.xml" \
@@ -152,6 +156,8 @@ sed -i "s#__<ENCRYPT_DATA_TRANSFER_CIPHER_KEY_BITLENGTH>__#$HADOOP_DFS_ENCRYPT_D
 
   if [[ $? == 0 ]]; then
     echo "[HADOOP-CONF] HADOOP $CORE_SITE and $HDFS_SITE configured succesfully"
+    echo "" >> ${VARIABLES}
+    echo "export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" >> ${VARIABLES}
     echo "" >> ${SYSTEM_VARIABLES}
     echo "export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" >> ${SYSTEM_VARIABLES}
   else
@@ -209,6 +215,8 @@ sed -i "s#__<FS_DEFAULT_NAME>__#$HADOOP_FS_DEFAULT_NAME#" "${HADOOP_CONF_DIR}/co
 
   if [[ $? == 0 ]]; then
     echo "[HADOOP-CONF] HADOOP $CORE_SITE not secured configured succesfully"
+    echo "" >> ${VARIABLES}
+    echo "export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" >> ${VARIABLES}
     echo "" >> ${SYSTEM_VARIABLES}
     echo "export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" >> ${SYSTEM_VARIABLES}
   else

docker/security-config.sh

@@ -46,7 +46,7 @@ fi
 ####################################################
 ## Kerberos config set SPARTA_PRINCIPAL_NAME and SPARTA_KEYTAB_PATH
 ####################################################
-if [ -v SECURITY_KERBEROS_ENABLE ] && [ ${#SECURITY_KERBEROS_ENABLE} != 0 ] && [ $SECURITY_KERBEROS_ENABLE == "true" ]; then
+if [ -v SECURITY_KRB_ENABLE ] && [ ${#SECURITY_KRB_ENABLE} != 0 ] && [ $SECURITY_KRB_ENABLE == "true" ]; then
   _log_sparta_sec "Configuring kerberos ..."
   source /kerberos-server-config.sh
   _log_sparta_sec "Configuring kerberos Ok"

docker/sparta-common.sh

@@ -47,17 +47,17 @@ function initSpark() {
 }
 
 function initHdfs() {
-  if [[ -v HDFS_USER_NAME ]]; then
-      echo "" >> ${VARIABLES}
-    echo "export HADOOP_USER_NAME=${HDFS_USER_NAME}" >> ${VARIABLES}
+  if [[ -v HADOOP_USER_NAME ]]; then
+    echo "" >> ${VARIABLES}
+    echo "export HADOOP_USER_NAME=${HADOOP_USER_NAME}" >> ${VARIABLES}
     echo "" >> ${SYSTEM_VARIABLES}
-    echo "export HADOOP_USER_NAME=${HDFS_USER_NAME}" >> ${SYSTEM_VARIABLES}
+    echo "export HADOOP_USER_NAME=${HADOOP_USER_NAME}" >> ${SYSTEM_VARIABLES}
   fi
 
-  if [[ ! -v HDFS_CONF_FROM_URI ]]; then
-   HDFS_CONF_FROM_URI="false"
+  if [[ ! -v HADOOP_CONF_FROM_URI ]]; then
+   HADOOP_CONF_FROM_URI="false"
   fi
-  if [ $HDFS_CONF_FROM_URI == "true" ] && [ -v HADOOP_CONF_URI ] && [ ${#HADOOP_CONF_URI} != 0 ]; then
+  if [ $HADOOP_CONF_FROM_URI == "true" ] && [ -v HADOOP_CONF_URI ] && [ ${#HADOOP_CONF_URI} != 0 ]; then
     if [ ! -v HADOOP_CONF_DIR ] && [ ${#HADOOP_CONF_DIR} != 0 ]; then
       HADOOP_CONF_DIR=/opt/sds/hadoop/conf
     fi
@@ -78,10 +78,10 @@ function initHdfs() {
     generate_core-site-from-uri
   fi
 
-  if [[ ! -v HDFS_CONF_FROM_DFS ]]; then
-   HDFS_CONF_FROM_DFS="false"
+  if [[ ! -v HADOOP_CONF_FROM_DFS ]]; then
+   HADOOP_CONF_FROM_DFS="false"
   fi
-  if [ $HDFS_CONF_FROM_DFS == "true" ] && [ -v HADOOP_FS_DEFAULT_NAME ] && [ ${#HADOOP_FS_DEFAULT_NAME} != 0 ]; then
+  if [ $HADOOP_CONF_FROM_DFS == "true" ] && [ -v HADOOP_FS_DEFAULT_NAME ] && [ ${#HADOOP_FS_DEFAULT_NAME} != 0 ]; then
     if [ ! -v HADOOP_CONF_DIR ] && [ ${#HADOOP_CONF_DIR} != 0 ]; then
       HADOOP_CONF_DIR=/opt/sds/hadoop/conf
     fi
@@ -90,10 +90,10 @@ function initHdfs() {
     generate_hdfs-conf-from-fs
   fi
 
-  if [[ ! -v HDFS_CONF_FROM_DFS_NOT_SECURED ]]; then
-   HDFS_CONF_FROM_DFS_NOT_SECURED="false"
+  if [[ ! -v HADOOP_CONF_FROM_DFS_NOT_SECURED ]]; then
+   HADOOP_CONF_FROM_DFS_NOT_SECURED="false"
   fi
-  if [ $HDFS_CONF_FROM_DFS_NOT_SECURED == "true" ] && [ -v HADOOP_FS_DEFAULT_NAME ] && [ ${#HADOOP_FS_DEFAULT_NAME} != 0 ]; then
+  if [ $HADOOP_CONF_FROM_DFS_NOT_SECURED == "true" ] && [ -v HADOOP_FS_DEFAULT_NAME ] && [ ${#HADOOP_FS_DEFAULT_NAME} != 0 ]; then
     if [ ! -v HADOOP_CONF_DIR ] && [ ${#HADOOP_CONF_DIR} != 0 ]; then
       HADOOP_CONF_DIR=/opt/sds/hadoop/conf
     fi

docker/sparta-server-utils.sh

@@ -25,10 +25,10 @@ function initJavaOptions() {
 
 function hdfsOptions() {
 
- if [[ ! -v HDFS_PORT ]]; then
-   HDFS_PORT=9000
+ if [[ ! -v HADOOP_PORT ]]; then
+   HADOOP_PORT=9000
  fi
- sed -i "s|.*sparta.hdfs.hdfsPort.*|sparta.hdfs.hdfsPort = ${HDFS_PORT}|" ${SPARTA_CONF_FILE}
+ sed -i "s|.*sparta.hdfs.hdfsPort.*|sparta.hdfs.hdfsPort = ${HADOOP_PORT}|" ${SPARTA_CONF_FILE}
 
  if [[ ! -v HDFS_SECURITY_ENABLED ]]; then
    HDFS_SECURITY_ENABLED=false

examples/docker/README.md

@@ -22,23 +22,23 @@ Entrypoint variables
 
 | PARAM        |
 | -------------:|
-| HDFS_USER_NAME |
-| HDFS_CONF_FROM_URI |
+| HADOOP_USER_NAME |
+| HADOOP_CONF_FROM_URI |
 | CORE_SITE_FROM_URI |
-| HDFS_CONF_FROM_DFS |
-| HDFS_CONF_FROM_DFS_NOT_SECURED |
+| HADOOP_CONF_FROM_DFS |
+| HADOOP_CONF_FROM_DFS_NOT_SECURED |
 | HADOOP_CONF_URI |
 | HADOOP_FS_DEFAULT_NAME |
 | HADOOP_SECURITY_AUTH |
 | HADOOP_RPC_PROTECTION |
 | HADOOP_DFS_ENCRYPT_DATA_TRANSFER |
 | HADOOP_SECURITY_TOKEN_USE_IP |
-| HADOOP_NAMENODE_KERBEROS_PRINCIPAL |
-| HADOOP_NAMENODE_KERBEROS_PRINCIPAL_PATTERN |
+| HADOOP_NAMENODE_KRB_PRINCIPAL |
+| HADOOP_NAMENODE_KRB_PRINCIPAL_PATTERN |
 | HADOOP_DFS_ENCRYPT_DATA_TRANSFER |
 | HADOOP_DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES |
 | HADOOP_SECURITY_AUTH |
-| HDFS_PORT |
+| HADOOP_PORT |
 | HDFS_KEYTAB_RELOAD |
 | HDFS_KEYTAB_RELOAD_TIME |
 
@@ -116,7 +116,6 @@ Entrypoint variables
 | SPARK_LOCAL_METRICS_CONF |
 
 
-
 ## MESOS EXECUTION OPTIONS
 | PARAM        |
 | -------------:|
@@ -227,7 +226,7 @@ docker run -dit --name sp -p 9090:9090 --env RUN_MODE=debug --env SERVICE_LOG_LE
    --env SPARTA_ZOOKEEPER_CONNECTION_STRING=zk.demo.stratio.com --env SPARTA_EXECUTION_MODE=mesos 
    --env SPARTA_CHECKPOINT_PATH=/user/stratio/checkpoint 
    --env SPARK_MESOS_MASTER=mesos://mm11.demo.stratio.com:7077 --env HDFS_MASTER=hm.demo.stratio.com 
-   --env HDFS_PORT=8020 qa.stratio.com/stratio/sparta:latest
+   --env HADOOP_PORT=8020 qa.stratio.com/stratio/sparta:latest
 ```
 
 Usage examples with VAULT 

serving-api/src/main/resources/marathon-app-template.json

@@ -32,20 +32,21 @@
     "SPARK_HOME": "???",
     "MESOS_NATIVE_JAVA_LIBRARY": "???",
     "LD_LIBRARY_PATH": "???",
-    "HDFS_USER_NAME": "???",
+    "HADOOP_USER_NAME": "???",
     "SPARK_USER": "???",
-    "HDFS_CONF_FROM_URI": "???",
+    "HADOOP_CONF_FROM_URI": "???",
     "CORE_SITE_FROM_URI": "???",
-    "HDFS_CONF_FROM_DFS": "???",
-    "HDFS_CONF_FROM_DFS_NOT_SECURED": "???",
+    "HDFS_CONF_URI": "???",
+    "HADOOP_CONF_FROM_DFS": "???",
+    "HADOOP_CONF_FROM_DFS_NOT_SECURED": "???",
     "HADOOP_FS_DEFAULT_NAME": "???",
     "HADOOP_CONF_URI": "???",
     "HADOOP_SECURITY_AUTH": "???",
     "HADOOP_RPC_PROTECTION": "???",
     "HADOOP_DFS_ENCRYPT_DATA_TRANSFER": "???",
     "HADOOP_SECURITY_TOKEN_USE_IP": "???",
-    "HADOOP_NAMENODE_KERBEROS_PRINCIPAL": "???",
-    "HADOOP_NAMENODE_KERBEROS_PRINCIPAL_PATTERN": "???",
+    "HADOOP_NAMENODE_KRB_PRINCIPAL": "???",
+    "HADOOP_NAMENODE_KRB_PRINCIPAL_PATTERN": "???",
     "HADOOP_DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES": "???",
     "HADOOP_DFS_ENCRYPT_DATA_CIPHER_KEY_BITLENGTH": "???",
     "HADOOP_CONF_DIR": "???",
@@ -56,7 +57,7 @@
     "ZOOKEEPER_LOG_LEVEL": "???",
     "SECURITY_TLS_ENABLE": "???",
     "SECURITY_TRUSTSTORE_ENABLE": "???",
-    "SECURITY_KERBEROS_ENABLE": "???",
+    "SECURITY_KRB_ENABLE": "???",
     "SECURITY_OAUTH2_ENABLE": "???",
     "SECURITY_MESOS_ENABLE": "???",
     "SPARK_SECURITY_KAFKA_ENABLE": "???"

serving-core/src/main/scala/com/stratio/sparta/serving/core/marathon/MarathonService.scala

@@ -103,12 +103,12 @@ class MarathonService(context: ActorContext,
   val AppHeapSizeEnv = "MARATHON_APP_HEAP_SIZE"
   val AppHeapMinimunSizeEnv = "MARATHON_APP_HEAP_MINIMUM_SIZE"
   val SparkHomeEnv = "SPARK_HOME"
-  val HadoopUserNameEnv = "HDFS_USER_NAME"
+  val HadoopUserNameEnv = "HADOOP_USER_NAME"
   val HdfsUserNameEnv = "HADOOP_USER_NAME"
-  val HdfsConfFromUriEnv = "HDFS_CONF_FROM_URI"
+  val HdfsConfFromUriEnv = "HADOOP_CONF_FROM_URI"
   val CoreSiteFromUriEnv = "CORE_SITE_FROM_URI"
-  val HdfsConfFromDfsEnv = "HDFS_CONF_FROM_DFS"
-  val HdfsConfFromDfsNotSecuredEnv = "HDFS_CONF_FROM_DFS_NOT_SECURED"
+  val HdfsConfFromDfsEnv = "HADOOP_CONF_FROM_DFS"
+  val HdfsConfFromDfsNotSecuredEnv = "HADOOP_CONF_FROM_DFS_NOT_SECURED"
   val DefaultFsEnv = "HADOOP_FS_DEFAULT_NAME"
   val DefaultHdfsConfUriEnv = "HADOOP_CONF_URI"
   val HadoopConfDirEnv = "HADOOP_CONF_DIR"
@@ -123,17 +123,18 @@ class MarathonService(context: ActorContext,
   val HdfsSecurityAuthEnv = "HADOOP_SECURITY_AUTH"
   val HdfsEncryptDataEnv = "HADOOP_DFS_ENCRYPT_DATA_TRANSFER"
   val HdfsTokenUseIpEnv = "HADOOP_SECURITY_TOKEN_USE_IP"
-  val HdfsKerberosPrincipalEnv = "HADOOP_NAMENODE_KERBEROS_PRINCIPAL"
-  val HdfsKerberosPrincipalPatternEnv = "HADOOP_NAMENODE_KERBEROS_PRINCIPAL_PATTERN"
+  val HdfsKerberosPrincipalEnv = "HADOOP_NAMENODE_KRB_PRINCIPAL"
+  val HdfsKerberosPrincipalPatternEnv = "HADOOP_NAMENODE_KRB_PRINCIPAL_PATTERN"
   val HdfsEncryptDataTransferEnv = "HADOOP_DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES"
   val HdfsEncryptDataBitLengthEnv = "HADOOP_DFS_ENCRYPT_DATA_CIPHER_KEY_BITLENGTH"
   val SparkUserEnv = "SPARK_USER"
   val SecurityTlsEnv = "SECURITY_TLS_ENABLE"
   val SecurityTrustoreEnv = "SECURITY_TRUSTSTORE_ENABLE"
-  val SecurityKerberosEnv = "SECURITY_KERBEROS_ENABLE"
+  val SecurityKerberosEnv = "SECURITY_KRB_ENABLE"
   val SecurityOauth2Env = "SECURITY_OAUTH2_ENABLE"
   val SecurityMesosEnv = "SECURITY_MESOS_ENABLE"
   val SecuritySparkKafkaEnv = "SPARK_SECURITY_KAFKA_ENABLE"
+  val SecuritySparkHdfsEnv = "HDFS_CONF_URI"
 
   /* Lazy variables */
 
@@ -222,6 +223,9 @@ class MarathonService(context: ActorContext,
     } else None
   }
 
+  private def envSparkSecurityHdfs(sparkConfigurations: Map[String, String]): Option[String] =
+    sparkConfigurations.get("spark.mesos.driverEnv.HDFS_CONF_URI")
+
   private def envSparkHome: Option[String] = Properties.envOrNone(SparkHomeEnv)
 
   private def envConstraint: Option[String] = Properties.envOrNone(Constraints)
@@ -414,6 +418,7 @@ class MarathonService(context: ActorContext,
       SecurityOauth2Env -> envOauth2,
       SecurityMesosEnv -> envMesos,
       SecuritySparkKafkaEnv -> envSparkSecurityKafka(submitRequest.sparkConfigurations),
+      SecuritySparkHdfsEnv -> envSparkSecurityHdfs(submitRequest.sparkConfigurations),
       DcosServiceName -> Option(ServiceName),
       SparkUserEnv -> policyModel.sparkUser
     ).flatMap { case (k, v) => v.map(value => Option(k -> value)) }.flatten.toMap