Batch of changes (#574)

- haraka: dont create config/loglevel
- mt: TOASTER_EDITOR & TOASTER_EDITOR_PORT
- dma: add MASQUERADE in jails
- pf tweaks
- mua: uri encode password for curl test
- test/run.sh: added
- moved test/get_jail_ip into mail-toaster.bats

.github/workflows/ci.yml

@@ -24,7 +24,9 @@ jobs:
     - name: Setup Bats
       run: git submodule update --init --recursive
     - name: bats test
-      run: ./test/bats/bin/bats test
+      run: |
+        ./test/bats/bin/bats test/*.bats
+        ./test/bats/bin/bats test/include/*.bats
 
   freebsd:
     if: false

.shellcheckrc

@@ -1 +1 @@
-disable=SC1004,SC1091,SC2009,SC2016,SC2039,SC2086,SC2119,SC2120,SC2153,SC2154,SC3033,SC3043
+disable=1004,1091,2009,2016,2039,2086,2119,2120,2153,2154,3033,3037,3043

include/editor.sh

@@ -72,7 +72,7 @@ configure_editor()
 {
 	local _base=${1:-""}
 
-	case "$TOASTER_EDITOR" in
+	case "$TOASTER_EDITOR_PORT" in
 		neovim)
 			configure_neovim
 			;;

include/mua.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+set -e
+
+# shellcheck disable=3003
 test_imap_empty()
 {
 	pkg info | grep -q ^empty || pkg install -y empty
@@ -40,7 +43,8 @@ EOF
 test_imap_curl()
 {
 	# shellcheck disable=SC2001
-	curl -k -v --login-options 'AUTH=PLAIN' "imaps://$(echo $MUA_TEST_USER | sed -e 's/@/%40/'):${MUA_TEST_PASS}@${MUA_TEST_HOST}/"
+	curl -k -v --login-options 'AUTH=PLAIN' \
+		"imaps://$(uriencode $MUA_TEST_USER):$(uriencode MUA_TEST_PASS)@${MUA_TEST_HOST}/"
 }
 
 test_imap()
@@ -51,6 +55,7 @@ test_imap()
 	# test_imap_empty
 }
 
+# shellcheck disable=3003
 test_pop3_empty()
 {
 	pkg info | grep -q ^empty || pkg install -y empty
@@ -77,5 +82,25 @@ test_pop3_empty()
 test_pop3()
 {
 	# shellcheck disable=SC2001
-	curl -k -v --login-options 'AUTH=PLAIN' "pop3s://$(echo $MUA_TEST_USER | sed -e 's/@/%40/'):${MUA_TEST_PASS}@${MUA_TEST_HOST}/"
-}
+	curl -k -v --login-options 'AUTH=PLAIN' \
+		"pop3s://$(uriencode $MUA_TEST_USER):$(uriencode MUA_TEST_PASS)@${MUA_TEST_HOST}/"
+}
+
+# https://stackoverflow.com/questions/296536/how-to-urlencode-data-for-curl-command
+# shellcheck disable=3005,3018,3024,3045,3057
+uriencode() {
+  local string="${1}"
+  local strlen=${#string}
+  local encoded=""
+  local pos c o
+
+  for (( pos=0 ; pos<strlen ; pos++ )); do
+     c=${string:$pos:1}
+     case "$c" in
+        [-_.~a-zA-Z0-9] ) o="${c}" ;;
+        * )               printf -v o '%%%02x' "'$c"
+     esac
+     encoded+="${o}"
+  done
+  echo "${encoded}"
+}

include/shell.sh

@@ -18,8 +18,7 @@ install_bash()
 		return
 	fi
 
-	tell_status "adding .bash_profile for root@jail"
-	configure_bash "$_profile"
+	configure_bash "$1"
 }
 
 install_zsh()
@@ -31,23 +30,40 @@ install_zsh()
 
 configure_bash()
 {
-	tee -a "$1" <<'EO_BASH_PROFILE'
+	if ! grep -q profile "$1/root/.profile"; then
+		tell_status "telling bash to read /etc/profile"
+		sed -i '' \
+			-e '/PAGER$/ a\
+\
+if [ -n "\$BASH" ]; then . /etc/profile; fi' \
+			"$1/root/.profile"
+		echo '' >> "$1/root/.profile"
+		echo 'if [ -n "$BASH" ] && [ -r ~/.bashrc ]; then . ~/.bashrc; fi' >> "$1/root/.profile"
+	fi
+
+	if [ ! -e "$1/root/.bashrc" ]; then
+		tell_status "creating $1/root/.bashrc"
+		cat <<'EO_BASH_RC' > "$1/root/.bashrc"
 
-export EDITOR="vim"
-export BLOCKSIZE=K;
 export HISTSIZE=10000
 export HISTCONTROL=ignoredups:erasedups
 export HISTIGNORE="&:[bf]g:exit"
+
 shopt -s histappend
 shopt -s cdspell
-alias h="history 200"
-alias ll="ls -alFG"
-EO_BASH_PROFILE
 
-	if ! grep -qs profile "$1"; then
-		tee -a "$1" <<EO_INCL
-. /etc/profile
-EO_INCL
+if [[ $- == *i* ]]
+then
+    bind '"\e[A": history-search-backward'
+    bind '"\e[B": history-search-forward'
+fi
+
+PS1="[\u@\[\033[0;36m\]\h\[\033[0m\]] \w "
+case $(id -u) in
+    0) PS1="${PS1}# ";;
+    *) PS1="${PS1}$ ";;
+esac
+EO_BASH_RC
 	fi
 }
 
@@ -56,22 +72,28 @@ configure_bourne_shell()
 	_f="$1/etc/profile.d/toaster.sh"
 	if ! grep -qs ^PS1 "$_f"; then
 		tell_status "customizing bourne shell prompt"
-		cat <<'EO_BOURNE_SHELL' > "$_f"
+		cat <<EO_BOURNE_SHELL > "$_f"
+export EDITOR="$TOASTER_EDITOR"
+export BLOCKSIZE=K;
+
 alias h='fc -l'
-alias m=$PAGER
+alias m=\$PAGER
+alias ls="ls -FG"
 alias ll="ls -alFG"
 alias g='egrep -i'
+#alias df="df -h -tnodevfs,procfs,nullfs,tmpfs"
 
-PS1="$(whoami)@$(hostname -s):\\w "
-case $(id -u) in
-    0) PS1="${PS1}# ";;
-    *) PS1="${PS1}$ ";;
+# set prompt for bourne shell (/bin/sh)
+PS1="\$(whoami)@\$(hostname -s):\\w "
+case \$(id -u) in
+    0) PS1="\${PS1}# ";;
+    *) PS1="\${PS1}\$ ";;
 esac
 
 jexecl() {
-  if   [ -z "$1" ]; then /usr/sbin/jexec;
-  elif [ -n "$2" ]; then /usr/sbin/jexec ${@:1};
-  else /usr/sbin/jexec $1 login -f -h $(hostname) root;
+  if   [ -z "\$1" ]; then /usr/sbin/jexec;
+  elif [ -n "\$2" ]; then /usr/sbin/jexec \${@:1};
+  else /usr/sbin/jexec \$1 login -f -h $(hostname) root;
   fi
 }
 EO_BOURNE_SHELL
@@ -91,18 +113,18 @@ configure_csh_shell()
 	fi
 
 	tell_status "configure C shell"
-	cat <<'EO_CSHRC' > "$_cshrc"
+	cat <<EO_CSHRC > "$_cshrc"
 alias h         history 25
 alias j         jobs -l
 alias la        ls -aF
 alias lf        ls -FA
 alias ll        ls -lAFG
 
-setenv  EDITOR  vi
+setenv  EDITOR  $TOASTER_EDITOR
 setenv  PAGER   less
 setenv  BLOCKSIZE       K
 
-if ($?prompt) then
+if (\$?prompt) then
         # An interactive shell -- set some stuff up
         set prompt = "%N@%m:%~ %# "
         set promptchars = "%#"
@@ -114,7 +136,7 @@ if ($?prompt) then
         # Use history to aid expansion
         set autoexpand
         set autorehash
-        if ( $?tcsh ) then
+        if ( \$?tcsh ) then
                 bindkey "^W" backward-delete-word
                 bindkey -k up history-search-backward
                 bindkey -k down history-search-forward

mail-toaster.sh

@@ -75,7 +75,8 @@ export JAIL_NET6="$(get_random_ip6net)"
 export ZFS_VOL="zroot"
 export ZFS_JAIL_MNT="/jails"
 export ZFS_DATA_MNT="/data"
-export TOASTER_EDITOR="vim-tiny"
+export TOASTER_EDITOR="vim"
+export TOASTER_EDITOR_PORT="vim-tiny"
 export TOASTER_MSA="haraka"
 export TOASTER_MYSQL="1"
 export TOASTER_MYSQL_PASS=""
@@ -172,7 +173,8 @@ export TLS_LIBRARY=${TLS_LIBRARY:=""}
 export TOASTER_BASE_MTA=${TOASTER_BASE_MTA:=""}
 export TOASTER_BASE_PKGS=${TOASTER_BASE_PKGS:="pkg ca_root_nss"}
 export TOASTER_BUILD_DEBUG=${TOASTER_BUILD_DEBUG:="0"}
-export TOASTER_EDITOR=${TOASTER_EDITOR:="vim-tiny"}
+export TOASTER_EDITOR=${TOASTER_EDITOR:="vim"}
+export TOASTER_EDITOR_PORT=${TOASTER_EDITOR_PORT:="vim-tiny"}
 # See https://github.com/msimerson/Mail-Toaster-6/wiki/MySQL
 export TOASTER_MYSQL=${TOASTER_MYSQL:="1"}
 export TOASTER_MARIADB=${TOASTER_MARIADB:="0"}
@@ -674,6 +676,7 @@ create_staged_fs()
 
 	assure_ip6_addr_is_declared "$1"
 	stage_resolv_conf
+	echo "MASQUERADE $1@$TOASTER_MAIL_DOMAIN" >> "$STAGE_MNT/etc/dma/dma.conf"
 
 	zfs_create_fs "$ZFS_DATA_VOL/$1" "$ZFS_DATA_MNT/$1"
 	install_fstab $1
@@ -1388,6 +1391,7 @@ store_exec()
 	chmod 755 "$1"
 }
 
+# shellcheck disable=3044,3018
 onexit() { while caller $((n++)); do :; done; }
 
 if [ "$TOASTER_BUILD_DEBUG" = "1" ]; then

provision/dcc.sh

@@ -55,7 +55,8 @@ configure_dcc()
 
 	_pf_etc="$ZFS_DATA_MNT/dcc/etc/pf.conf.d"
 	store_config "$_pf_etc/allow.conf" <<EO_PF_ALLOW
-table <dcc_server> { \$ext_ip4 \$ext_ip6 $(get_jail_ip dcc) $(get_jail_ip6 dcc) }
+table <dcc_server> { $(get_jail_ip dcc), $(get_jail_ip6 dcc) }
+pass in quick proto udp from any port 6277 to <ext_ip>
 pass in quick proto udp from any port 6277 to <dcc_server>
 EO_PF_ALLOW
 

provision/dovecot.sh

@@ -541,7 +541,7 @@ rdr inet  proto tcp from <insecure_mua> to <ext_ip4> port { 110 143 } -> $int_ip
 rdr inet6 proto tcp from <insecure_mua> to <ext_ip6> port { 110 143 } -> $int_ip6
 EO_PF_RDR
 
-	store_config "$_pf_etc/allow.conf" <<EO_PF_RDR
+	store_config "$_pf_etc/allow.conf" <<EO_PF_ALLOW
 int_ip4 = "$(get_jail_ip dovecot)"
 int_ip6 = "$(get_jail_ip6 dovecot)"
 
@@ -551,7 +551,7 @@ pass in quick proto tcp from any to <ext_ip> port { 993 995 }
 pass in quick proto tcp from any to <dovecot_int> port { 993 995 }
 
 pass in quick proto tcp from <insecure_mua> to <dovecot_int> port { 110 143 }
-EO_PF_RDR
+EO_PF_ALLOW
 }
 
 configure_dovecot()

provision/haraka.sh

@@ -565,7 +565,6 @@ configure_haraka_helo()
 		tee "$HARAKA_CONF/helo.checks.ini" <<EO_HELO_INI
 [reject]
 host_mismatch=false
-valid_hostname=false
 EO_HELO_INI
 	fi
 
@@ -677,7 +676,7 @@ configure_haraka()
 	stage_exec haraka -i /data
 
 	tell_status "configuring Haraka"
-	echo 'LOGINFO' > "$HARAKA_CONF/loglevel"
+	# echo 'LOGINFO' > "$HARAKA_CONF/loglevel"
 	if [ ! -f "$HARAKA_CONF/tarpit.timeout" ]; then
 		echo '3' > "$HARAKA_CONF/tarpit.timeout"
 	fi
@@ -749,7 +748,7 @@ start_haraka()
 		"$STAGE_MNT/usr/local/etc/rc.d/haraka"
 	chmod 555 "$STAGE_MNT/usr/local/etc/rc.d/haraka"
 	stage_sysrc haraka_enable=YES
-	sysrc -f "$STAGE_MNT/etc/rc.conf" haraka_flags='-c /data'
+	stage_sysrc haraka_flags='-c /data'
 
 	if [ ! -d "$HARAKA_CONF/queue" ]; then
 		mkdir -p "$HARAKA_CONF/queue"