diagnose

#!/bin/bash

spinner=(⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏)
done=⠿
red='\033[0;31m'
green='\033[0;32m'
yellow='\033[0;33m'

SUBJECT=some-unique-id
DISCARD_MONGO_STRING_REGX="\\x1b]0;mongosh mongodb:\/\/<credentials>@127\.0\.0\.1:27017\/cte?directConnection=true&serverSelectionTimeoutMS=2000\\x07\\x1b\[1G \\x1b\[1G"

# --- Locks -------------------------------------------------------
LOCK_FILE=/tmp/$SUBJECT.lock
if [ -f "$LOCK_FILE" ]; then
  echo "Script is already running"
  exit
fi

trap "rm -f $LOCK_FILE" EXIT
touch $LOCK_FILE

. ./.env
if [ -n "${HA_IP_LIST}" ]; then
  docker_compose="docker compose -f docker-compose-ha.yml"
  podman_compose="podman-compose -f podman-compose-ha.yml"
else
  docker_compose="docker compose -f docker-compose.yml"
  podman_compose="podman-compose -f podman-compose.yml"
fi

# --- Output functions --------------------------------------------

RHEL=$(hostnamectl | grep "Operating System:" | sed 's/Operating System://')
MAINTENANCE_PASSWORD=$(sudo cat $LOCATION  | grep MAINTENANCE_PASSWORD=)
MAINTENANCE_PASSWORD=${MAINTENANCE_PASSWORD#*=}
MAINTENANCE_PASSWORD=$(echo $MAINTENANCE_PASSWORD | tr -d "'")
core_check=false
core_running=false
ui_check=false
mongo_check=false
rabbitmq_check=false
rabbitmq_running=false

check_mongo_running() {
  if [[ $RHEL: == *"Red Hat"* ]]; then
    if [[ $(podman ps -q -a -f name="mongodb") ]]; then
      retry=0
      while [ $retry -lt 4 ]; do
        if [[ $(podman ps -q -f status=running -f name="mongodb") ]]; then
          echo "true"
          exit
        else
          echo -ne "\r$yellow$done    $1Error occured while connecting to Mongodb Container. Retries: $retry.\n\033[0m" >/dev/tty
          sleep 2
          ((retry++))
        fi
      done
    fi
    echo "false"
  else
    if [[ $(docker ps -q -a -f name="mongodb") ]]; then
      retry=0
      while [ $retry -lt 4 ]; do
        if [[ $(docker ps -q -f status=running -f name="mongodb") ]]; then
          echo "true"
          exit
        else
          echo -ne "\r$yellow$done    $1Error occured while connecting to Mongodb Container. Retries: $retry.\n\033[0m" >/dev/tty
          sleep 2
          ((retry++))
        fi
      done
    fi
    echo "false"
  fi
}

trap_ctrlC() {
  kill -9 $1
  exit 2
}

spin() {
  while [ 1 ]; do
    for i in "${spinner[@]}"; do
      echo -ne "\r$i    $1Collecting" >/dev/tty
      sleep 0.1
    done
  done
}

stop_spin() {
  trap "kill -9 $2" $(seq 0 15)
  echo -ne "\033[2K" >/dev/tty
  if [[ $3 == *"Error"* ]]; then
    echo -ne "\r$red$done    $1$3\n\033[0m" >/dev/tty
  else
    echo -ne "\r$green$done    $1Collected\n\033[0m" >/dev/tty
  fi
}

out_docker_core_logs() {
  OUT="core.log"
  spin "Core Logs\t\t" &
  pid=$!
  if [ $core_check == "false" ]; then
    stop_spin "Core Logs\t\t" $pid "Error: Core container is not up. Thus, not able to collect Core logs."
    return 1
  fi
  if [[ $RHEL: == *"Red Hat"* ]]; then
    $podman_compose logs core >&$OUT
  else
    $docker_compose logs --no-color core >&$OUT
  fi
  stop_spin "Core Logs\t\t" $pid "success"
  echo $OUT
}

out_docker_ui_logs() {
  OUT="ui.log"
  spin "UI Logs\t\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  if [ $ui_check == "false" ]; then
    stop_spin "UI Logs\t\t" $pid "Error: UI container is not up. Thus, not able to collect UI logs."
    return 1
  fi
  if [[ $RHEL: == *"Red Hat"* ]]; then
    $podman_compose logs ui >&$OUT
  else
    $docker_compose logs --no-color ui >&$OUT
  fi
  stop_spin "UI Logs\t\t" $pid "success"
  echo $OUT
}

out_docker_mongo_logs() {
  OUT="mongo.log"
  spin "MongoDB Logs\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  if [ $mongo_check == "false" ]; then
    stop_spin "MongoDB Logs\t" $pid "Error: MongoDB container is not up. Thus, not able to collect MongoDB logs."
    return 1
  fi
  if [[ $RHEL: == *"Red Hat"* ]]; then
    $podman_compose logs mongodb-primary >&$OUT
  else
    $docker_compose logs --no-color mongodb-primary >&$OUT
  fi
  stop_spin "MongoDB Logs\t" $pid "success"
  echo $OUT
}

out_docker_rabbitmq_logs() {
  OUT="rabbitmq.log"
  spin "RabbitMQ Logs\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  if [ $rabbitmq_check == "false" ]; then
    stop_spin "RabbitMQ Logs\t" $pid "Error: Rabbitmq container is not up. Thus, not able to collect rabbitmq logs."
    return 1
  fi
  if [[ $RHEL: == *"Red Hat"* ]]; then
    $podman_compose logs rabbitmq-stats | sed -e "s/\x1B[^m]*m//g" >&$OUT
  else
    $docker_compose logs --no-color rabbitmq-stats | sed -e "s/\x1B[^m]*m//g" >&$OUT
  fi
  stop_spin "RabbitMQ Logs\t" $pid "success"
  echo $OUT
}

out_stats() {
  OUT="system_stats.txt"
  spin "System Stats\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  echo -e "\n----------------------------------------------------------------------------------" >&$OUT
  echo " OS details" >>$OUT
  echo "> cat /etc/os-release" >>$OUT
  cat /etc/os-release >>$OUT
  echo -e "\n----------------------------------------------------------------------------------" >>$OUT
  echo " Kernel Version" >>$OUT
  echo "> uname -r" >>$OUT
  uname -r >>$OUT
  echo -e "\n----------------------------------------------------------------------------------" >>$OUT
  echo " Time details" >>$OUT
  echo "> timedatectl" >>$OUT
  timedatectl >>$OUT
  echo -e "\n----------------------------------------------------------------------------------" >>$OUT
  echo " Uptime of the Host" >>$OUT
  echo "> uptime" >>$OUT
  uptime >>$OUT
  echo -e "\n----------------------------------------------------------------------------------" >>$OUT
  echo "> lscpu" >>$OUT
  lscpu >>$OUT
  echo -e "\n----------------------------------------------------------------------------------" >>$OUT
  echo "> free -h" >>$OUT
  free -h >>$OUT
  echo -e "\n----------------------------------------------------------------------------------" >>$OUT
  echo "> df -h" >>$OUT
  df -h >>$OUT
  echo -e "\n----------------------------------------------------------------------------------" >>$OUT
  echo "> du -sh data/rabbitmq/data" >>$OUT
  du -sh data/rabbitmq/data >>$OUT
  stop_spin "System Stats\t" $pid "success"
  echo -e "\n----------------------------------------------------------------------------------" >>$OUT
  echo "> sysctl net.ipv4.ip_forward" >>$OUT
  sysctl net.ipv4.ip_forward >>$OUT
  echo -e "\n----------------------------------------------------------------------------------" >>$OUT
  device_type=$(lsblk -n -o ROTA $(df -P . | awk 'NR==2 {print $1}'))
  echo "Current working directory:" >> $OUT
  pwd >> $OUT
  if [ "$device_type" -eq 1 ]; then
      echo "Working directory storage type: HDD." >> $OUT
  elif [ "$device_type" -eq 0 ]; then
      echo "Working directory storage type: SSD." >> $OUT
  else
      echo "Working directory storage type: Unable to detect." >> $OUT
  fi
  echo $OUT
}

out_docker_stats() {
  if [[ $RHEL: == *"Red Hat"* ]]; then
    OUT="podman_stats.txt"
    spin "Podman Stats\t" &
    pid=$!
    trap "trap_ctrlC $pid" 2
    echo "> $podman_compose version" >&$OUT
    $podman_compose version &>>$OUT
    echo "---------------------------------------------------------------------------------" >>$OUT
    echo "> podman ps --all" >>$OUT
    podman ps --all >>$OUT
    echo "---------------------------------------------------------------------------------" >>$OUT
    echo "> podman system info" >>$OUT
    podman system info >>$OUT
    echo "---------------------------------------------------------------------------------" >>$OUT
    echo "> podman system df" >>$OUT
    podman system df >>$OUT
    stop_spin "Podman Stats\t" $pid "success"
    echo $OUT
  else
    OUT="docker_stats.txt"
    spin "Docker Stats\t" &
    pid=$!
    trap "trap_ctrlC $pid" 2
    echo "> docker ps --all" >&$OUT
    docker ps --all >>$OUT
    echo "---------------------------------------------------------------------------------" >>$OUT
    echo "> docker system info" >>$OUT
    docker system info >>$OUT
    echo "---------------------------------------------------------------------------------" >>$OUT
    echo "> $docker_compose version" >>$OUT
    $docker_compose version >>$OUT
    echo "---------------------------------------------------------------------------------" >>$OUT
    echo "> docker system df" >>$OUT
    docker system df >>$OUT
    stop_spin "Docker Stats\t" $pid "success"
    echo $OUT
  fi
}

out_plugin_version(){
  # Execute the command and filter the output
  command_output=$($2 exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.$1.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")

  # Store the plugin values in a variable
  plugin_values=$(echo "$command_output" | grep -Po "(?<=plugin: ')[^']*")

  # Replace spaces with newlines
  plugin_values=$(echo "$plugin_values" | tr ' ' '\n')

  # Traverse the space-separated values
  for plugin in $plugin_values; do
    # Replace "main" with "manifest.json"
    plugin_path=$(echo "$plugin" | sed 's/main$//' | tr '.' '/')

    # Get version using grep and manifest.json
    version=$($2 exec core cat "$plugin_path/manifest.json" | grep -Po '"version": "\K[^"]*' | cut -d'"' -f4)

    # Display results or store in a variable, as needed
    echo "plugin: $plugin" >> $OUT
    echo "version: $version" >> $OUT
    echo
  done
}

out_platform_details() {
  OUT="ce_platform_details.txt"
  spin "CE Details\t\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  if [[ $(check_mongo_running "CE Details\t\t") == "false" ]]; then
    stop_spin "CE Details\t\t" $pid "Error: MongoDB container is not running. Thus, not able to collect CE Details."
    return 1
  fi
  if [[ $RHEL: == *"Red Hat"* ]]; then
    SETTINGS=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{'databaseVersion': 1, 'uid':1, _id:0})" | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    CE_VERSION=$(echo "$SETTINGS" | grep -Po "(?<=databaseVersion: ')[^']*")
    echo "------- CE Platform Details -------" >&$OUT
    echo "> CE-Version: $CE_VERSION" >>$OUT

    CE_UID=$(echo "$SETTINGS" | grep -Po "(?<=uid: ')[^']*")
    echo "> CE-INSTALLATION ID: $CE_UID" >>$OUT

    echo -e "\n------- Proxy Details -------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{proxy:{scheme: {\$ifNull: ['\$proxy.scheme', null]}, server:{\$ifNull: ['\$proxy.server', null]}, username:{\$ifNull: ['\$proxy.username', null]}}, _id:0})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- Settings Details -------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{_id:0, logLevel:{\$ifNull: ['\$logLevel', null]}, logsCleanup:{\$ifNull: ['\$logsCleanup', null]}, ssoEnable:{\$ifNull: ['\$ssoEnable', null]}, forceAuth:{\$ifNull: ['\$forceAuth', null]}, secretsManagerSettings:{enabled:{\$ifNull: ['\$secretsManagerSettings.enabled', null]}, params:{authMethod:{\$ifNull: ['\$secretsManagerSettings.params.authMethod', null]}, provider:{\$ifNull: ['\$secretsManagerSettings.provider', null]}}}})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- Module Details -------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{platforms: {\$ifNull: ['\$platforms', null]}, _id:0})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- Plugin Repo Details -------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.repos.find({}, {name: 1, url:1, isDefault:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- CTE Plugin Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    active_indicators=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.indicators.countDocuments({active:true})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Active Indicator(s): $active_indicators" >> $OUT
    not_active_indicators=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.indicators.countDocuments({active:false})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Not Active Indicator(s): $not_active_indicators" >> $OUT

    echo -e "\n------- CTO Plugin Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.itsm_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    alerts=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.itsm_alerts.countDocuments({})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Alert(s): $alerts" >> $OUT
    
    echo -e "\n------- URE Plugin Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cre_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    users=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.cre_users.countDocuments({type:'user'})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "User(s): $users" >> $OUT
    hosts=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.cre_users.countDocuments({type:'host'})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Host(s): $hosts" >> $OUT

    echo -e "\n------- ARE Plugin Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.grc_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    applications=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.grc_applications.countDocuments({})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Application(s): $applications" >> $OUT

    echo -e "\n------- CLS Plugin Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cls_configurations.find({}, {name: 1, plugin:1, attributeMapping: 1, parameters: {transformData: 1}, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e 's/\"transformData\"/\"MappingTransformation\"/g' -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >> $OUT
    
    echo -e "\n------- Configured Plugin Version --------\n" >>$OUT
    out_plugin_version "configurations" "$podman_compose"
    out_plugin_version "cls_configurations" "$podman_compose"
    out_plugin_version "grc_configurations" "$podman_compose"
    out_plugin_version "cre_configurations" "$podman_compose"
    out_plugin_version "itsm_configurations" "$podman_compose"

    echo -e "\n------- CTE Business Rule Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cte_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    
    echo -e "\n------- CTO Business Rule Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.itsm_business_rules.find({}, {name: 1, muted:1, alertFilters:1, dedupeRules:1, muteRules:1, queues:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- URE Business Rule Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cre_business_rules.find({}, {name: 1, muted:1, userFilters:1, muteRules:1, actions:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
  
    echo -e "\n------- ARE Business Rule Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.grc_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
  
    echo -e "\n------- CLS Business Rule Details --------" >>$OUT
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cls_business_rules.find({}, {name: 1, muted:1, filters:1, siemMappings:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
  else
    SETTINGS=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{'databaseVersion': 1, 'uid':1, _id:0})" | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    CE_VERSION=$(echo "$SETTINGS" | grep -Po "(?<=databaseVersion: ')[^']*")

    echo "------- CE Platform Details -------" >&$OUT
    echo "> CE-Version: $CE_VERSION" >>$OUT

    CE_UID=$(echo "$SETTINGS" | grep -Po "(?<=uid: ')[^']*")
    echo "> CE-INSTALLATION ID: $CE_UID" >>$OUT

    echo -e "\n------- Proxy Details -------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{proxy:{scheme: {\$ifNull: ['\$proxy.scheme', null]}, server:{\$ifNull: ['\$proxy.server', null]}, username:{\$ifNull: ['\$proxy.username', null]}}, _id:0})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- Settings Details -------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{_id:0, logLevel:{\$ifNull: ['\$logLevel', null]}, logsCleanup:{\$ifNull: ['\$logsCleanup', null]}, ssoEnable:{\$ifNull: ['\$ssoEnable', null]}, forceAuth:{\$ifNull: ['\$forceAuth', null]}, secretsManagerSettings:{enabled:{\$ifNull: ['\$secretsManagerSettings.enabled', null]}, params:{authMethod:{\$ifNull: ['\$secretsManagerSettings.params.authMethod', null]}}, provider:{\$ifNull: ['\$secretsManagerSettings.provider', null]}}})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- Module Details -------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{platforms: {\$ifNull: ['\$platforms', null]}, _id:0})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- Plugin Repo Details -------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.repos.find({}, {name: 1, url:1, isDefault:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- CTE Plugin Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    active_indicators=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.indicators.countDocuments({active:true})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Active Indicator(s): $active_indicators" >> $OUT
    not_active_indicators=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.indicators.countDocuments({active:false})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Not Active Indicator(s): $not_active_indicators" >> $OUT

    echo -e "\n------- CTO Plugin Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.itsm_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    alerts=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.itsm_alerts.countDocuments({})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Alert(s): $alerts" >> $OUT

    echo -e "\n------- URE Plugin Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cre_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    users=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.cre_users.countDocuments({type:'user'})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "User(s): $users" >> $OUT
    hosts=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.cre_users.countDocuments({type:'host'})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Host(s): $hosts" >> $OUT

    echo -e "\n------- ARE Plugin Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.grc_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    applications=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.grc_applications.countDocuments({})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g")
    echo "Application(s): $applications" >> $OUT

    echo -e "\n------- CLS Plugin Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cls_configurations.find({}, {name: 1, plugin:1, attributeMapping: 1, parameters: {transformData: 1}, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" -e 's/\"transformData\"/\"MappingTransformation\"/g' >>$OUT
  
    echo -e "\n------- Configured Plugin Version --------\n" >>$OUT
    out_plugin_version "configurations" "$docker_compose"
    out_plugin_version "cls_configurations" "$docker_compose"
    out_plugin_version "grc_configurations" "$docker_compose"
    out_plugin_version "cre_configurations" "$docker_compose"
    out_plugin_version "itsm_configurations" "$docker_compose"

    echo -e "\n------- CTE Business Rule Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cte_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
    
    echo -e "\n------- CTO Business Rule Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.itsm_business_rules.find({}, {name: 1, muted:1, alertFilters:1, dedupeRules:1, muteRules:1, queues:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT

    echo -e "\n------- URE Business Rule Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cre_business_rules.find({}, {name: 1, muted:1, userFilters:1, muteRules:1, actions:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
  
    echo -e "\n------- ARE Business Rule Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.grc_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
  
    echo -e "\n------- CLS Business Rule Details --------" >>$OUT
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cls_business_rules.find({}, {name: 1, muted:1, filters:1, siemMappings:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT
  fi
  stop_spin "CE Details\t\t" $pid "success"
  echo $OUT
}

out_ce_logs() {
  OUT="ce_platform_logs.json"
  spin "CE Logs\t\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  if [[ $(check_mongo_running "CE Logs\t\t") == "false" ]]; then
    stop_spin "CE Logs\t\t" $pid "Error: MongoDB container is not running. Thus, not able to collect CE Logs."
    return 1
  fi
  if [[ $RHEL: == *"Red Hat"* ]]; then
    $podman_compose exec mongodb-primary mongoexport \
    --username cteadmin \
    --password $MAINTENANCE_PASSWORD \
    --authenticationDatabase cte \
    --db cte \
    --collection logs \
    --type json \
    --query "{}" \
    --fields createdAt,type,message,errorCode,details,_id \
    > $OUT
  else
    $docker_compose exec mongodb-primary mongoexport \
    --username cteadmin \
    --password $MAINTENANCE_PASSWORD \
    --authenticationDatabase cte \
    --db cte \
    --collection logs \
    --type json \
    --query "{}" \
    --fields createdAt,type,message,errorCode,details,_id \
    > $OUT
  fi
  stop_spin "CE Logs\t\t" $pid "success"
  echo $OUT
}

out_docker_images() {
  if [[ $RHEL: == *"Red Hat"* ]]; then
    OUT="podman_images.txt"
    spin "Podman Images\t" &
    pid=$!
    trap "trap_ctrlC $pid" 2
    podman inspect --format='{{.Id}} {{.Name}} {{.Image}}' $(podman ps -aq) >&"$OUT"
    stop_spin "Podman Images\t" $pid "success"
    echo $OUT
  else
    OUT="docker_images.txt"
    spin "Docker Images\t" &
    pid=$!
    trap "trap_ctrlC $pid" 2
    docker inspect --format='{{.Id}} {{.Name}} {{.Image}}' $(docker ps -aq) >&"$OUT"
    stop_spin "Docker Images\t" $pid "success"
    echo $OUT
  fi
}

out_kernel_logs() {
  OUT="kernel.log"
  spin "Kernel Logs\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  dmesg -k -T >&"$OUT"
  stop_spin "kernel Logs\t" $pid "success"
  echo $OUT
}

out_tenant_names() {
  OUT="tenant_names.txt"
  spin "Tenant Names\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  if [[ $(check_mongo_running "Tenant Names\t") == "false" ]]; then
    stop_spin "Tenant Names\t" $pid "Error: MongoDB container is not running. Thus, not able to collect Tenant Names."
    return 1
  fi
  if [[ $RHEL: == *"Red Hat"* ]]; then
    $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.tenants.find({},{_id:0, name:1, tenantName:1, initialRange:1, alert_types:1})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >> $OUT
  else
    $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.tenants.find({},{_id:0, name:1, tenantName:1, initialRange:1, alert_types:1})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >> $OUT
  fi
  stop_spin "Tenant Names\t" $pid "success"
  echo $OUT
}

out_rabbitmq_stats() {
  OUT="rabbitmq_stats.txt"
  spin "RabbitMQ Stats\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  if [ $rabbitmq_running == "false" ]; then
    stop_spin "RabbitMQ Stats\t" $pid "Error: Rabbitmq container is not running. Thus, not able to collect rabbitmq logs."
    return 1
  fi
  if [[ $RHEL: == *"Red Hat"* ]]; then
    $podman_compose exec rabbitmq-stats rabbitmqctl list_queues name messages message_bytes message_bytes_persistent | sed -e '/podman/d' -e '/exit code/d' >&$OUT
  else
    $docker_compose exec rabbitmq-stats rabbitmqctl list_queues name messages message_bytes message_bytes_persistent >&$OUT
  fi
  stop_spin "RabbitMQ Stats\t" $pid "success"
  echo $OUT
}

out_worker_details() {
  OUT="worker_status.txt"
  spin "Worker Status\t" &
  pid=$!
  trap "trap_ctrlC $pid" 2
  if [ $core_running == "false" ]; then
    stop_spin "Worker Status\t" $pid "Error: Core container is not running. Thus, not able to collect Worker Status."
    return 1
  fi
  if [[ $RHEL: == *"Red Hat"* ]]; then
    $podman_compose exec core celery -A netskope.common.celery.main inspect active | sed -e '/podman/d' -e '/exit code/d' -e "s/\x1B[^m]*m//g" >&$OUT
  else
    $docker_compose exec core celery -A netskope.common.celery.main inspect active | sed -e "s/\x1B[^m]*m//g" >&$OUT
  fi
  stop_spin "Worker Status\t" $pid "success"
  echo $OUT
}

# --- Body --------------------------------------------------------
#  SCRIPT LOGIC GOES HERE
output_generators=(
  out_platform_details
  out_stats
  out_docker_stats
  out_docker_images
  out_docker_core_logs
  out_docker_ui_logs
  out_docker_mongo_logs
  out_docker_rabbitmq_logs
  out_ce_logs
  out_kernel_logs
  out_tenant_names
  out_rabbitmq_stats
  out_worker_details
)

outputs=()

# check whether containers are up or not

if [[ $RHEL: == *"Red Hat"* ]]; then
  if [[ $(podman ps -q -a -f name="core") ]]; then
    core_check=true
    if [[ $(podman ps -q -f status=running -f name="core") ]]; then
      core_running=true
    fi
  fi
  if [[ $(podman ps -q -a -f name="ui") ]]; then
    ui_check=true
  fi
  if [[ $(podman ps -q -a -f name="mongodb") ]]; then
    mongo_check=true
  fi
  if [[ $(podman ps -q -a -f name="rabbitmq") ]]; then
    rabbitmq_check=true
    if [[ $(podman ps -q -f status=running -f name="rabbitmq") ]]; then
      rabbitmq_running=true
    fi
  fi
else
  if [[ $(docker ps -q -a -f name="core") ]]; then
    core_check=true
    if [[ $(docker ps -q -f status=running -f name="core") ]]; then
      core_running=true
    fi
  fi
  if [[ $(docker ps -q -a -f name="ui") ]]; then
    ui_check=true
  fi
  if [[ $(docker ps -q -a -f name="mongodb") ]]; then
    mongo_check=true
  fi
  if [[ $(docker ps -q -a -f name="rabbitmq") ]]; then
    rabbitmq_check=true
    if [[ $(docker ps -q -f status=running -f name="rabbitmq") ]]; then
      rabbitmq_running=true
    fi
  fi
fi

# check whether zip is installed or not
zip -v 1>/dev/null
if [ $? -eq 127 ]; then
  echo "Zip is not installed"
  exit 1
fi

echo -e "\n------ Running Diagnostic Script ------\n"

for i in ${output_generators[@]}; do
  output=$($i)
  outputs+="${output} "
done

# create zip file
echo "----- Creating zip file -----"
OUT="$(date -u '+%a %d %b %Y %T %Z' | tr " " "_").zip"

file_path="/.cloud_exchange_vm.marker"

storage_location="$OUT"

if [ -e "$file_path" ]; then
  storage_location="/opt/cloudexchange/$OUT"
fi

zip -r "$storage_location" ${outputs[@]}

echo "$storage_location"

# remove individual files
rm -rf ${outputs[@]}

# -----------------------------------------------------------------