proxlb

#!/usr/bin/env python3

# ProxLB
# ProxLB (re)balances VM workloads across nodes in Proxmox clusters.
# ProxLB obtains current metrics from all nodes within the cluster for
# further auto balancing by memory, disk or cpu and rebalances the VMs
# over all available nodes in a cluster by having an equal resource usage.
# Copyright (C) 2024 Florian Paul Azim Hoberg @gyptazy <gyptazy@gyptazy.ch>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import argparse
import configparser
import copy
import json
import logging
import os
try:
    import proxmoxer
    _imports = True
except ImportError:
    _imports = False
import random
import re
import requests
import socket
import sys
import time
import urllib3


# Constants
__appname__        = "ProxLB"
__version__        = "1.0.5"
__config_version__ = 3
__author__         = "Florian Paul Azim Hoberg <gyptazy@gyptazy.com> @gyptazy"
__errors__         = False


# Classes
## Logging class
class SystemdHandler(logging.Handler):
    """ Class to handle logging options. """
    PREFIX = {
        logging.CRITICAL: "<2> " + __appname__ + ": ",
        logging.ERROR: "<3> " + __appname__ + ": ",
        logging.WARNING: "<4> " + __appname__ + ": ",
        logging.INFO: "<6> " + __appname__ + ": ",
        logging.DEBUG: "<7> " + __appname__ + ": ",
        logging.NOTSET: "<7 " + __appname__ + ": ",
    }

    def __init__(self, stream=sys.stdout):
        self.stream = stream
        logging.Handler.__init__(self)

    def emit(self, record):
        try:
            msg = self.PREFIX[record.levelno] + self.format(record) + "\n"
            self.stream.write(msg)
            self.stream.flush()
        except Exception:
            self.handleError(record)


# Functions
def initialize_logger(log_level, update_log_verbosity=False):
    """ Initialize ProxLB logging handler. """
    info_prefix = 'Info: [logger]:'

    root_logger = logging.getLogger()
    root_logger.setLevel(log_level)

    if not update_log_verbosity:
        root_logger.addHandler(SystemdHandler())
        logging.info(f'{info_prefix} Logger got initialized.')
    else:
        logging.info(f'{info_prefix} Logger verbosity got updated to: {log_level}.')


def pre_validations(config_path, proxlb_config=False):
    """ Run pre-validations as sanity checks. """
    info_prefix = 'Info: [pre-validations]:'

    if proxlb_config:
        logging.info(f'{info_prefix} Validating ProxLB config file content.')
        __validate_config_content(proxlb_config)
        logging.info(f'{info_prefix} ProxLB config file content validation done.')
    else:
        logging.info(f'{info_prefix} Validating basic configuration.')
        __validate_imports()
        __validate_config_file(config_path)
        logging.info(f'{info_prefix} All pre-validations done.')


def post_validations():
    """ Run post-validations as sanity checks. """
    error_prefix = 'Error: [post-validations]:'
    info_prefix  = 'Info: [post-validations]:'

    if __errors__:
        logging.critical(f'{error_prefix} Not all post-validations succeeded. Please validate!')
    else:
        logging.info(f'{info_prefix} All post-validations succeeded.')


def validate_daemon(daemon, schedule):
    """ Validate if ProxLB runs as a daemon. """
    info_prefix  = 'Info: [daemon]:'

    if bool(int(daemon)):
        logging.info(f'{info_prefix} Running in daemon mode. Next run in {schedule} hours.')
        time.sleep(int(schedule) * 60 * 60)
    else:
        logging.info(f'{info_prefix} Not running in daemon mode. Quitting.')
        sys.exit(0)


def __validate_imports():
    """ Validate if all Python imports succeeded. """
    error_prefix = 'Error: [python-imports]:'
    info_prefix  = 'Info: [python-imports]:'

    if not _imports:
        logging.critical(f'{error_prefix} Could not import all dependencies. Please install "proxmoxer".')
        sys.exit(2)
    else:
        logging.info(f'{info_prefix} All required dependencies were imported.')


def __validate_config_file(config_path):
    """ Validate if all Python imports succeeded. """
    error_prefix = 'Error: [config]:'
    info_prefix  = 'Info: [config]:'

    if not os.path.isfile(config_path):
        logging.critical(f'{error_prefix} Could not find config file in: {config_path}.')
        sys.exit(2)
    else:
        logging.info(f'{info_prefix} Configuration file loaded from: {config_path}.')


def __validate_config_content(proxlb_config):
    """ Validate the user's config options. """
    error_prefix = 'Error: [config]:'
    info_prefix  = 'Info: [config]:'

    validate_bool_options = [
        'proxmox_api_ssl_v',
        'vm_balancing_enable',
        'vm_parallel_migrations',
        'storage_balancing_enable',
        'storage_parallel_migrations',
        'update_service',
        'api',
        'master_only',
        'daemon'
    ]

    for bool_val in validate_bool_options:
        if type(proxlb_config.get(bool_val, None)) == bool:
            logging.info(f'{info_prefix} Config option {bool_val} is in a correct format.')
        else:
            logging.critical(f'{error_prefix} Config option {bool_val} is incorrect: {proxlb_config.get(bool_val, None)}')
            sys.exit(2)

    validate_string_options = [
        'vm_balancing_method',
        'vm_balancing_mode',
        'vm_balancing_mode_option',
        'vm_balancing_type',
        'storage_balancing_method',
        'log_verbosity'
    ]

    whitelist_string_options = {
        'vm_balancing_method': ['memory', 'disk', 'cpu'],
        'vm_balancing_mode': ['used', 'assigned'],
        'vm_balancing_mode_option': ['bytes', 'percent'],
        'vm_balancing_type': ['vm', 'ct', 'all'],
        'storage_balancing_method': ['disk_space'],
        'log_verbosity': ['DEBUG', 'INFO', 'WARNING', 'CRITICAL']
    }

    for string_val in validate_string_options:
        if proxlb_config[string_val] in whitelist_string_options[string_val]:
            logging.info(f'{info_prefix} Config option {string_val} is in a correct format.')
        else:
            logging.critical(f'{error_prefix} Config option {string_val} is incorrect: {proxlb_config.get(string_val, None)}')
            sys.exit(2)


def initialize_args():
    """ Initialize given arguments for ProxLB. """
    argparser = argparse.ArgumentParser(description='ProxLB')
    argparser.add_argument('-c', '--config',      help='Path to config file',                                   type=str,            required=False)
    argparser.add_argument('-d', '--dry-run',     help='Perform a dry-run without doing any actions.',          action='store_true', required=False)
    argparser.add_argument('-j', '--json',        help='Return a JSON of the VM movement.',                     action='store_true', required=False)
    argparser.add_argument('-b', '--best-node',   help='Returns the best next node.',                           action='store_true', required=False)
    argparser.add_argument('-m', '--maintenance', help='Sets node to maintenance mode & moves workloads away.', type=str,            required=False)
    argparser.add_argument('-v', '--version',     help='Returns the current ProxLB version.',                   action='store_true', required=False)
    return argparser.parse_args()


def proxlb_output_version():
    """ Print ProxLB version information on CLI. """
    print(f'{__appname__} version {__version__}\nRequired config version: >= {__config_version__}')
    print('ProxLB support: https://github.com/gyptazy/ProxLB\nDeveloper: gyptazy.com')
    sys.exit(0)


def initialize_config_path(app_args):
    """ Initialize path to ProxLB config file. """
    info_prefix  = 'Info: [config]:'

    config_path = app_args.config
    if app_args.config is None:
        config_path = '/etc/proxlb/proxlb.conf'
        logging.info(f'{info_prefix} No config file provided. Falling back to: {config_path}.')
    else:
        logging.info(f'{info_prefix} Using config file: {config_path}.')
    return config_path


def initialize_config_options(config_path):
    """ Read configuration from given config file for ProxLB. """
    error_prefix  = 'Error: [config]:'
    info_prefix   = 'Info: [config]:'
    proxlb_config = {}

    try:
        config = configparser.ConfigParser()
        config.read(config_path)
        # Proxmox config
        proxlb_config['proxmox_api_host']            = config['proxmox']['api_host']
        proxlb_config['proxmox_api_user']            = config['proxmox']['api_user']
        proxlb_config['proxmox_api_pass']            = config['proxmox']['api_pass']
        proxlb_config['proxmox_api_ssl_v']           = config['proxmox']['verify_ssl']
        proxlb_config['proxmox_api_timeout']         = config['proxmox'].get('timeout', 10)
        # VM Balancing
        proxlb_config['vm_balancing_enable']         = config['vm_balancing'].get('enable', 1)
        proxlb_config['vm_balancing_method']         = config['vm_balancing'].get('method', 'memory')
        proxlb_config['vm_balancing_mode']           = config['vm_balancing'].get('mode', 'used')
        proxlb_config['vm_balancing_mode_option']    = config['vm_balancing'].get('mode_option', 'bytes')
        proxlb_config['vm_balancing_type']           = config['vm_balancing'].get('type', 'vm')
        proxlb_config['vm_balanciness']              = config['vm_balancing'].get('balanciness', 10)
        proxlb_config['vm_parallel_migrations']      = config['vm_balancing'].get('parallel_migrations', 1)
        proxlb_config['vm_maintenance_nodes']        = config['vm_balancing'].get('maintenance_nodes', '')
        proxlb_config['vm_ignore_nodes']             = config['vm_balancing'].get('ignore_nodes', '')
        proxlb_config['vm_ignore_vms']               = config['vm_balancing'].get('ignore_vms', '')
        proxlb_config['vm_enforce_affinity_groups']  = config['vm_balancing'].get('enforce_affinity_groups', 1)
        # Storage Balancing
        proxlb_config['storage_balancing_enable']    = config['storage_balancing'].get('enable', 0)
        proxlb_config['storage_balancing_method']    = config['storage_balancing'].get('method', 'disk_space')
        proxlb_config['storage_balanciness']         = config['storage_balancing'].get('balanciness', 10)
        proxlb_config['storage_parallel_migrations'] = config['storage_balancing'].get('parallel_migrations', 1)
        # Update Support
        proxlb_config['update_service']              = config['update_service'].get('enable', 0)
        # API
        proxlb_config['api']                         = config['update_service'].get('enable', 0)
        # Service
        proxlb_config['master_only']                 = config['service'].get('master_only', 0)
        proxlb_config['daemon']                      = config['service'].get('daemon', 1)
        proxlb_config['schedule']                    = config['service'].get('schedule', 24)
        proxlb_config['log_verbosity']               = config['service'].get('log_verbosity', 'CRITICAL')
        proxlb_config['config_version']              = config['service'].get('config_version', 2)
    except configparser.NoSectionError:
        logging.critical(f'{error_prefix} Could not find the required section.')
        sys.exit(2)
    except configparser.ParsingError:
        logging.critical(f'{error_prefix} Unable to parse the config file.')
        sys.exit(2)
    except KeyError:
        logging.critical(f'{error_prefix} Could not find the required options in config file.')
        sys.exit(2)

    # Normalize and update bools. Afterwards, validate minimum required config version.
    proxlb_config = __update_config_parser_bools(proxlb_config)
    validate_config_minimum_version(proxlb_config)
    logging.info(f'{info_prefix} Configuration file loaded.')

    return proxlb_config


def __update_config_parser_bools(proxlb_config):
    """ Update bools in config from configparser to real bools """
    info_prefix   = 'Info: [config-bool-converter]:'

    # Normalize and update config parser values to bools.
    for section, option_value in proxlb_config.items():

        if option_value in [1, '1', 'yes', 'Yes', 'true', 'True', 'enable']:
            logging.info(f'{info_prefix} Converting {section} to bool: True.')
            proxlb_config[section] = True

        if option_value in [0, '0', 'no', 'No', 'false', 'False', 'disable']:
            logging.info(f'{info_prefix} Converting {section} to bool: False.')
            proxlb_config[section] = False

    return proxlb_config


def validate_config_minimum_version(proxlb_config):
    """ Validate the minimum required config file for ProxLB """
    info_prefix   = 'Info: [config-version-validator]:'
    error_prefix  = 'Error: [config-version-validator]:'

    if int(proxlb_config['config_version']) < __config_version__:
        logging.error(f'{error_prefix} ProxLB config version {proxlb_config["config_version"]} is too low. Required: {__config_version__}.')
        print(f'{error_prefix} ProxLB config version {proxlb_config["config_version"]} is too low. Required: {__config_version__}.')
        sys.exit(1)
    else:
        logging.info(f'{info_prefix} ProxLB config version {proxlb_config["config_version"]} is fine. Required: {__config_version__}.')


def api_connect(proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v, proxmox_api_timeout):
    """ Connect and authenticate to the Proxmox remote API. """
    error_prefix = 'Error: [api-connection]:'
    warn_prefix  = 'Warning: [api-connection]:'
    info_prefix  = 'Info: [api-connection]:'
    proxmox_api_ssl_v = bool(int(proxmox_api_ssl_v))

    if not proxmox_api_ssl_v:
        requests.packages.urllib3.disable_warnings()
        logging.warning(f'{warn_prefix} API connection does not verify SSL certificate.')

    proxmox_api_host = __api_connect_get_host(proxmox_api_host)

    try:
        api_object = proxmoxer.ProxmoxAPI(proxmox_api_host, user=proxmox_api_user, password=proxmox_api_pass, verify_ssl=proxmox_api_ssl_v, timeout=int(proxmox_api_timeout))
    except proxmoxer.backends.https.AuthenticationError as proxmox_api_error:
        logging.critical(f'{error_prefix} Provided credentials do not work: {proxmox_api_error}')
        sys.exit(2)
    except urllib3.exceptions.NameResolutionError:
        logging.critical(f'{error_prefix} Could not resolve the given host: {proxmox_api_host}.')
        sys.exit(2)
    except requests.exceptions.ConnectTimeout:
        logging.critical(f'{error_prefix} Connection time out to host: {proxmox_api_host}.')
        sys.exit(2)
    except requests.exceptions.SSLError:
        logging.critical(f'{error_prefix} SSL certificate verification failed for host: {proxmox_api_host}.')
        sys.exit(2)

    logging.info(f'{info_prefix} API connection succeeded to host: {proxmox_api_host}.')
    return api_object


def __api_connect_get_host(proxmox_api_host):
    """ Validate if a list of API hosts got provided and pre-validate the hosts. """
    info_prefix  = 'Info: [api-connect-get-host]:'
    proxmox_port = 8006

    if ',' in proxmox_api_host:
        logging.info(f'{info_prefix} Multiple hosts for API connection are given. Testing hosts for further usage.')
        proxmox_api_host =  proxmox_api_host.split(',')

        # Validate all given hosts and check for responsive on Proxmox web port.
        for host in proxmox_api_host:
            logging.info(f'{info_prefix} Testing host {host} on port tcp/{proxmox_port}.')
            reachable = __api_connect_test_ipv4_host(host, proxmox_port)
            if reachable:
                return host
    else:
        logging.info(f'{info_prefix} Using host {proxmox_api_host} on port tcp/{proxmox_port}.')
        return proxmox_api_host


def __api_connect_test_ipv4_host(proxmox_api_host, port):
    """ Validate if a given host on the IPv4 management address is reachable. """
    error_prefix               = 'Error: [api-connect-test-host]:'
    info_prefix                = 'Info: [api-connect-test-host]:'
    proxmox_connection_timeout = 2

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.settimeout(proxmox_connection_timeout)
    logging.info(f'{info_prefix} Timeout for host {proxmox_api_host} is set to {proxmox_connection_timeout} seconds.')
    result = sock.connect_ex((proxmox_api_host,port))

    if result == 0:
        sock.close()
        logging.info(f'{info_prefix} Host {proxmox_api_host} is reachable on port tcp/{port}.')
        return True
    else:
        sock.close()
        logging.critical(f'{error_prefix} Host {proxmox_api_host} is unreachable on port tcp/{port}.')
        return False


def __api_connect_test_ipv6_host(proxmox_api_host, port):
    """ Validate if a given host on the IPv6 management address is reachable. """
    error_prefix               = 'Error: [api-connect-test-host]:'
    info_prefix                = 'Info: [api-connect-test-host]:'
    proxmox_connection_timeout = 2

    sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
    sock.settimeout(proxmox_connection_timeout)
    logging.info(f'{info_prefix} Timeout for host {proxmox_api_host} is set to {proxmox_connection_timeout}.')
    result = sock.connect_ex((proxmox_api_host,port))

    if result == 0:
        sock.close()
        logging.info(f'{info_prefix} Host {proxmox_api_host} is reachable on port tcp/{port}.')
        return True
    else:
        sock.close()
        logging.critical(f'{error_prefix} Host {proxmox_api_host} is unreachable on port tcp/{port}.')
        return False


def execute_rebalancing_only_by_master(api_object, master_only):
    """ Validate if balancing should only be done by the cluster master. Afterwards, validate if this node is the cluster master. """
    info_prefix  = 'Info: [only-on-master-executor]:'
    master_only = bool(int(master_only))

    if bool(int(master_only)):
        logging.info(f'{info_prefix} Master only rebalancing is defined. Starting validation.')
        cluster_master_node = get_cluster_master(api_object)
        cluster_master = validate_cluster_master(cluster_master_node)
        return cluster_master, master_only
    else:
        logging.info(f'{info_prefix} No master only rebalancing is defined. Skipping validation.')
        return False, master_only


def get_cluster_master(api_object):
    """ Get the current master of the Proxmox cluster. """
    error_prefix = 'Error: [cluster-master-getter]:'
    info_prefix  = 'Info: [cluster-master-getter]:'

    try:
        ha_status_object = api_object.cluster().ha().status().manager_status().get()
        logging.info(f'{info_prefix} Master node: {ha_status_object.get("manager_status", None).get("master_node", None)}')
    except urllib3.exceptions.NameResolutionError:
        logging.critical(f'{error_prefix} Could not resolve the API.')
        sys.exit(2)
    except requests.exceptions.ConnectTimeout:
        logging.critical(f'{error_prefix} Connection time out to API.')
        sys.exit(2)
    except requests.exceptions.SSLError:
        logging.critical(f'{error_prefix} SSL certificate verification failed for API.')
        sys.exit(2)

    cluster_master = ha_status_object.get("manager_status", None).get("master_node", None)

    if cluster_master:
        return cluster_master
    else:
        logging.critical(f'{error_prefix} Could not obtain cluster master. Please check your configuration and ensure HA services in Proxmox are enabled. Stopping.')
        sys.exit(2)


def validate_cluster_master(cluster_master):
    """ Validate if the current execution node is the cluster master. """
    info_prefix  = 'Info: [cluster-master-validator]:'

    node_executor_hostname = socket.gethostname()
    logging.info(f'{info_prefix} Node executor hostname is: {node_executor_hostname}')

    if node_executor_hostname != cluster_master:
        logging.info(f'{info_prefix} {node_executor_hostname} is not the cluster master ({cluster_master}).')
        return False
    else:
        return True


def get_node_statistics(api_object, ignore_nodes, maintenance_nodes):
    """ Get statistics of cpu, memory and disk for each node in the cluster. """
    info_prefix            = 'Info: [node-statistics]:'
    node_statistics        = {}
    ignore_nodes_list      =  ignore_nodes.split(',')
    maintenance_nodes_list =  maintenance_nodes.split(',')

    for node in api_object.nodes.get():
        if node['status'] == 'online':
            node_statistics[node['node']] = {}
            node_statistics[node['node']]['maintenance']                      = False
            node_statistics[node['node']]['ignore']                           = False
            node_statistics[node['node']]['cpu_total']                        = node['maxcpu']
            node_statistics[node['node']]['cpu_assigned']                     = 0
            node_statistics[node['node']]['cpu_assigned_percent']             = int((node_statistics[node['node']]['cpu_assigned']) / int(node_statistics[node['node']]['cpu_total']) * 100)
            node_statistics[node['node']]['cpu_assigned_percent_last_run']    = 0
            node_statistics[node['node']]['cpu_used']                         = node['cpu']
            node_statistics[node['node']]['cpu_free']                         = (node['maxcpu']) - (node['cpu'] * node['maxcpu'])
            node_statistics[node['node']]['cpu_free_percent']                 = int((node_statistics[node['node']]['cpu_free']) / int(node['maxcpu']) * 100)
            node_statistics[node['node']]['cpu_free_percent_last_run']        = 0
            node_statistics[node['node']]['memory_total']                     = node['maxmem']
            node_statistics[node['node']]['memory_assigned']                  = 0
            node_statistics[node['node']]['memory_assigned_percent']          = int((node_statistics[node['node']]['memory_assigned']) / int(node_statistics[node['node']]['memory_total']) * 100)
            node_statistics[node['node']]['memory_assigned_percent_last_run'] = 0
            node_statistics[node['node']]['memory_used']                      = node['mem']
            node_statistics[node['node']]['memory_free']                      = int(node['maxmem']) - int(node['mem'])
            node_statistics[node['node']]['memory_free_percent']              = int((node_statistics[node['node']]['memory_free']) / int(node['maxmem']) * 100)
            node_statistics[node['node']]['memory_free_percent_last_run']     = 0
            node_statistics[node['node']]['disk_total']                       = node['maxdisk']
            node_statistics[node['node']]['disk_assigned']                    = 0
            node_statistics[node['node']]['disk_assigned_percent']            = int((node_statistics[node['node']]['disk_assigned']) / int(node_statistics[node['node']]['disk_total']) * 100)
            node_statistics[node['node']]['disk_assigned_percent_last_run']   = 0
            node_statistics[node['node']]['disk_used']                        = node['disk']
            node_statistics[node['node']]['disk_free']                        = int(node['maxdisk']) - int(node['disk'])
            node_statistics[node['node']]['disk_free_percent']                = int((node_statistics[node['node']]['disk_free']) / int(node['maxdisk']) * 100)
            node_statistics[node['node']]['disk_free_percent_last_run']       = 0
            logging.info(f'{info_prefix} Added node {node["node"]}.')

            # Update node specific vars
            if node['node'] in maintenance_nodes_list:
                node_statistics[node['node']]['maintenance']                      = True
                logging.info(f'{info_prefix} Maintenance mode: {node["node"]} is set to maintenance mode.')

            if node['node'] in ignore_nodes_list:
                node_statistics[node['node']]['ignore']                           = True
                logging.info(f'{info_prefix} Ignore Node: {node["node"]} is set to be ignored.')

    logging.info(f'{info_prefix} Created node statistics.')
    return node_statistics


def get_vm_statistics(api_object, ignore_vms, balancing_type):
    """ Get statistics of cpu, memory and disk for each vm in the cluster. """
    info_prefix                 = 'Info: [vm-statistics]:'
    warn_prefix                 = 'Warn: [vm-statistics]:'
    vm_statistics               = {}
    ignore_vms_list             = ignore_vms.split(',')
    group_include               = None
    group_exclude               = None
    vm_ignore                   = None
    vm_ignore_wildcard          = False
    _vm_details_storage_allowed = ['ide', 'nvme', 'scsi', 'virtio', 'sata', 'rootfs']

    # Wildcard support: Initially validate if we need to honour
    # any wildcards within the vm_ignore list.
    vm_ignore_wildcard = __validate_ignore_vm_wildcard(ignore_vms)

    for node in api_object.nodes.get():

        # Get VM/CT objects only when the node is online and reachable.
        if node['status'] == 'online':

            # Add all virtual machines if type is vm or all.
            if balancing_type == 'vm' or balancing_type == 'all':
                for vm in api_object.nodes(node['node']).qemu.get():

                    # Get the VM tags from API.
                    vm_tags       = __get_vm_tags(api_object, node, vm['vmid'], 'vm')
                    if vm_tags is not None:
                        group_include, group_exclude, vm_ignore = __get_proxlb_groups(vm_tags)

                    # Get wildcard match for VMs to ignore if a wildcard pattern was
                    # previously found. Wildcards may slow down the task when using
                    # many patterns in the ignore list. Therefore, run this only if
                    # a wildcard pattern was found. We also do not need to validate
                    # this if the VM is already being ignored by a defined tag.
                    if vm_ignore_wildcard and not vm_ignore:
                        vm_ignore = __check_vm_name_wildcard_pattern(vm['name'], ignore_vms_list)

                    if vm['status'] == 'running' and vm['name'] not in ignore_vms_list and not vm_ignore:
                        vm_statistics[vm['name']] = {}
                        vm_statistics[vm['name']]['group_include']  = group_include
                        vm_statistics[vm['name']]['group_exclude']  = group_exclude
                        vm_statistics[vm['name']]['cpu_total']      = vm['cpus']
                        vm_statistics[vm['name']]['cpu_used']       = vm['cpu']
                        vm_statistics[vm['name']]['memory_total']   = vm['maxmem']
                        vm_statistics[vm['name']]['memory_used']    = vm['mem']
                        vm_statistics[vm['name']]['disk_total']     = vm['maxdisk']
                        vm_statistics[vm['name']]['disk_used']      = vm['disk']
                        vm_statistics[vm['name']]['vmid']           = vm['vmid']
                        vm_statistics[vm['name']]['node_parent']    = node['node']
                        vm_statistics[vm['name']]['node_rebalance'] = node['node']
                        vm_statistics[vm['name']]['storage']        = {}
                        vm_statistics[vm['name']]['type']           = 'vm'

                        # Get disk details of the related object.
                        _vm_details = api_object.nodes(node['node']).qemu(vm['vmid']).config.get()
                        logging.info(f'{info_prefix} Getting disk information for vm {vm["name"]}.')

                        for vm_detail_key, vm_detail_value in _vm_details.items():
                            # vm_detail_key_validator = re.sub('\d+$', '', vm_detail_key)
                            vm_detail_key_validator = re.sub(r'\d+$', '', vm_detail_key)

                            if vm_detail_key_validator in _vm_details_storage_allowed:
                                vm_statistics[vm['name']]['storage'][vm_detail_key] = {}
                                match = re.match(r'([^:]+):[^/]+/(.+),iothread=\d+,size=(\d+G)', _vm_details[vm_detail_key])

                                # Create an efficient match group and split the strings to assign them to the storage information.
                                if match:
                                    _volume    = match.group(1)
                                    _disk_name = match.group(2)
                                    _disk_size = match.group(3)

                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['name']               = _disk_name
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['device_name']        = vm_detail_key
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['volume']             = _volume
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['storage_parent']     = _volume
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['storage_rebalance']  = _volume
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['size']               = _disk_size[:-1]
                                    logging.info(f'{info_prefix} Added disk for {vm["name"]}: Name {_disk_name} on volume {_volume} with size {_disk_size}.')
                                else:
                                    logging.info(f'{info_prefix} No (or unsupported) disk(s) for {vm["name"]} found.')

                        logging.info(f'{info_prefix} Added vm {vm["name"]}.')

            # Add all containers if type is ct or all.
            if balancing_type == 'ct' or balancing_type == 'all':
                for vm in api_object.nodes(node['node']).lxc.get():

                    logging.warning(f'{warn_prefix} Rebalancing on LXC containers (CT) always requires them to shut down.')
                    logging.warning(f'{warn_prefix} {vm["name"]} is from type CT and cannot be live migrated!')
                    # Get the VM tags from API.
                    vm_tags       = __get_vm_tags(api_object, node, vm['vmid'], 'ct')
                    if vm_tags is not None:
                        group_include, group_exclude, vm_ignore = __get_proxlb_groups(vm_tags)

                    # Get wildcard match for VMs to ignore if a wildcard pattern was
                    # previously found. Wildcards may slow down the task when using
                    # many patterns in the ignore list. Therefore, run this only if
                    # a wildcard pattern was found. We also do not need to validate
                    # this if the VM is already being ignored by a defined tag.
                    if vm_ignore_wildcard and not vm_ignore:
                        vm_ignore = __check_vm_name_wildcard_pattern(vm['name'], ignore_vms_list)

                    if vm['status'] == 'running' and vm['name'] not in ignore_vms_list and not vm_ignore:
                        vm_statistics[vm['name']] = {}
                        vm_statistics[vm['name']]['group_include']  = group_include
                        vm_statistics[vm['name']]['group_exclude']  = group_exclude
                        vm_statistics[vm['name']]['cpu_total']      = vm['cpus']
                        vm_statistics[vm['name']]['cpu_used']       = vm['cpu']
                        vm_statistics[vm['name']]['memory_total']   = vm['maxmem']
                        vm_statistics[vm['name']]['memory_used']    = vm['mem']
                        vm_statistics[vm['name']]['disk_total']     = vm['maxdisk']
                        vm_statistics[vm['name']]['disk_used']      = vm['disk']
                        vm_statistics[vm['name']]['vmid']           = vm['vmid']
                        vm_statistics[vm['name']]['node_parent']    = node['node']
                        vm_statistics[vm['name']]['node_rebalance'] = node['node']
                        vm_statistics[vm['name']]['storage']        = {}
                        vm_statistics[vm['name']]['type']           = 'ct'

                        # Get disk details of the related object.
                        _vm_details = api_object.nodes(node['node']).lxc(vm['vmid']).config.get()
                        logging.info(f'{info_prefix} Getting disk information for vm {vm["name"]}.')

                        for vm_detail_key, vm_detail_value in _vm_details.items():
                            # vm_detail_key_validator = re.sub('\d+$', '', vm_detail_key)
                            vm_detail_key_validator = re.sub(r'\d+$', '', vm_detail_key)

                            if vm_detail_key_validator in _vm_details_storage_allowed:
                                vm_statistics[vm['name']]['storage'][vm_detail_key] = {}
                                match = re.match(r'(?P<volume>[^:]+):(?P<disk_name>[^,]+),size=(?P<disk_size>\S+)', _vm_details[vm_detail_key])

                                # Create an efficient match group and split the strings to assign them to the storage information.
                                if match:
                                    _volume    = match.group(1)
                                    _disk_name = match.group(2)
                                    _disk_size = match.group(3)

                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['name']               = _disk_name
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['device_name']        = vm_detail_key
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['volume']             = _volume
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['storage_parent']     = _volume
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['storage_rebalance']  = _volume
                                    vm_statistics[vm['name']]['storage'][vm_detail_key]['size']               = _disk_size[:-1]
                                    logging.info(f'{info_prefix} Added disk for {vm["name"]}: Name {_disk_name} on volume {_volume} with size {_disk_size}.')
                                else:
                                    logging.info(f'{info_prefix} No disks for {vm["name"]} found.')

                        logging.info(f'{info_prefix} Added vm {vm["name"]}.')

    logging.info(f'{info_prefix} Created VM statistics.')
    return vm_statistics


def update_node_statistics(node_statistics, vm_statistics):
    """ Update node statistics by VMs statistics. """
    info_prefix = 'Info: [node-update-statistics]:'
    warn_prefix = 'Warning: [node-update-statistics]:'

    for vm, vm_value in vm_statistics.items():
        node_statistics[vm_value['node_parent']]['cpu_assigned']            = node_statistics[vm_value['node_parent']]['cpu_assigned'] + int(vm_value['cpu_total'])
        node_statistics[vm_value['node_parent']]['cpu_assigned_percent']    = (node_statistics[vm_value['node_parent']]['cpu_assigned'] / node_statistics[vm_value['node_parent']]['cpu_total']) * 100
        node_statistics[vm_value['node_parent']]['memory_assigned']         = node_statistics[vm_value['node_parent']]['memory_assigned'] + int(vm_value['memory_total'])
        node_statistics[vm_value['node_parent']]['memory_assigned_percent'] = (node_statistics[vm_value['node_parent']]['memory_assigned'] / node_statistics[vm_value['node_parent']]['memory_total']) * 100
        node_statistics[vm_value['node_parent']]['disk_assigned']           = node_statistics[vm_value['node_parent']]['disk_assigned'] + int(vm_value['disk_total'])
        node_statistics[vm_value['node_parent']]['disk_assigned_percent']   = (node_statistics[vm_value['node_parent']]['disk_assigned'] / node_statistics[vm_value['node_parent']]['disk_total']) * 100

        if node_statistics[vm_value['node_parent']]['cpu_assigned_percent'] > 99:
            logging.warning(f'{warn_prefix} Node {vm_value["node_parent"]} is overprovisioned for CPU by {int(node_statistics[vm_value["node_parent"]]["cpu_assigned_percent"])}%.')

        if node_statistics[vm_value['node_parent']]['memory_assigned_percent'] > 99:
            logging.warning(f'{warn_prefix} Node {vm_value["node_parent"]} is overprovisioned for memory by {int(node_statistics[vm_value["node_parent"]]["memory_assigned_percent"])}%.')

        if node_statistics[vm_value['node_parent']]['disk_assigned_percent'] > 99:
            logging.warning(f'{warn_prefix} Node {vm_value["node_parent"]} is overprovisioned for disk by {int(node_statistics[vm_value["node_parent"]]["disk_assigned_percent"])}%.')

    logging.info(f'{info_prefix} Updated node resource assignments by all VMs.')
    logging.debug('node_statistics')
    return node_statistics


def get_storage_statistics(api_object):
    """ Get statistics of all storage in the cluster. """
    info_prefix        = 'Info: [storage-statistics]:'
    storage_whitelist  = ['nfs']
    storage_statistics = {}

    for node in api_object.nodes.get():

        for storage in api_object.nodes(node['node']).storage.get():

            # Only add enabled and active storage repositories that might be suitable for further
            # storage balancing.
            if storage['enabled'] and storage['active'] and storage['shared'] and storage['type'] in storage_whitelist:
                storage_statistics[storage['storage']]                          = {}
                storage_statistics[storage['storage']]['name']                  = storage['storage']
                storage_statistics[storage['storage']]['total']                 = storage['total']
                storage_statistics[storage['storage']]['used']                  = storage['used']
                storage_statistics[storage['storage']]['used_percent']          = storage['used'] / storage['total'] * 100
                storage_statistics[storage['storage']]['used_percent_last_run'] = 0
                storage_statistics[storage['storage']]['free']                  = storage['total'] - storage['used']
                storage_statistics[storage['storage']]['free_percent']          = storage_statistics[storage['storage']]['free'] / storage['total'] * 100
                storage_statistics[storage['storage']]['used_fraction']         = storage['used_fraction']
                storage_statistics[storage['storage']]['type']                  = storage['type']
                storage_statistics[storage['storage']]['content']               = storage['content']
                storage_statistics[storage['storage']]['usage_type']            = ''

                # Split the Proxmox returned values to a list and validate the supported
                # types of the underlying storage for further migrations.
                storage_content_list =  storage['content'].split(',')
                usage_ct             = False
                usage_vm             = False

                if 'rootdir' in storage_content_list:
                    usage_ct = True
                    storage_statistics[storage['storage']]['usage_type']        = 'ct'
                    logging.info(f'{info_prefix} Storage {storage["storage"]} support CTs.')

                if 'images' in storage_content_list:
                    usage_vm = True
                    storage_statistics[storage['storage']]['usage_type']        = 'vm'
                    logging.info(f'{info_prefix} Storage {storage["storage"]} support VMs.')

                if usage_ct and usage_vm:
                    storage_statistics[storage['storage']]['usage_type']        = 'all'
                    logging.info(f'{info_prefix} Updateing storage {storage["storage"]} support to CTs and VMs.')

                logging.info(f'{info_prefix} Added storage {storage["storage"]}.')

    logging.info(f'{info_prefix} Created storage statistics.')
    return storage_statistics


def __validate_ignore_vm_wildcard(ignore_vms):
    """ Validate if a wildcard is used for ignored VMs. """
    if '*' in ignore_vms:
        return True


def __check_vm_name_wildcard_pattern(vm_name, ignore_vms_list):
    """ Validate if the VM name is in the ignore list pattern included. """
    for ignore_vm in ignore_vms_list:
        if '*' in ignore_vm:
            if ignore_vm[:-1] in vm_name:
                return True


def __get_vm_tags(api_object, node, vmid, balancing_type):
    """ Get tags for a VM/CT for a given VMID. """
    info_prefix = 'Info: [api-get-vm-tags]:'

    if balancing_type == 'vm':
        vm_config = api_object.nodes(node['node']).qemu(vmid).config.get()

    if balancing_type == 'ct':
        vm_config = api_object.nodes(node['node']).lxc(vmid).config.get()

    if vm_config.get("tags", None) is None:
        logging.info(f'{info_prefix} Got no VM/CT tag for VM {vm_config.get("name", None)} from API.')
    else:
        logging.info(f'{info_prefix} Got VM/CT tag {vm_config.get("tags", None)} for VM  {vm_config.get("name", None)} from API.')
    return vm_config.get('tags', None)


def __get_proxlb_groups(vm_tags):
    """ Get ProxLB related include and exclude groups. """
    info_prefix   = 'Info: [api-get-vm-include-exclude-tags]:'
    group_include = None
    group_exclude = None
    vm_ignore     = None

    group_list = re.split(";", vm_tags)
    for group in group_list:

        if group.startswith('plb_include_'):
            logging.info(f'{info_prefix} Got PLB include group.')
            group_include = group

        if group.startswith('plb_affinity_'):
            logging.info(f'{info_prefix} Got PLB include group.')
            group_include = group

        if group.startswith('plb_exclude_'):
            logging.info(f'{info_prefix} Got PLB exclude group.')
            group_exclude = group

        if group.startswith('plb_antiaffinity_'):
            logging.info(f'{info_prefix} Got PLB exclude group.')
            group_exclude = group

        if group.startswith('plb_ignore_vm'):
            logging.info(f'{info_prefix} Got PLB ignore group.')
            vm_ignore = True

    return group_include, group_exclude, vm_ignore


def balancing_vm_calculations(balancing_method, balancing_mode, balancing_mode_option, node_statistics, vm_statistics, balanciness, app_args, rebalance, processed_vms):
    """ Calculate re-balancing of VMs on present nodes across the cluster. """
    info_prefix  = 'Info: [rebalancing-vm-calculator]:'

    # Validate for a supported balancing method, mode and if rebalancing is required.
    __validate_balancing_method(balancing_method)
    __validate_balancing_mode(balancing_mode)
    __validate_vm_statistics(vm_statistics)
    rebalance = __validate_balanciness(balanciness, balancing_method, balancing_mode, node_statistics)

    # Run rebalancing calculations.
    if rebalance:
        # Get most used/assigned resources of the VM and the most free or less allocated node.
        resources_vm_most_used, processed_vms  = __get_most_used_resources_vm(balancing_method, balancing_mode, vm_statistics, processed_vms)
        resources_node_most_free               = __get_most_free_resources_node(balancing_method, balancing_mode, balancing_mode_option, node_statistics)

        # If most used vm is on most free node then skip it and get another one.
        while resources_vm_most_used[1]['node_parent'] == resources_node_most_free[0] and len(processed_vms) < len(vm_statistics):
            resources_vm_most_used, processed_vms  = __get_most_used_resources_vm(balancing_method, balancing_mode, vm_statistics, processed_vms)
            logging.debug(f'{info_prefix} processed {len(processed_vms)} out of {len(vm_statistics)} vms.')

        # Update resource statistics for VMs and nodes.
        node_statistics, vm_statistics         = __update_vm_resource_statistics(resources_vm_most_used, resources_node_most_free,
                                                                             vm_statistics, node_statistics, balancing_method, balancing_mode)

        # Start recursion until we do not have any needs to rebalance anymore.
        balancing_vm_calculations(balancing_method, balancing_mode, balancing_mode_option, node_statistics, vm_statistics, balanciness, app_args, rebalance, processed_vms)

    # If only best node argument set we simply return the next best node for VM
    # and CT placement on the CLI and stop ProxLB.
    if app_args.best_node:
        logging.info(f'{info_prefix} Only best next node for new VM & CT placement requsted.')
        best_next_node = __get_most_free_resources_node(balancing_method, balancing_mode, balancing_mode_option, node_statistics)
        print(best_next_node[0])
        logging.info(f'{info_prefix} Best next node for VM & CT placement: {best_next_node[0]}')
        sys.exit(0)

    logging.info(f'{info_prefix} Balancing calculations done.')
    return node_statistics, vm_statistics


def balancing_vm_maintenance(proxlb_config, app_args, node_statistics, vm_statistics):
    """ Calculate re-balancing of VMs that need to be moved away from maintenance nodes. """
    info_prefix            = 'Info: [rebalancing-maintenance-vm-calculator]:'
    maintenance_nodes_list = proxlb_config['vm_maintenance_nodes'].split(',')
    nodes_present          = list(node_statistics.keys())
    balancing_method       = proxlb_config['vm_balancing_method']
    balancing_mode         = proxlb_config['vm_balancing_mode']
    balancing_mode_option  = proxlb_config['vm_balancing_mode_option']

    # Merge maintenance nodes from config and cli args.
    if app_args.maintenance is not None:
        logging.info(f'{info_prefix} Maintenance nodes from CLI arg and config will be merged.')
        maintenance_nodes_list = maintenance_nodes_list + app_args.maintenance.split(',')

    # Ensure that only existing nodes in the cluster will be used.
    if len(proxlb_config['vm_maintenance_nodes']) > 1:
        maintenance_nodes_list = set(maintenance_nodes_list) & set(nodes_present)
        logging.info(f'{info_prefix} Maintenance mode for the following hosts defined: {maintenance_nodes_list}')
    else:
        logging.info(f'{info_prefix} No nodes for maintenance mode defined.')
        return node_statistics, vm_statistics

    for node_name in maintenance_nodes_list:
        node_vms = list(filter(lambda item: item[0] if item[1]['node_parent'] == node_name else [], vm_statistics.items()))
        # Update resource statistics for VMs and nodes.
        for vm in node_vms:
            resources_node_most_free        = __get_most_free_resources_node(balancing_method, balancing_mode, balancing_mode_option, node_statistics)
            node_statistics, vm_statistics  = __update_vm_resource_statistics(vm, resources_node_most_free, vm_statistics, node_statistics, balancing_method, balancing_mode)

    return node_statistics, vm_statistics


def __validate_balancing_method(balancing_method):
    """ Validate for valid and supported balancing method. """
    error_prefix = 'Error: [balancing-method-validation]:'
    info_prefix  = 'Info: [balancing-method-validation]:'

    if balancing_method not in ['memory', 'disk', 'cpu']:
        logging.error(f'{error_prefix} Invalid balancing method: {balancing_method}')
        sys.exit(2)
    else:
        logging.info(f'{info_prefix} Valid balancing method: {balancing_method}')


def __validate_balancing_mode(balancing_mode):
    """ Validate for valid and supported balancing mode. """
    error_prefix = 'Error: [balancing-mode-validation]:'
    info_prefix  = 'Info: [balancing-mode-validation]:'

    if balancing_mode not in ['used', 'assigned']:
        logging.error(f'{error_prefix} Invalid balancing method: {balancing_mode}')
        sys.exit(2)
    else:
        logging.info(f'{info_prefix} Valid balancing method: {balancing_mode}')


def __validate_vm_statistics(vm_statistics):
    """ Validate for at least a single object of type CT/VM to rebalance. """
    error_prefix = 'Error: [balancing-vm-stats-validation]:'

    if len(vm_statistics) == 0:
        logging.error(f'{error_prefix} Not a single CT/VM found in cluster.')
        sys.exit(1)


def __validate_balanciness(balanciness, balancing_method, balancing_mode, node_statistics):
    """ Validate for balanciness to ensure further rebalancing is needed. """
    info_prefix                   = 'Info: [balanciness-validation]:'
    node_resource_percent_list    = []
    node_assigned_percent_match = []

    # Remap balancing mode to get the related values from nodes dict.
    if balancing_mode == 'used':
        node_resource_selector = 'free'
    if balancing_mode == 'assigned':
        node_resource_selector = 'assigned'

    for node_name, node_info in node_statistics.items():

        # Save information of nodes from current run to compare them in the next recursion.
        if node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent_last_run'] == node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent']:
            node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent_match'] = True
        else:
            node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent_match'] = False
        # Update value to the current value of the recursion run.
        node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent_last_run'] = node_statistics[node_name][f'{balancing_method}_{node_resource_selector}_percent']

        # If all node resources are unchanged, the recursion can be left.
        for key, value in node_statistics.items():
            node_assigned_percent_match.append(value.get(f'{balancing_method}_{node_resource_selector}_percent_match', False))

        if False not in node_assigned_percent_match:
            return False

        # Add node information to resource list.
        if not node_statistics[node_name]['maintenance']:
            node_resource_percent_list.append(int(node_info[f'{balancing_method}_{node_resource_selector}_percent']))
            logging.debug(f'{info_prefix} Node: {node_name} with values: {node_info}')

    # Create a sorted list of the delta + balanciness between the node resources.
    node_resource_percent_list_sorted    = sorted(node_resource_percent_list)
    node_lowest_percent                  = node_resource_percent_list_sorted[0]
    node_highest_percent                 = node_resource_percent_list_sorted[-1]

    # Validate if the recursion should  be proceeded for further rebalancing.
    if (int(node_lowest_percent) + int(balanciness)) < int(node_highest_percent):
        logging.info(f'{info_prefix} Rebalancing for {balancing_method} is needed. Highest usage: {int(node_highest_percent)}% | Lowest usage: {int(node_lowest_percent)}%.')
        return True
    else:
        logging.info(f'{info_prefix} Rebalancing for {balancing_method} is not needed. Highest usage: {int(node_highest_percent)}% | Lowest usage: {int(node_lowest_percent)}%.')
        return False


def __get_most_used_resources_vm(balancing_method, balancing_mode, vm_statistics, processed_vms):
    """ Get and return the most used resources of a VM by the defined balancing method. """
    info_prefix = 'Info: [get-most-used-resources-vm]:'

    # Remap balancing mode to get the related values from nodes dict.
    if balancing_mode == 'used':
        vm_resource_selector = 'used'
    if balancing_mode == 'assigned':
        vm_resource_selector = 'total'

    vm = max(vm_statistics.items(), key=lambda item: item[1][f'{balancing_method}_{vm_resource_selector}'] if item[0] not in processed_vms else -float('inf'))
    processed_vms.append(vm[0])

    logging.info(f'{info_prefix} {vm}')
    return vm, processed_vms


def __get_most_free_resources_node(balancing_method, balancing_mode, balancing_mode_option, node_statistics):
    """ Get and return the most free resources of a node by the defined balancing method. """
    info_prefix = 'Info: [get-most-free-resources-nodes]:'

    # Return the node information based on the balancing mode.
    if balancing_mode == 'used' and balancing_mode_option == 'bytes':
        node = max(node_statistics.items(), key=lambda item: item[1][f'{balancing_method}_free'] if not item[1]['maintenance'] else -float('inf'))
    if balancing_mode == 'used' and balancing_mode_option == 'percent':
        node = max(node_statistics.items(), key=lambda item: item[1][f'{balancing_method}_free_percent'] if not item[1]['maintenance'] else -float('inf'))
    if balancing_mode == 'assigned':
        node = min(node_statistics.items(), key=lambda item: item[1][f'{balancing_method}_assigned'] if not item[1]['maintenance'] and (item[1][f'{balancing_method}_assigned_percent'] > 0 or item[1][f'{balancing_method}_assigned_percent'] < 100) else -float('inf'))

    logging.info(f'{info_prefix} {node}')
    return node


def __update_vm_resource_statistics(resource_highest_used_resources_vm, resource_highest_free_resources_node, vm_statistics, node_statistics, balancing_method, balancing_mode):
    """ Update VM and node resource statistics. """
    info_prefix = 'Info: [rebalancing-resource-statistics-update]:'

    if resource_highest_used_resources_vm[1]['node_parent'] != resource_highest_free_resources_node[0]:
        vm_name            = resource_highest_used_resources_vm[0]
        vm_node_parent     = resource_highest_used_resources_vm[1]['node_parent']
        vm_node_rebalance  = resource_highest_free_resources_node[0]
        vm_resource_used   = vm_statistics[resource_highest_used_resources_vm[0]][f'{balancing_method}_used']
        vm_resource_total  = vm_statistics[resource_highest_used_resources_vm[0]][f'{balancing_method}_total']

        # Update dictionaries for new values
        # Assign new rebalance node to vm
        vm_statistics[vm_name]['node_rebalance'] = vm_node_rebalance

        logging.info(f'{info_prefix} Moving {vm_name} from {vm_node_parent} to {vm_node_rebalance}')

        # Recalculate values for nodes
        ## Add freed resources to old parent node
        node_statistics[vm_node_parent][f'{balancing_method}_used']                = int(node_statistics[vm_node_parent][f'{balancing_method}_used']) - int(vm_resource_used)
        node_statistics[vm_node_parent][f'{balancing_method}_free']                = int(node_statistics[vm_node_parent][f'{balancing_method}_free']) + int(vm_resource_used)
        node_statistics[vm_node_parent][f'{balancing_method}_free_percent']        = int(int(node_statistics[vm_node_parent][f'{balancing_method}_free']) / int(node_statistics[vm_node_parent][f'{balancing_method}_total']) * 100)
        node_statistics[vm_node_parent][f'{balancing_method}_assigned']            = int(node_statistics[vm_node_parent][f'{balancing_method}_assigned']) - int(vm_resource_total)
        node_statistics[vm_node_parent][f'{balancing_method}_assigned_percent']    = int(int(node_statistics[vm_node_parent][f'{balancing_method}_assigned']) / int(node_statistics[vm_node_parent][f'{balancing_method}_total']) * 100)

        ## Removed newly allocated resources to new rebalanced node
        node_statistics[vm_node_rebalance][f'{balancing_method}_used']             = int(node_statistics[vm_node_rebalance][f'{balancing_method}_used']) + int(vm_resource_used)
        node_statistics[vm_node_rebalance][f'{balancing_method}_free']             = int(node_statistics[vm_node_rebalance][f'{balancing_method}_free']) - int(vm_resource_used)
        node_statistics[vm_node_rebalance][f'{balancing_method}_free_percent']     = int(int(node_statistics[vm_node_rebalance][f'{balancing_method}_free']) / int(node_statistics[vm_node_rebalance][f'{balancing_method}_total']) * 100)
        node_statistics[vm_node_rebalance][f'{balancing_method}_assigned']         = int(node_statistics[vm_node_rebalance][f'{balancing_method}_assigned']) + int(vm_resource_total)
        node_statistics[vm_node_rebalance][f'{balancing_method}_assigned_percent'] = int(int(node_statistics[vm_node_rebalance][f'{balancing_method}_assigned']) / int(node_statistics[vm_node_rebalance][f'{balancing_method}_total']) * 100)

    logging.info(f'{info_prefix} Updated VM and node statistics.')
    return node_statistics, vm_statistics


def __get_vm_tags_include_groups(vm_statistics, node_statistics, balancing_method, balancing_mode):
    """ Get VMs tags for include groups. """
    info_prefix = 'Info: [rebalancing-tags-group-include]:'
    tags_include_vms = {}
    processed_vm = []

    # Create groups of tags with belongings hosts.
    for vm_name, vm_values in vm_statistics.items():
        if vm_values.get('group_include', None):
            if not tags_include_vms.get(vm_values['group_include'], None):
                tags_include_vms[vm_values['group_include']] = [vm_name]
            else:
                tags_include_vms[vm_values['group_include']] = tags_include_vms[vm_values['group_include']] + [vm_name]

    # Update the VMs to the corresponding node to their group assignments.
    for group, vm_names in tags_include_vms.items():
        # Do not take care of tags that have only a single host included.
        if len(vm_names) < 2:
            logging.info(f'{info_prefix} Only one host in group assignment.')
            return node_statistics, vm_statistics

        vm_node_rebalance = False
        logging.info(f'{info_prefix} Create include groups of VM hosts.')
        for vm_name in vm_names:
            if vm_name not in processed_vm:
                if not vm_node_rebalance:
                    vm_node_rebalance = vm_statistics[vm_name]['node_rebalance']
                else:
                    _mocked_vm_object = (vm_name, vm_statistics[vm_name])
                    node_statistics, vm_statistics = __update_vm_resource_statistics(_mocked_vm_object, [vm_node_rebalance], vm_statistics, node_statistics, balancing_method, balancing_mode)
            processed_vm.append(vm_name)

    return node_statistics, vm_statistics


def __get_vm_tags_exclude_groups(vm_statistics, node_statistics, balancing_method, balancing_mode):
    """ Get VMs tags for exclude groups. """
    info_prefix      = 'Info: [rebalancing-tags-group-exclude]:'
    tags_exclude_vms = {}

    # Create groups of tags with belongings hosts.
    for vm_name, vm_values in vm_statistics.items():
        if vm_values.get('group_exclude', None):
            if not tags_exclude_vms.get(vm_values['group_exclude'], None):
                tags_exclude_vms[vm_values['group_exclude']]               = {}
                tags_exclude_vms[vm_values['group_exclude']]['nodes_used'] = []
                tags_exclude_vms[vm_values['group_exclude']]['nodes_used'].append(vm_statistics[vm_name]['node_rebalance'])
                tags_exclude_vms[vm_values['group_exclude']]['vms']        = [vm_name]
            else:
                tags_exclude_vms[vm_values['group_exclude']]['vms']        = tags_exclude_vms[vm_values['group_exclude']]['vms'] + [vm_name]
                tags_exclude_vms[vm_values['group_exclude']]['nodes_used'].append(vm_statistics[vm_name]['node_rebalance'])

    # Evaluate all VMs assigned for each exclude groups and validate that they will be moved to another random node.
    # However, if there are still more VMs than nodes we need to deal with it.
    for exclude_group, group_values in tags_exclude_vms.items():

        group_values['nodes_used'] = []
        for vm in group_values['vms']:

            proceed = True
            counter = 0

            while proceed:

                if vm_statistics[vm]['node_rebalance'] in group_values['nodes_used']:
                    # Find another possible new target node if possible by randomly get any node from
                    # the cluster and validating if this is already used for this anti-affinity group.
                    logging.info(f'{info_prefix} Rebalancing of VM {vm} is needed due to anti-affinity group policy.')
                    random_node, counter, proceed = __get_random_node(counter, node_statistics, vm)

                    if random_node not in group_values['nodes_used']:
                        logging.info(f'{info_prefix} New random node {random_node} has not yet been used for the anti-affinity group {exclude_group}.')
                        group_values['nodes_used'].append(random_node)
                        logging.info(f'{info_prefix} New random node {random_node} has been added as an already used node to the anti-affinity group {exclude_group}.')
                        logging.info(f'{info_prefix} VM {vm} switched node from {vm_statistics[vm]["node_rebalance"]} to {random_node} due to the anti-affinity group {exclude_group}.')
                        vm_statistics[vm]['node_rebalance'] = random_node

                else:
                    # Add the used node to the list for the anti-affinity group to ensure no
                    # other VM with the same anti-affinity group will use it (if possible).
                    logging.info(f'{info_prefix} Node {vm_statistics[vm]["node_rebalance"]} has been added as an already used node to the anti-affinity group {exclude_group}.')
                    logging.info(f'{info_prefix} No rebalancing for VM {vm} needed due to any anti-affinity group policies.')
                    group_values['nodes_used'].append(vm_statistics[vm]['node_rebalance'])
                    proceed = False

    return node_statistics, vm_statistics


def __get_random_node(counter, node_statistics, vm):
    """ Get a random node within the Proxmox cluster. """
    warning_prefix = 'Warning: [random-node-getter]:'
    info_prefix    = 'Info: [random-node-getter]:'

    counter = counter + 1
    random_node = None
    if counter < 30:
        random_node = random.choice(list(node_statistics.keys()))
        logging.info(f'{info_prefix} New random node {random_node} evaluated for vm {vm} in run {counter}.')
        return random_node, counter, False
    else:
        logging.warning(f'{warning_prefix} Reached limit for random node evaluation for vm {vm}. Unable to find a suitable new node.')
        return random_node, counter, False


def __wait_job_finalized(api_object, node_name, job_id, counter):
    """ Wait for a job to be finalized. """
    error_prefix = 'Error: [job-status-getter]:'
    info_prefix  = 'Info: [job-status-getter]:'

    logging.info(f'{info_prefix} Getting job status for job {job_id}.')
    task = api_object.nodes(node_name).tasks(job_id).status().get()
    logging.info(f'{info_prefix} {task}')

    if task['status'] == 'running':
        logging.info(f'{info_prefix} Validating job {job_id} for the {counter} run.')

        # Do not run for infinity this recursion and fail when reaching the limit.
        if counter == 300:
            logging.critical(f'{error_prefix} The job {job_id} on node {node_name} did not finished in time for migration.')

        time.sleep(5)
        counter = counter + 1
        logging.info(f'{info_prefix} Revalidating job {job_id} in a next run.')
        __wait_job_finalized(api_object, node_name, job_id, counter)

    logging.info(f'{info_prefix} Job {job_id} for migration from {node_name} terminiated succesfully.')


def __run_vm_rebalancing(api_object, _vm_vm_statistics, app_args, parallel_migrations):
    """ Run & execute the VM rebalancing via API. """
    error_prefix = 'Error: [vm-rebalancing-executor]:'
    info_prefix  = 'Info: [vm-rebalancing-executor]:'

    # Remove VMs/CTs that do not have a new node location.
    vms_to_remove = [vm_name for vm_name, vm_info in _vm_vm_statistics.items() if 'node_rebalance' in vm_info and vm_info['node_rebalance'] == vm_info.get('node_parent')]
    for vm_name in vms_to_remove:
        del _vm_vm_statistics[vm_name]

    if len(_vm_vm_statistics) > 0 and not app_args.dry_run:
        for vm, value in _vm_vm_statistics.items():

            try:
                # Migrate type VM (live migration).
                if value['type'] == 'vm':
                    logging.info(f'{info_prefix} Rebalancing VM {vm} from node {value["node_parent"]} to node {value["node_rebalance"]}.')
                    options = {'target': value['node_rebalance'], 'online': 1, 'with-local-disks': 1}
                    job_id = api_object.nodes(value['node_parent']).qemu(value['vmid']).migrate().post(**options)

                # Migrate type CT (requires restart of container).
                if value['type'] == 'ct':
                    logging.info(f'{info_prefix} Rebalancing CT {vm} from node {value["node_parent"]} to node {value["node_rebalance"]}.')
                    job_id = api_object.nodes(value['node_parent']).lxc(value['vmid']).migrate().post(target=value['node_rebalance'],restart=1)

            except proxmoxer.core.ResourceException as error_resource:
                logging.critical(f'{error_prefix} {error_resource}')

            # Wait for migration to be finished unless running parallel migrations.
            if not bool(int(parallel_migrations)):
                logging.info(f'{info_prefix} Rebalancing will be performed sequentially.')
                __wait_job_finalized(api_object, value['node_parent'], job_id, counter=1)
            else:
                logging.info(f'{info_prefix} Rebalancing will be performed parallely.')

    else:
        if app_args.dry_run:
            logging.info(f'{info_prefix} Running in dry run mode. Not executing any balancing.')
        else:
            logging.info(f'{info_prefix} No rebalancing needed.')

    return _vm_vm_statistics


def __run_storage_rebalancing(api_object, _storage_vm_statistics, app_args, parallel_migrations):
    """ Run & execute the storage rebalancing via API. """
    error_prefix = 'Error: [storage-rebalancing-executor]:'
    info_prefix  = 'Info: [storage-rebalancing-executor]:'

    # Remove VMs/CTs that do not have a new storage location.
    vms_to_remove = [vm_name for vm_name, vm_info in _storage_vm_statistics.items() if all(storage.get('storage_rebalance') == storage.get('storage_parent') for storage in vm_info.get('storage', {}).values())]
    for vm_name in vms_to_remove:
        del _storage_vm_statistics[vm_name]

    if len(_storage_vm_statistics) > 0 and not app_args.dry_run:
        for vm, value in _storage_vm_statistics.items():
            for disk, disk_info in value['storage'].items():

                if disk_info.get('storage_rebalance', None) is not None:
                    try:
                        # Migrate type VM (live migration).
                        logging.info(f'{info_prefix} Rebalancing storage of VM {vm} from node.')
                        job_id = api_object.nodes(value['node_parent']).qemu(value['vmid']).move_disk().post(disk=disk,storage=disk_info.get('storage_rebalance', None), delete=1)

                    except proxmoxer.core.ResourceException as error_resource:
                        logging.critical(f'{error_prefix} {error_resource}')

                    # Wait for migration to be finished unless running parallel migrations.
                    if not bool(int(parallel_migrations)):
                        logging.info(f'{info_prefix} Rebalancing will be performed sequentially.')
                        __wait_job_finalized(api_object, value['node_parent'], job_id, counter=1)
                    else:
                        logging.info(f'{info_prefix} Rebalancing will be performed parallely.')

    else:
        logging.info(f'{info_prefix} No rebalancing needed.')

    return _storage_vm_statistics


def __create_json_output(vm_statistics, app_args):
    """ Create a machine parsable json output of VM rebalance statitics. """
    info_prefix  = 'Info: [json-output-generator]:'

    if app_args.json:
        logging.info(f'{info_prefix} Printing json output of VM statistics.')
        print(json.dumps(vm_statistics))


def __create_cli_output(vm_statistics, app_args):
    """ Create output for CLI when running in dry-run mode. """
    info_prefix_dry_run  = 'Info: [cli-output-generator-dry-run]:'
    info_prefix_run      = 'Info: [cli-output-generator]:'
    vm_to_node_list      = []

    if app_args.dry_run:
        info_prefix = info_prefix_dry_run
        logging.info(f'{info_prefix} Starting dry-run to rebalance vms to their new nodes.')
    else:
        info_prefix = info_prefix_run
        logging.info(f'{info_prefix} Start rebalancing vms to their new nodes.')

    vm_to_node_list.append(['VM', 'Current Node', 'Rebalanced Node', 'Current Storage', 'Rebalanced Storage', 'VM Type'])
    for vm_name, vm_values in vm_statistics.items():
        for disk, disk_values in vm_values['storage'].items():
            vm_to_node_list.append([vm_name, vm_values['node_parent'], vm_values['node_rebalance'], f'{disk_values.get("storage_parent", "N/A")} ({disk_values.get("device_name", "N/A")})', f'{disk_values.get("storage_rebalance", "N/A")} ({disk_values.get("device_name", "N/A")})', vm_values['type']])

    if len(vm_statistics) > 0:
        logging.info(f'{info_prefix} Printing cli output of VM rebalancing.')
        __print_table_cli(vm_to_node_list, app_args.dry_run)
    else:
        logging.info(f'{info_prefix} No rebalancing needed.')


def __print_table_cli(table, dry_run=False):
    """ Pretty print a given table to the cli. """
    info_prefix_dry_run  = 'Info: [cli-output-generator-table-dryn-run]:'
    info_prefix_run      = 'Info: [cli-output-generator-table]:'
    info_prefix          = info_prefix_run

    longest_cols = [
        (max([len(str(row[i])) for row in table]) + 3)
        for i in range(len(table[0]))
    ]

    row_format = "".join(["{:>" + str(longest_col) + "}" for longest_col in longest_cols])

    for row in table:
        # Print CLI output when running in dry-run mode to make the user's life easier.
        if dry_run:
            info_prefix = info_prefix_dry_run
            print(row_format.format(*row))

        # Log all items in info mode.
        logging.info(f'{info_prefix} {row_format.format(*row)}')


def run_rebalancing(api_object, vm_statistics, app_args, parallel_migrations, balancing_type):
    """ Run rebalancing of vms to new nodes in cluster. """
    _vm_vm_statistics      = {}
    _storage_vm_statistics = {}

    if balancing_type == 'vm':
        _vm_vm_statistics = copy.deepcopy(vm_statistics)
        _vm_vm_statistics = __run_vm_rebalancing(api_object, _vm_vm_statistics, app_args, parallel_migrations)
        return _vm_vm_statistics

    if balancing_type == 'storage':
        _storage_vm_statistics = copy.deepcopy(vm_statistics)
        _storage_vm_statistics = __run_storage_rebalancing(api_object, _storage_vm_statistics, app_args, parallel_migrations)
        return _storage_vm_statistics


def run_output_rebalancing(app_args, vm_output_statistics, storage_output_statistics):
    """ Generate output of rebalanced resources. """
    output_statistics = {**vm_output_statistics, **storage_output_statistics}
    __create_json_output(output_statistics, app_args)
    __create_cli_output(output_statistics, app_args)


def balancing_storage_calculations(storage_balancing_method, storage_statistics, vm_statistics, balanciness, rebalance, processed_vms):
    """ Calculate re-balancing of storage on present datastores across the cluster. """
    info_prefix  = 'Info: [storage-rebalancing-calculator]:'

    # Validate for a supported balancing method, mode and if rebalancing is required.
    __validate_vm_statistics(vm_statistics)
    rebalance = __validate_storage_balanciness(balanciness, storage_balancing_method, storage_statistics)

    if rebalance:
        vm_name, vm_disk_device              = __get_most_used_resources_vm_storage(vm_statistics)

        if vm_name not in processed_vms:
            processed_vms.append(vm_name)
            resources_storage_most_free      = __get_most_free_storage(storage_balancing_method, storage_statistics)

            # Update resource statistics for VMs and storage.
            storage_statistics, vm_statistic = __update_resource_storage_statistics(storage_statistics, resources_storage_most_free, vm_statistics, vm_name, vm_disk_device)

            # Start recursion until we do not have any needs to rebalance anymore.
            balancing_storage_calculations(storage_balancing_method, storage_statistics, vm_statistics, balanciness, rebalance, processed_vms)

    logging.info(f'{info_prefix} Balancing calculations done.')
    return storage_statistics, vm_statistics


def __validate_storage_balanciness(balanciness, storage_balancing_method, storage_statistics):
    """ Validate for balanciness of storage to ensure further rebalancing is needed. """
    info_prefix                    = 'Info: [storage-balanciness-validation]:'
    error_prefix                   = 'Error: [storage-balanciness-validation]:'
    storage_resource_percent_list  = []
    storage_assigned_percent_match = []

    # Validate for an allowed balancing method and define the storage resource selector.
    if storage_balancing_method == 'disk_space':
        logging.info(f'{info_prefix} Getting most free storage volume by disk size.')
        storage_resource_selector = 'used'
    elif storage_balancing_method == 'disk_io':
        logging.error(f'{error_prefix} Getting most free storage volume by disk IO is not yet supported.')
        sys.exit(2)
    else:
        logging.error(f'{error_prefix} Getting most free storage volume by disk IO is not yet supported.')
        sys.exit(2)

    # Obtain the metrics
    for storage_name, storage_info in storage_statistics.items():

        logging.info(f'{info_prefix} Validating storage: {storage_name} for balanciness for usage with: {storage_balancing_method}.')
        # Save information of nodes from current run to compare them in the next recursion.
        if storage_statistics[storage_name][f'{storage_resource_selector}_percent_last_run'] == storage_statistics[storage_name][f'{storage_resource_selector}_percent']:
            storage_statistics[storage_name][f'{storage_resource_selector}_percent_match'] = True
        else:
            storage_statistics[storage_name][f'{storage_resource_selector}_percent_match'] = False

        # Update value to the current value of the recursion run.
        storage_statistics[storage_name][f'{storage_resource_selector}_percent_last_run'] = storage_statistics[storage_name][f'{storage_resource_selector}_percent']

        # If all node resources are unchanged, the recursion can be left.
        for key, value in storage_statistics.items():
            storage_assigned_percent_match.append(value.get(f'{storage_resource_selector}_percent_match', False))

        if False not in storage_assigned_percent_match:
            return False

        # Add node information to resource list.
        storage_resource_percent_list.append(int(storage_info[f'{storage_resource_selector}_percent']))
        logging.info(f'{info_prefix} Storage: {storage_name} with values: {storage_info}')

    # Create a sorted list of the delta + balanciness between the node resources.
    storage_resource_percent_list_sorted    = sorted(storage_resource_percent_list)
    storage_lowest_percent                  = storage_resource_percent_list_sorted[0]
    storage_highest_percent                 = storage_resource_percent_list_sorted[-1]

    # Validate if the recursion should  be proceeded for further rebalancing.
    if (int(storage_lowest_percent) + int(balanciness)) < int(storage_highest_percent):
        logging.info(f'{info_prefix} Rebalancing for type "{storage_resource_selector}" of storage is needed. Highest usage: {int(storage_highest_percent)}% | Lowest usage: {int(storage_lowest_percent)}%.')
        return True
    else:
        logging.info(f'{info_prefix} Rebalancing for type "{storage_resource_selector}" of storage is not needed. Highest usage: {int(storage_highest_percent)}% | Lowest usage: {int(storage_lowest_percent)}%.')
        return False


def __get_most_used_resources_vm_storage(vm_statistics):
    """ Get and return the most used disk of a VM by storage. """
    info_prefix = 'Info: [get-most-used-disks-resources-vm]:'

    # Get the biggest storage of a VM/CT. A VM/CT can hold multiple disks. Therefore, we need to iterate
    # over all assigned disks to get the biggest one.
    vm_object = sorted(
        vm_statistics.items(),
        key=lambda x: max(
            (size_in_bytes(storage['size']) for storage in x[1].get('storage', {}).values() if 'size' in storage),
            default=0
        ),
        reverse=True
    )

    vm_object      = vm_object[0]
    vm_name        = vm_object[0]
    vm_disk_device = max(vm_object[1]['storage'], key=lambda x: int(vm_object[1]['storage'][x]['size']))
    logging.info(f'{info_prefix} Got most used VM: {vm_name} with storage device: {vm_disk_device}.')

    return vm_name, vm_disk_device


def __get_most_free_storage(storage_balancing_method, storage_statistics):
    """ Get the storage with the most free space or IO, depending on the balancing mode. """
    info_prefix    = 'Info: [get-most-free-storage]:'
    error_prefix   = 'Error: [get-most-free-storage]:'
    storage_volume = None
    logging.info(f'{info_prefix} Starting to evaluate the most free storage volume.')

    if storage_balancing_method == 'disk_space':
        logging.info(f'{info_prefix} Getting most free storage volume by disk space.')
        storage_volume = max(storage_statistics, key=lambda x: storage_statistics[x]['free_percent'])

    if storage_balancing_method == 'disk_io':
        logging.info(f'{info_prefix} Getting most free storage volume by disk IO.')
        logging.error(f'{error_prefix} Getting most free storage volume by disk IO is not yet supported.')
        sys.exit(2)

    return storage_volume


def __update_resource_storage_statistics(storage_statistics, resources_storage_most_free, vm_statistics, vm_name, vm_disk_device):
    """ Update VM and storage resource statistics. """
    info_prefix            = 'Info: [rebalancing-storage-resource-statistics-update]:'
    current_storage        = vm_statistics[vm_name]['storage'][vm_disk_device]['storage_parent']
    current_storage_size   = storage_statistics[current_storage]['free'] / (1024 ** 3)
    rebalance_storage      = resources_storage_most_free
    rebalance_storage_size = storage_statistics[rebalance_storage]['free'] / (1024 ** 3)
    vm_storage_size        = vm_statistics[vm_name]['storage'][vm_disk_device]['size']
    vm_storage_size_bytes  = int(vm_storage_size) * 1024**3

    # Assign new storage device to vm
    logging.info(f'{info_prefix} Validating VM {vm_name} for potential storage rebalancing.')
    if vm_statistics[vm_name]['storage'][vm_disk_device]['storage_rebalance'] == vm_statistics[vm_name]['storage'][vm_disk_device]['storage_parent']:
        logging.info(f'{info_prefix} Setting VM {vm_name} from {current_storage} to {rebalance_storage} storage.')
        vm_statistics[vm_name]['storage'][vm_disk_device]['storage_rebalance'] = resources_storage_most_free
    else:
        logging.info(f'{info_prefix} Setting VM {vm_name} from {current_storage} to {rebalance_storage} storage.')

    # Recalculate values for storage
    ## Add freed resources to old parent storage device
    storage_statistics[current_storage]['used']           = storage_statistics[current_storage]['used'] - vm_storage_size_bytes
    storage_statistics[current_storage]['free']           = storage_statistics[current_storage]['free'] + vm_storage_size_bytes
    storage_statistics[current_storage]['free_percent']   = (storage_statistics[current_storage]['free'] / storage_statistics[current_storage]['total']) * 100
    storage_statistics[current_storage]['used_percent']   = (storage_statistics[current_storage]['used'] / storage_statistics[current_storage]['total']) * 100
    logging.info(f'{info_prefix} Adding free space of {vm_storage_size}G to old storage with {current_storage_size}G. [free: {int(current_storage_size) + int(vm_storage_size)}G | {storage_statistics[current_storage]["free_percent"]}%]')

    ## Removed newly allocated resources to new rebalanced storage device
    storage_statistics[rebalance_storage]['used']         = storage_statistics[rebalance_storage]['used'] + vm_storage_size_bytes
    storage_statistics[rebalance_storage]['free']         = storage_statistics[rebalance_storage]['free'] - vm_storage_size_bytes
    storage_statistics[rebalance_storage]['free_percent'] = (storage_statistics[rebalance_storage]['free'] / storage_statistics[rebalance_storage]['total']) * 100
    storage_statistics[rebalance_storage]['used_percent'] = (storage_statistics[rebalance_storage]['used'] / storage_statistics[rebalance_storage]['total']) * 100
    logging.info(f'{info_prefix} Adding used space of {vm_storage_size}G to new storage with {rebalance_storage_size}G. [free: {int(rebalance_storage_size) - int(vm_storage_size)}G | {storage_statistics[rebalance_storage]["free_percent"]}%]')

    logging.info(f'{info_prefix} Updated VM and storage statistics.')
    return storage_statistics, vm_statistics


def size_in_bytes(size_str):
    size_unit = size_str[-1].upper()
    size_value = float(size_str)
    size_multipliers = {'K': 1024, 'M': 1024**2, 'G': 1024**3, 'T': 1024**4}
    return size_value * size_multipliers.get(size_unit, 1)


def balancing_vm_affinity_groups(node_statistics, vm_statistics, balancing_method, balancing_mode):
    """ Enforce (anti-)affinity groups for further VM movement across the cluster. """
    node_statistics, vm_statistics = __get_vm_tags_include_groups(vm_statistics, node_statistics, balancing_method, balancing_mode)
    node_statistics, vm_statistics = __get_vm_tags_exclude_groups(vm_statistics, node_statistics, balancing_method, balancing_mode)
    return node_statistics, vm_statistics


def main():
    """ Run ProxLB for balancing VM workloads across a Proxmox cluster. """
    vm_output_statistics      = {}
    storage_output_statistics = {}

    # Initialize PAS.
    initialize_logger('CRITICAL')
    app_args = initialize_args()
    if app_args.version:
        proxlb_output_version()
    config_path = initialize_config_path(app_args)
    pre_validations(config_path)

    # Parse global config.
    proxlb_config = initialize_config_options(config_path)
    pre_validations(config_path, proxlb_config)

    # Overwrite logging handler with user defined log verbosity.
    initialize_logger(proxlb_config['log_verbosity'], update_log_verbosity=True)

    while True:
        # API Authentication.
        api_object = api_connect(proxlb_config['proxmox_api_host'], proxlb_config['proxmox_api_user'], proxlb_config['proxmox_api_pass'], proxlb_config['proxmox_api_ssl_v'], proxlb_config['proxmox_api_timeout'])

        # Get master node of cluster and ensure that ProxLB is only performed on the
        # cluster master node to avoid ongoing rebalancing.
        cluster_master, master_only = execute_rebalancing_only_by_master(api_object, proxlb_config['master_only'])

        # Validate daemon service and skip following tasks when not being the cluster master.
        if not cluster_master and master_only:
            validate_daemon(proxlb_config['daemon'], proxlb_config['schedule'])
            continue

        # Get metrics & statistics for vms and nodes.
        if proxlb_config['vm_balancing_enable'] or proxlb_config['storage_balancing_enable'] or app_args.best_node:
            node_statistics    = get_node_statistics(api_object, proxlb_config['vm_ignore_nodes'], proxlb_config['vm_maintenance_nodes'])
            vm_statistics      = get_vm_statistics(api_object, proxlb_config['vm_ignore_vms'], proxlb_config['vm_balancing_type'])
            node_statistics    = update_node_statistics(node_statistics, vm_statistics)
            # Obtaining metrics for the storage may take longer times and is not needed for VM/CT balancing.
            # We can save time by skipping this when not really needed.
            if proxlb_config['storage_balancing_enable']:
                storage_statistics = get_storage_statistics(api_object)

        # Execute VM/CT balancing sub-routines.
        if proxlb_config['vm_balancing_enable'] or app_args.best_node:
            node_statistics, vm_statistics = balancing_vm_calculations(proxlb_config['vm_balancing_method'], proxlb_config['vm_balancing_mode'], proxlb_config['vm_balancing_mode_option'], node_statistics, vm_statistics, proxlb_config['vm_balanciness'], app_args, rebalance=False, processed_vms=[])
            node_statistics, vm_statistics = balancing_vm_maintenance(proxlb_config, app_args, node_statistics, vm_statistics)
            node_statistics, vm_statistics = balancing_vm_affinity_groups(node_statistics, vm_statistics, proxlb_config['vm_balancing_method'], proxlb_config['vm_balancing_mode'],)
            vm_output_statistics = run_rebalancing(api_object, vm_statistics, app_args, proxlb_config['vm_parallel_migrations'], 'vm')

        # Execute storage balancing sub-routines.
        if proxlb_config['storage_balancing_enable']:
            storage_statistics, vm_statistics = balancing_storage_calculations(proxlb_config['storage_balancing_method'], storage_statistics, vm_statistics, proxlb_config['storage_balanciness'], rebalance=False, processed_vms=[])
            storage_output_statistics = run_rebalancing(api_object, vm_statistics, app_args, proxlb_config['storage_parallel_migrations'], 'storage')

        # Generate balancing output
        if proxlb_config['vm_balancing_enable'] or proxlb_config['storage_balancing_enable']:
            run_output_rebalancing(app_args, vm_output_statistics, storage_output_statistics)

        # Validate for any errors.
        post_validations()

        # Validate daemon service.
        validate_daemon(proxlb_config['daemon'], proxlb_config['schedule'])


if __name__ == '__main__':
    main()