diff --git a/docker/openstack-accounting/Dockerfile b/docker/openstack-accounting/Dockerfile new file mode 100644 index 00000000..579fd482 --- /dev/null +++ b/docker/openstack-accounting/Dockerfile @@ -0,0 +1,5 @@ +FROM registry.cern.ch/cmsmonitoring/cmsmon-py:test + +# Copy only the specific files you need instead of cloning entire repos +COPY src/ ./ + diff --git a/docker/openstack-accounting/README.md b/docker/openstack-accounting/README.md new file mode 100644 index 00000000..f1135c47 --- /dev/null +++ b/docker/openstack-accounting/README.md @@ -0,0 +1,3 @@ +# Openstack Accounting cron job + +Code base of the Docker image used for the openstack-accounting cronjob, which generates data about Openstack usage from different groups and publishes it in the [Openstack Quotas And Usage Monitoring website](https://cmsdatapop.web.cern.ch/cmsdatapop/eos_openstack/openstack_accounting.html) of the CMS Data Popularity service. diff --git a/docker/openstack-accounting/src/cron4openstack_accounting.sh b/docker/openstack-accounting/src/cron4openstack_accounting.sh new file mode 100755 index 00000000..ebb3434e --- /dev/null +++ b/docker/openstack-accounting/src/cron4openstack_accounting.sh @@ -0,0 +1,58 @@ +#!/bin/bash +##H Script to create CMS Eos path sizes with conditions +##H CMSVOC and CMSMONITORING groups are responsible for this script. +set -e +TZ=UTC +myname=$(basename "$0") +script_dir="$(cd "$(dirname "$0")" && pwd)" +# Get nice util functions +. "${script_dir}"/utils.sh + +# Do not change the order of "--output_file"([0],[1]) which is replaced in K8s run +py_input_args=( + --output_file "/eos/user/c/cmsmonit/www/eos_openstack/openstack_accounting.html" + --summary_json "/eos/cms/store/accounting/openstack_accounting_summary.json" + --static_html_dir "${script_dir}/html" +) + +# ---------------------------------------------------------------------------------------------------------- Run in K8S +if [ -n "$K8S_ENV" ]; then + # $1: output + # Replace static output file with user arg for testability. + py_input_args[1]=$1 + + util4logi "${myname} is starting.." + util_cron_send_start "$myname" "1h" + + util_kerberos_auth_with_keytab /etc/secrets/keytab + python3 "${script_dir}"/openstack_accounting.py "${py_input_args[@]}" 2>&1 + + util_cron_send_end "$myname" "1h" "$?" + util4logi "${myname} successfully finished." + exit 0 + # break +fi +# Run in LxPlus for test ---------------------------------------------------------------------------------------------- + +. /cvmfs/sft.cern.ch/lcg/views/LCG_101/x86_64-centos7-gcc8-opt/setup.sh + +# Catch output to not print successful jobs stdout to email, print when failed +output=$(pip install --user schema 2>&1) +ec=$? +if [ $ec -ne 0 ]; then + echo "$output" - exit code: $ec + exit $ec +fi + +if ! [ "$(python -c 'import sys; print(sys.version_info.major)')" = 3 ]; then + echo "It seem python version is not 3.X! Exiting..." + exit 1 +fi + +# Catch output to not print successful jobs stdout to email, print when failed +output=$(python "{$script_dir}"/openstack_accounting.py "${py_input_args[@]}" 2>&1) +ec=$? +if [ $ec -ne 0 ]; then + echo "$output" - exit code: $ec + exit $ec +fi diff --git a/docker/openstack-accounting/src/html/main.html b/docker/openstack-accounting/src/html/main.html new file mode 100644 index 00000000..800bdc56 --- /dev/null +++ b/docker/openstack-accounting/src/html/main.html @@ -0,0 +1,227 @@ + + + + + + + + + + + + + + + + + +
+
+
+
+ CERN CMS +
+
+
+

Openstack Quotas And Usage Monitoring

+
+
+
+ CERN CMS O&C +
+
+
+ +
+
+
+ + +
+ ____SUMMARY_BLOCK____ + +
+ + + + + + + + + + + diff --git a/docker/openstack-accounting/src/openstack_accounting.py b/docker/openstack-accounting/src/openstack_accounting.py new file mode 100644 index 00000000..30e4222d --- /dev/null +++ b/docker/openstack-accounting/src/openstack_accounting.py @@ -0,0 +1,193 @@ +# !/usr/bin/env python +# -*- coding: utf-8 -*- +# Author: Ceyhun Uzunoglu +# Create html table for Openstack Projects accounting +# +# cron job script : scripts/cron4openstack_accounting.sh +# Kubernetes service : https://github.com/dmwm/CMSKubernetes/blob/master/kubernetes/monitoring/services/cron-size-quotas.yaml +# + +import json +import os +import sys +from datetime import datetime + +import click +import pandas as pd +from schema import Schema, Use, SchemaError + +pd.options.display.float_format = "{:,.2f}".format +pd.set_option("display.max_colwidth", None) +total_row_openstackProjectName = "TOTAL" + +SUMMARY_SCHEMA = Schema([{'openstackProjectName': str, + 'maxTotalInstances': Use(int), + 'maxTotalCores': Use(int), + 'maxTotalRAMSize': Use(int), + 'maxSecurityGroups': Use(int), + 'maxTotalFloatingIps': Use(int), + 'maxServerMeta': Use(int), + 'maxImageMeta': Use(int), + 'maxPersonality': Use(int), + 'maxPersonalitySize': Use(int), + 'maxSecurityGroupRules': Use(int), + 'maxTotalKeypairs': Use(int), + 'maxServerGroups': Use(int), + 'maxServerGroupMembers': Use(int), + 'totalRAMUsed': Use(int), + 'totalCoresUsed': Use(int), + 'totalInstancesUsed': Use(int), + 'totalFloatingIpsUsed': Use(int), + 'totalSecurityGroupsUsed': Use(int), + 'totalServerGroupsUsed': Use(int), + 'maxTotalVolumes': Use(int), + 'maxTotalSnapshots': Use(int), + 'maxTotalVolumeGigabytes': Use(int), + 'maxTotalBackups': Use(int), + 'maxTotalBackupGigabytes': Use(int), + 'totalVolumesUsed': Use(int), + 'totalGigabytesUsed': Use(int), + 'totalSnapshotsUsed': Use(int), + 'totalBackupsUsed': Use(int), + 'totalBackupGigabytesUsed': Use(int), + 'contacts': str, }]) + +# DO NOT FORGET TO UPDATE "columnDefs" VISIBILITY IN JavaScript TEMPLATE: src/html/openstack_accounting/main.html +# ORDER IS IMPORTANT IN PYTHON DICTS AND IT IS USED IN JS "columnDefs" ARRAY +SUMMARY_COL_ORDER = {'openstackProjectName': 'Openstack Project Name', + 'maxTotalInstances': 'Max Total Instances', + 'maxTotalCores': 'Max Total Cores', + 'maxTotalRAMSize': 'Max Total RAM Size', + 'maxSecurityGroups': 'Max Security Groups', + 'maxTotalFloatingIps': 'Max Total Floating Ips', + 'maxServerMeta': 'Max Server Meta', + 'maxImageMeta': 'Max Image Meta', + 'maxPersonality': 'Max Personality', + 'maxPersonalitySize': 'Max Personality Size', + 'maxSecurityGroupRules': 'Max Security Group Rules', + 'maxTotalKeypairs': 'Max Total Keypairs', + 'maxServerGroups': 'Max Server Groups', + 'maxServerGroupMembers': 'Max Server Group Members', + 'totalRAMUsed': 'Total RAM Used', + 'totalCoresUsed': 'Total Cores Used', + 'totalInstancesUsed': 'Total Instances Used', + 'totalFloatingIpsUsed': 'Total Floating Ips Used', + 'totalSecurityGroupsUsed': 'Total Security Groups Used', + 'totalServerGroupsUsed': 'Total Server Groups Used', + 'maxTotalVolumes': 'Max Total Volumes', + 'maxTotalSnapshots': 'Max Total Snapshots', + 'maxTotalVolumeGigabytes': 'Max Total Volume Gigabytes', + 'maxTotalBackups': 'Max Total Backups', + 'maxTotalBackupGigabytes': 'Max Total Backup Gigabytes', + 'totalVolumesUsed': 'Total Volumes Used', + 'totalGigabytesUsed': 'Total Gigabytes Used', + 'totalSnapshotsUsed': 'Total Snapshots Used', + 'totalBackupsUsed': 'Total Backups Used', + 'totalBackupGigabytesUsed': 'Total Backup Gigabytes Used', + 'contacts': 'Contacts'} + + +def tstamp(): + """Return timestamp for logging""" + return datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + + +def get_update_time_of_file(summary_file): + """Create update time depending on reading Openstack accounting results from file""" + if summary_file: + # Set update time to Openstack accounting file modification time, minimum of 2 + summary_ts = os.path.getmtime(summary_file) + try: + return datetime.utcfromtimestamp(summary_ts).strftime('%Y-%m-%d %H:%M:%S') + except OSError as e: + print(tstamp(), "ERROR: could not get last modification time of file:", str(e)) + else: + # !! means time did not come from file but cron job time + return "!!" + datetime.utcnow().strftime("%Y-%m-%d H:%M:%S") + + +def get_df_with_validation(json_file, schema, column_order): + """Read json file, validate, cast types and convert to pandas dataframe + """ + try: + with open(json_file) as f: + json_arr = json.load(f) + + json_arr = schema.validate(json_arr) + + # orient values reads json array + df = pd.DataFrame(json_arr, columns=column_order.keys()).rename(columns=column_order) + + # Find the dict with openstackProjectName= "TOTAL" in JSON array + total_dict = next( + (sub for sub in json_arr if sub['openstackProjectName'] == total_row_openstackProjectName), + None + ) + # remove TOTAL row from json array + json_arr.remove(total_dict) + + # Create 2 level columns dataframe to make total row to fit the top like column names. + columns = list(zip(SUMMARY_COL_ORDER.values(), total_dict.values())) + df.columns = pd.MultiIndex.from_tuples(columns) + + return df + except SchemaError as e: + print(tstamp(), "Data not exist or not valid:", str(e)) + sys.exit(1) + except json.JSONDecodeError as e: + print(tstamp(), json_file, "file is empty:", str(e)) + sys.exit(1) + + +def get_html_template(base_html_directory=None): + """ Reads partial html file and return it as strings + """ + if base_html_directory is None: + base_html_directory = os.getcwd() + with open(os.path.join(base_html_directory, "main.html")) as f: + main_html = f.read() + return main_html + + +def prepare_html(df): + html = df.to_html(escape=False, index=False) + # cleanup of the default dump + html = html.replace( + 'table border="1" class="dataframe"', + 'table id="" class="display compact" style="width:90%;"', + ) + html = html.replace('style="text-align: right;"', "") + return html + + +def create_main_html(df_summary, update_time, base_html_directory): + """Create html page with given dataframe + """ + df_summary_html = prepare_html(df_summary) + + # Get main html + main_html = get_html_template(base_html_directory=base_html_directory) + main_html = main_html.replace("__UPDATE_TIME__", update_time) + + # Add pandas dataframe html to main body + main_html = main_html.replace('____SUMMARY_BLOCK____', df_summary_html) + return main_html + + +@click.command() +@click.option("--output_file", default=None, required=True, help="For example: /eos/.../www/test/test.html") +@click.option("--summary_json", required=True, help="/eos/cms/store/accounting/openstack_accounting_summary.json") +@click.option("--static_html_dir", default=None, required=True, + help="Html directory for main html template. For example: ~/CMSMonitoring/src/html/openstack_accounting") +def main(output_file=None, summary_json=None, static_html_dir=None): + joint_update_time = get_update_time_of_file(summary_json) + print("[INFO] Update time of input files:", joint_update_time) + + df_summary = get_df_with_validation(summary_json, SUMMARY_SCHEMA, SUMMARY_COL_ORDER) + main_html = create_main_html(df_summary, joint_update_time, static_html_dir) + with open(output_file, "w+") as f: + f.write(main_html) + + +if __name__ == "__main__": + main()