CP-20097 add prometheus version check

[josephbarnett]

Solved Issue #8

1. Add verification check which runs prometheus --version, capturing output for telemetry.
2. Add addition of version for chart (if set) to telemetry report
3. Minor update to Makefile for development builds

Testing

1. Added unit tests
2. Local deployment - when running test see:

   {
    "level": "info",
    "log_sequence": 1,
    "msg": "reporting status",
    "report": {
    "account":"00000000",
    "region":"us-east-1",
    "name":"jb-test-cluster",
    "state":"STATUS_TYPE_POD_STARTED",
    "chartVersion":"0.0.10",
    "agentVersion":"prometheus, version 2.50.1 (branch: HEAD, revision: 8c9b0285360a0b6288d76214a75ce3025bce4050)\\n  build user:       root@6213bb3ee580\\n  build date:       20240226-11:38:47\\n  go version:       go1.21.7\\n  platform:         linux/arm64\\n  tags:             netgo,builtinassets,stringlabels\\n",
    "scrapeConfig":"# my global config\\nglobal:\\n  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.\\n  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.\\n  # scrape_timeout is set to the global default (10s).\\n\\n# Alertmanager configuration\\nalerting:\\n  alertmanagers:\\n    - static_configs:\\n        - targets:\\n          # - alertmanager:9093\\n\\n# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.\\nrule_files:\\n  # - \\"first_rules.yml\\"\\n  # - \\"second_rules.yml\\"\\n\\n# A scrape configuration containing exactly one endpoint to scrape:\\n# Here it's Prometheus itself.\\nscrape_configs:\\n  # The job name is added as a label `job=\u003cjob_name\u003e` to any timeseries scraped from this config.\\n  - job_name: \\"prometheus\\"\\n\\n    # metrics_path defaults to '/metrics'\\n    # scheme defaults to 'http'.\\n\\n    static_configs:\\n      - targets: [\\"localhost:9090\\"]\\n\\nglobal:\\n  scrape_interval: 60s\\nscrape_configs:\\n  - job_name: cloudzero-service-endpoints # kube_*, node_* metrics\\n    honor_labels: true\\n    honor_timestamps: true\\n    track_timestamps_staleness: false\\n    scrape_interval: 1m\\n    scrape_timeout: 10s\\n    scrape_protocols:\\n    - OpenMetricsText1.0.0\\n    - OpenMetricsText0.0.1\\n    - PrometheusText0.0.4\\n    metrics_path: /metrics\\n    scheme: http\\n    enable_compression: true\\n    follow_redirects: true\\n    enable_http2: true\\n    relabel_configs:\\n    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]\\n      separator: ;\\n      regex: \\"true\\"\\n      replacement: $1\\n      action: keep\\n    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape_slow]\\n      separator: ;\\n      regex: \\"true\\"\\n      replacement: $1\\n      action: drop\\n    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]\\n      separator: ;\\n      regex: (https?)\\n      target_label: __scheme__\\n      replacement: $1\\n      action: replace\\n    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]\\n      separator: ;\\n      regex: (.+)\\n      target_label: __metrics_path__\\n      replacement: $1\\n      action: replace\\n    - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]\\n      separator: ;\\n      regex: (.+?)(?::\\\\d+)?;(\\\\d+)\\n      target_label: __address__\\n      replacement: $1:$2\\n      action: replace\\n    - separator: ;\\n      regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)\\n      replacement: __param_$1\\n      action: labelmap\\n    - separator: ;\\n      regex: __meta_kubernetes_service_label_(.+)\\n      replacement: $1\\n      action: labelmap\\n    - source_labels: [__meta_kubernetes_namespace]\\n      separator: ;\\n      regex: (.*)\\n      target_label: namespace\\n      replacement: $1\\n      action: replace\\n    - source_labels: [__meta_kubernetes_service_name]\\n      separator: ;\\n      regex: (.*)\\n      target_label: service\\n      replacement: $1\\n      action: replace\\n    - source_labels: [__meta_kubernetes_pod_node_name]\\n      separator: ;\\n      regex: (.*)\\n      target_label: node\\n      replacement: $1\\n      action: replace\\n    metric_relabel_configs:\\n    - source_labels: [__name__]\\n      regex: \\"^(kube_node_info|kube_node_status_capacity|kube_pod_container_resource_limits|kube_pod_container_resource_requests|kube_pod_labels|kube_pod_info|node_dmi_info)$\\"\\n      action: keep\\n    - action: labelkeep\\n      regex: \\"^(board_asset_tag|container|created_by_kind|created_by_name|image|instance|name|namespace|node|node_kubernetes_io_instance_type|pod|product_name|provider_id|resource|unit|uid|_.*|label_.*)$\\"\\n    kubernetes_sd_configs:\\n    - role: endpoints\\n      kubeconfig_file: \\"\\"\\n      follow_redirects: true\\n      enable_http2: true\\n  - job_name: cloudzero-nodes-cadvisor # container_* metrics\\n    honor_timestamps: true\\n    track_timestamps_staleness: false\\n    scrape_interval: 1m\\n    scrape_timeout: 10s\\n    scrape_protocols:\\n    - OpenMetricsText1.0.0\\n    - OpenMetricsText0.0.1\\n    - PrometheusText0.0.4\\n    metrics_path: /metrics\\n    scheme: https\\n    enable_compression: true\\n    authorization:\\n      type: Bearer\\n      credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token\\n    tls_config:\\n      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\\n      insecure_skip_verify: true\\n    follow_redirects: true\\n    enable_http2: true\\n    relabel_configs:\\n    - separator: ;\\n      regex: __meta_kubernetes_node_label_(.+)\\n      replacement: $1\\n      action: labelmap\\n    - separator: ;\\n      regex: (.*)\\n      target_label: __address__\\n      replacement: kubernetes.default.svc:443\\n      action: replace\\n    - source_labels: [__meta_kubernetes_node_name]\\n      separator: ;\\n      regex: (.+)\\n      target_label: __metrics_path__\\n      replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor\\n      action: replace\\n    - source_labels: [__meta_kubernetes_node_name]\\n      target_label: node\\n      action: replace\\n    metric_relabel_configs:\\n    - action: labelkeep\\n      regex: \\"^(board_asset_tag|container|created_by_kind|created_by_name|image|instance|name|namespace|node|node_kubernetes_io_instance_type|pod|product_name|provider_id|resource|unit|uid|_.*|label_.*)$\\"\\n    - source_labels: [__name__]\\n      regex: \\"^(container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_network_receive_bytes_total|container_network_transmit_bytes_total)$\\"\\n      action: keep\\n    kubernetes_sd_configs:\\n    - role: node\\n      kubeconfig_file: \\"\\"\\n      follow_redirects: true\\n      enable_http2: true\\nremote_write:\\n  - url: 'https://api.cloudzero.com/v1/container-metrics?cluster_name=jb-test-cluster\u0026cloud_account_id=00000000\u0026region=us-east-1'\\n    authorization:\\n      credentials_file: /etc/config/prometheus/secrets/value\\n    write_relabel_configs:\\n      - source_labels: [__name__]\\n        regex: \\"^(kube_node_info|kube_node_status_capacity|kube_pod_container_resource_limits|kube_pod_container_resource_requests|kube_pod_labels|kube_pod_info|node_dmi_info|container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_network_receive_bytes_total|container_network_transmit_bytes_total)$\\"\\n        action: keep\\n    metadata_config:\\n      send: false",
    "validatorVersion":"cloudzero-agent-validator.0ef023a402356389927658b421682203339d8e83.dev-0ef023a402356389927658b421682203339d8e83-2024-07-15_02:28:19PM",
    "k8sVersion":"1.29",
    "checks":[
        {"name":"scrape_cfg","passing":true}, 
        {"name":"scrape_cfg",
        "passing":true},
        {"name":"kube_state_metrics_reachable","passing":true},
        {"name":"k8s_version","passing":true},
        {"name":"node_exporter_reachable","passing":true},
        {"name":"prometheus_version","passing":true}
        ]
    }
    ,
    "time": "2024-07-15T14:31:00Z"

}