Merge pull request #33 from starschema/main

deploy: deploy EMQX to prod
starschema · Jun 13, 2023 · ee28ea2 · ee28ea2
2 parents d5b8044 + d5e6867
commit ee28ea2
Show file tree

Hide file tree

Showing 9 changed files with 196 additions and 115 deletions.
diff --git a/containers/growatt/growatt.js b/containers/growatt/growatt.js
@@ -1,18 +1,18 @@
-"use strict";
-var api = require("growatt");
-var sparkplug = require("sparkplug-client");
-var os = require("os");
-var _ = require("lodash");
-var flatten = require("flat");
-var cron = require("node-cron");
+"use strict"
+var api = require("growatt")
+var sparkplug = require("sparkplug-client")
+var os = require("os")
+var _ = require("lodash")
+var flatten = require("flat")
+var cron = require("node-cron")
 
-require("dotenv").config();
+require("dotenv").config()
 
-const user = process.env.GROWATT_USER;
-const password = process.env.GROWATT_PASSWORD;
-const plantId = process.env.GROWATT_PLANT_ID;
+const user = process.env.GROWATT_USER
+const password = process.env.GROWATT_PASSWORD
+const plantId = process.env.GROWATT_PLANT_ID
 
-const publishSchedule = process.env.PUBLISH_SCHEDULE || "*/5 * * * *";
+const publishSchedule = process.env.PUBLISH_SCHEDULE || "*/5 * * * *"
 
 const config = {
   serverUrl: `tcp://${process.env.MQTT_HOST || "mqtt"}:${
@@ -24,111 +24,108 @@ const config = {
   edgeNode: os.hostname(),
   clientId: `Growatt Plant ${plantId}`,
   version: "spBv1.0",
-};
-var client = sparkplug.newClient(config);
+}
+var client = sparkplug.newClient(config)
 
 const mapDataType = (v, k) => {
-  var dataType;
+  var dataType
 
   if (typeof v === "number" && v % 1 === 0) {
-    dataType = "long";
+    dataType = "long"
   } else if (typeof v === "number") {
-    dataType = "float";
+    dataType = "float"
   } else if (typeof v === "string") {
-    dataType = "string";
+    dataType = "string"
   } else if (typeof v === "boolean") {
-    dataType = "boolean";
+    dataType = "boolean"
   } else if (typeof v === "object") {
-    dataType = "string";
-    v = JSON.stringify(v);
+    dataType = "string"
+    v = JSON.stringify(v)
   } else {
-    dataType = "string";
+    dataType = "string"
   }
 
-  return { name: k, value: v, type: dataType };
-};
+  return { name: k, value: v, type: dataType }
+}
 
 const retrieveGrowattData = async function () {
-  const growatt = new api({});
+  const growatt = new api({})
   await growatt.login(user, password).catch((e) => {
-    console.log("Cannot login to Growatt API:", e);
-    throw e;
-  });
-  let getAllPlantData = await growatt.getAllPlantData({});
+    console.log("Cannot login to Growatt API:", e)
+    throw e
+  })
+  let getAllPlantData = await growatt.getAllPlantData({})
   await growatt.logout().catch((e) => {
-    console.log("Cannot retrieve data from Growatt API:", e);
-    throw e;
-  });
-  return getAllPlantData;
-};
+    console.log("Cannot retrieve data from Growatt API:", e)
+    throw e
+  })
+  return getAllPlantData
+}
 
 const getGrowattData = function (publishNodeData, publishDeviceData) {
   retrieveGrowattData()
     .then((data) => {
-      const plant = data[plantId];
+      const plant = data[plantId]
       const plantData = _.mapKeys(
         plant["plantData"],
         (v, k) => "plantData_" + k
-      );
-      const weather = flatten(plant["weather"]["data"]);
+      )
+      const weather = flatten(plant["weather"]["data"])
 
-      const nodeData = _.assign(plantData, weather);
+      const nodeData = _.assign(plantData, weather)
       if (publishNodeData) {
-        publishNodeData({ metrics: _.map(nodeData, mapDataType) });
+        publishNodeData({ metrics: _.map(nodeData, mapDataType) })
       }
 
       _.forIn(plant.devices, (device, deviceId) => {
-        const deviceData = _.assign(
-          device["deviceData"],
-          device["historyLast"]
-        );
+        const deviceData = _.assign(device["deviceData"], device["historyLast"])
 
         if (publishDeviceData) {
           publishDeviceData(deviceId, {
             timestamp: new Date(deviceData["calendar"]).getTime(),
             metrics: _.map(deviceData, mapDataType),
-          });
+          })
         }
-      });
+      })
     })
     .catch((err) => {
-      console.log(err);
-      return [null, null];
-    });
-};
+      console.log(err)
+      return [null, null]
+    })
+}
 
 // Create "birth" handler
 client.on("birth", function () {
-  getGrowattData(client.publishNodeBirth, client.publishDeviceBirth);
-});
+  getGrowattData(client.publishNodeBirth, client.publishDeviceBirth)
+})
 
 // Create node command handler
 client.on("ncmd", function (payload) {
-  const metrics = payload.metrics;
+  const metrics = payload.metrics
 
   if (metrics !== undefined && metrics !== null) {
     for (var i = 0; i < metrics.length; i++) {
-      var metric = metrics[i];
+      var metric = metrics[i]
       if (metric.name == "Node Control/Rebirth" && metric.value) {
-        console.log("Received 'Rebirth' command");
-        getGrowattData(client.publishNodeBirth, client.publishDeviceBirth);
+        console.log("Received 'Rebirth' command")
+        getGrowattData(client.publishNodeBirth, client.publishDeviceBirth)
       }
     }
   }
-});
+})
 
 const run = function () {
   // getGrowattData(client.publishNodeData, client.publishDeviceData)
 
   cron.schedule(publishSchedule, () => {
     try {
-      getGrowattData(client.publishNodeData, client.publishDeviceData);
+      getGrowattData(client.publishNodeData, client.publishDeviceData)
     } catch (e) {
-      console.log("ERROR:", e);
+      console.log("ERROR:", e)
     }
-  });
-};
+  })
+}
 
-run();
+run()
 
 //#client.stop()
diff --git a/containers/lm-sensor-sparkplug/lm-sensor-mqtt.py b/containers/lm-sensor-sparkplug/lm-sensor-mqtt.py
@@ -31,7 +31,7 @@ def net_dev():
         if line.find(":") < 0:
             continue
         face, data = line.split(":")
-        faceData = dict(zip(cols, data.split()))
+        faceData = dict(zip(cols, [int(x) for x in data.split()]))
         faces[face.strip()] = faceData
 
     return faces

diff --git a/deployment/kubernetes/Pulumi.github-actions.yaml b/deployment/kubernetes/Pulumi.github-actions.yaml
@@ -10,6 +10,7 @@ config:
     growatt: true
     lm-sensors: true
     monitoring: true
+    influxdb2: true
   antares-idl-k8s:config:
     cloud:
       type: aws

diff --git a/deployment/kubernetes/Pulumi.yaml b/deployment/kubernetes/Pulumi.yaml
@@ -15,7 +15,9 @@ config:
     antares-idl-k8s:components.emqx: false
     antares-idl-k8s:components.growatt: false
     antares-idl-k8s:components.lm-sensors: false
-    # Monitoring includes Prometheus, Grafana and Etcd
+    antares-idl-k8s:components.influxdb2: false
+    # Monitoring includes Prometheus, Grafana and Etcd on a separate 
+    # namespace
     antares-idl-k8s:components.monitoring: false
 
 
diff --git a/deployment/kubernetes/__main__.py b/deployment/kubernetes/__main__.py
@@ -43,6 +43,7 @@
 import hvr
 import postgresql
 import emqx
+import influxdb2
 import cert_manager
 import growatt
 import lm_sensors
@@ -96,6 +97,9 @@
 if component_enabled("emqx"):
     emqx.deploy()
 
+if component_enabled("influxdb2"):
+    influxdb2.deploy()
+
 if component_enabled("growatt"):
     growatt.deploy()
 

diff --git a/deployment/kubernetes/emqx.py b/deployment/kubernetes/emqx.py
@@ -37,8 +37,14 @@
 
 
 def deploy():
-    if not component_enabled("cert-manager"):
+    if not component_enabled("cert-manager") and config.get(
+        "/emqx/install-operator", True
+    ):
         raise Exception("cert-manager is required for emqx")
+    elif not component_enabled("cert-manager"):
+        resources["cert-manager"] = Release.get(
+            "cert-manager", "cert-manager/cert-manager"
+        )
 
     if component_enabled("efs-eks"):
         emqx_helm_values = {
@@ -49,16 +55,11 @@ def deploy():
             },
         }
         emqx_pvc_template = {
-            "metadata": {
-                "name": "emqx-data",
-            },
-            "spec": {
-                "storageClassName": "efs-sc-user-1000",
-                "accessModes": ["ReadWriteMany"],
-                "resources": {
-                    "requests": {
-                        "storage": "10Gi",
-                    },
+            "storageClassName": "efs-sc-user-1000",
+            "accessModes": ["ReadWriteMany"],
+            "resources": {
+                "requests": {
+                    "storage": "10Gi",
                 },
             },
         }
@@ -68,61 +69,69 @@ def deploy():
         depends_on = [resources["cert-manager"]]
         emqx_pvc_template = {}
 
-    emqx_release = Release(
-        "emqx",
-        ReleaseArgs(
-            chart="emqx-operator",
-            name="emqx",
-            repository_opts=RepositoryOptsArgs(
-                repo="https://repos.emqx.io/charts",
+    if config.get("/emqx/install-operator", True):
+        emqx_release = Release(
+            "emqx",
+            ReleaseArgs(
+                chart="emqx-operator",
+                name="emqx",
+                repository_opts=RepositoryOptsArgs(
+                    repo="https://repos.emqx.io/charts",
+                ),
+                namespace="emqx-operator-system",
+                create_namespace=True,
+                values={**emqx_helm_values, **(config.get("/emqx/helm-values", {}))},
             ),
-            namespace="emqx-operator-system",
-            create_namespace=True,
-            values={**emqx_helm_values, **(config.get("/emqx/helm-values", {}))},
-        ),
-        opts=pulumi.ResourceOptions(
-            depends_on=depends_on,
-        ),
-    )
+            opts=pulumi.ResourceOptions(
+                depends_on=depends_on,
+            ),
+        )
 
-    resources["emqx"] = emqx_release
+        resources["emqx"] = emqx_release
+    else:
+        resources["emqx"] = Release.get("emqx", "emqx-operator-system/emqx")
 
     emqx_ee = CustomResource(
         "emqx-ee",
-        api_version="apps.emqx.io/v1beta4",
-        kind="EmqxEnterprise",
+        api_version="apps.emqx.io/v2alpha1",
+        kind="EMQX",
         metadata=ObjectMetaArgs(
             name="emqx-ee",
             namespace=resources["namespace"].metadata["name"],
         ),
         spec={
-            **(
-                {
-                    "license": config.get("/emqx/license"),
-                }
-                if config.get("/emqx/license", "")
-                else {}
-            ),
-            "replicas": config.get("/emqx/replicas", 1),
-            "volumeClaimTemplates": config.get("/emqx/persistent-volume", {}),
-            "template": {
+            "image": config.get("/emqx/image", "emqx/emqx-enterprise:5.0.4"),
+            "coreTemplate": {
                 "spec": {
-                    "emqxContainer": {
-                        "image": {
-                            "version": config.get("/emqx/version", "4.4.14"),
-                            "repository": "emqx/emqx-ee",
-                        },
-                        "emqxConfig": config.get("/emqx/emqx-config", {}),
-                        "emqxACL": config.get("/emqx/emqx-acl", []),
-                    }
+                    "podSecurityContext": {
+                        "runAsUser": 1000,
+                        "runAsGroup": 1000,
+                        "fsGroup": 1000,
+                        "fsGroupChangePolicy": "Always",
+                        "supplementalGroups": [1000],
+                    },
+                    "volumeClaimTemplates": {
+                        **emqx_pvc_template,
+                        **(config.get("/emqx/pvc-template", {})),
+                    },
+                    "replicas": config.get("/emqx/replicas", 1),
+                    **(config.get("/emqx/core-template", {})),
                 }
             },
-            "persistent": {
-                **emqx_pvc_template,
-                **(config.get("/emqx/pvc-template", {})),
+            "replicantTemplate": {
+                "spec": {
+                    **(config.get("/emqx/replicant-template", {})),
+                    "replicas": config.get("/emqx/replicas", 0),
+                },
+            },
+            "listenersServiceTemplate": {
+                **(config.get("/emqx/service-template", {})),
+                "metadata": {
+                    "name": "emqx-ee",
+                },
             },
         },
-        opts=pulumi.ResourceOptions(depends_on=[emqx_release]),
+        opts=pulumi.ResourceOptions(depends_on=[resources["emqx"]]),
     )
 
     resources["emqx-ee"] = emqx_ee

diff --git a/deployment/kubernetes/growatt.py b/deployment/kubernetes/growatt.py
@@ -76,6 +76,16 @@ def deploy():
                             image=docker_image,
                             env=growatt_env,
                             env_from=config.get("/growatt/env-from", []),
+                            resources=config.get(
+                                "/growatt/resources",
+                                {
+                                    "requests": {
+                                        "cpu": "10m",
+                                        "memory": "64Mi",
+                                        "ephemeral-storage": "10Mi",
+                                    }
+                                },
+                            ),
                         )
                     ],
                 ),