-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #35 from starschema/pred_maint_containers
feat: add predictive maintenance demo containers
- Loading branch information
Showing
19 changed files
with
20,898 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
FROM python:3.10 | ||
|
||
WORKDIR /app_pred | ||
|
||
COPY requirements.txt . | ||
|
||
RUN pip install --no-cache-dir -r requirements.txt | ||
|
||
COPY . . | ||
|
||
# load or predict | ||
# ENV LOAD_MODE="load" | ||
|
||
ENTRYPOINT ["./entrypoint.sh"] |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
import os | ||
|
||
# influx | ||
token = os.environ.get("INFLUX_TOKEN") | ||
org = os.environ.get("INFLUX_ORG") or "Starschema" | ||
url_base = os.environ.get("INFLUX_URL") or "http://influxdb2" | ||
url_port = int(os.environ.get("INFLUX_PORT") or 80) | ||
url = url_base + ":" + str(url_port) | ||
|
||
bucket = os.environ.get("INFLUX_BUCKET") or "default" | ||
|
||
# mqtt | ||
broker_address = os.environ.get("MQTT_HOST") | ||
broker_port = int(os.environ.get("MQTT_PORT") or 1883) | ||
wait_time_on_publish = 6 | ||
|
||
raw_data_topic = "/engines/raw" | ||
predictions_topic = "/engines/predictions" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
import pandas as pd | ||
|
||
|
||
def get_test_data(): | ||
TEST_DATA_PATH = "test_data_looped.csv" | ||
test_set = pd.read_csv(TEST_DATA_PATH).sort_values("time_in_cycles") | ||
test_set = test_set.sort_values(by=["new_cycle", "engine_no"]) | ||
test_set = test_set.drop("new_cycle", axis=1) | ||
return test_set | ||
|
||
|
||
def rename_sensors(row, pred_feautures, anomaly_features): | ||
sensor_names = [ | ||
"Total temperature at fan inlet", | ||
"Total temperature at LPC outlet", | ||
"Total temperature at HPC outlet", | ||
"Total Temperature LPT outlet", | ||
"Pressure at fan inlet", | ||
"Total pressure in bypass-duct", | ||
"Total pressure at HPC outlet", | ||
"Physical fan speed", | ||
"Physical core speed", | ||
"Engine pressure ratio (P50/P2)", | ||
"Static pressure at HPC outlet", | ||
"Ratio of fuel flow to Ps", | ||
"Corrected fan speed", | ||
"Corrected core speed", | ||
"Bypass Ratio", | ||
"Burner fuel-air ratio", | ||
"Bleed enthalpy", | ||
"Demanded fan speed", | ||
"Demanded corrected fan speed", | ||
"HPT coolant bleed", | ||
"LPT coolant bleed", | ||
] | ||
|
||
sid_to_fname = {f"sensor_{i+1}": name for i, name in enumerate(sensor_names)} | ||
|
||
col_rename_dict = { | ||
"engine_no": "Engine ID", | ||
"time_in_cycles": "Completed cycles", | ||
"op_setting_1": "Altitude", | ||
"op_setting_2": "Mach number", | ||
**sid_to_fname, | ||
} | ||
|
||
row.index = [col_rename_dict[attr_name] for attr_name in row.index] | ||
top_predicitive_features = { | ||
f"top_{ind+1}_predictive_feature": col_rename_dict[f.replace("_avg", "")] | ||
for ind, f in enumerate(pred_feautures) | ||
} | ||
top_anomalous_features = { | ||
f"anomaly_feature_{ind+1}": col_rename_dict[f] | ||
for ind, f in enumerate(anomaly_features) | ||
} | ||
return row, top_predicitive_features, top_anomalous_features |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
#!/bin/bash | ||
|
||
# The first argument is assigned to the variable 'input' | ||
input=${LOAD_MODEL:-"predict"} | ||
|
||
# Check the input value and execute the corresponding script | ||
if [[ $input == "predict" ]]; then | ||
echo "Executing main_predict.py" | ||
python main_predict.py | ||
elif [[ $input == "load" ]]; then | ||
echo "Executing main_data_load.py" | ||
python main_data_load.py | ||
else | ||
echo "Invalid input. Please enter either 'load' or 'predict'." | ||
fi |
91 changes: 91 additions & 0 deletions
91
containers/predictive_maintenance/first_9_records_test.csv
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
engine_no,time_in_cycles,op_setting_1,op_setting_2,sensor_2,sensor_3,sensor_4,sensor_6,sensor_7,sensor_8,sensor_9,sensor_11,sensor_12,sensor_13,sensor_15,sensor_17,sensor_20,sensor_21 | ||
1,1,0.0017,0.0003,642.39,1586.1,1402.11,21.61,554.1,2388.06,9052.13,47.54,520.94,2388.09,8.4214,394,38.78,23.2348 | ||
1,2,-0.0004,0.0004,642.43,1592.56,1406.54,21.61,553.17,2388.11,9042.55,47.5,521.81,2388.12,8.4005,393,38.95,23.3202 | ||
1,3,-0.0005,-0.0,642.35,1587.14,1403.33,21.61,553.6,2388.1,9033.93,47.44,521.48,2388.11,8.4089,393,38.79,23.2922 | ||
1,4,-0.002,0.0004,642.5,1594.4,1409.25,21.61,552.67,2388.11,9046.28,47.54,521.43,2388.18,8.438,392,38.84,23.4524 | ||
1,5,0.0064,-0.0002,642.81,1584.77,1410.35,21.61,553.07,2388.06,9045.8,47.68,521.47,2388.11,8.4478,393,39.04,23.3366 | ||
1,6,0.0018,-0.0002,642.64,1586.47,1409.04,21.61,553.98,2388.09,9051.29,47.56,521.61,2388.09,8.43,393,38.86,23.2776 | ||
1,7,0.0005,-0.0003,642.54,1585.61,1411.32,21.61,553.46,2388.09,9048.64,47.62,521.57,2388.1,8.4406,392,38.8,23.1772 | ||
1,8,-0.003,0.0002,642.37,1590.47,1410.7,21.61,553.08,2388.12,9049.68,47.42,521.67,2388.12,8.4247,391,38.76,23.3676 | ||
1,9,0.001,-0.0001,641.95,1585.75,1399.4,21.61,553.66,2388.06,9048.35,47.34,521.28,2388.14,8.4223,393,38.81,23.3485 | ||
2,1,-0.001,-0.0002,641.94,1594.32,1405.5,21.61,554.14,2388.07,9051.55,47.46,520.87,2388.11,8.4393,392,38.76,23.3793 | ||
2,2,0.0039,-0.0,642.61,1592.31,1406.78,21.61,553.08,2388.11,9053.39,47.5,521.45,2388.08,8.4325,393,38.97,23.2731 | ||
2,3,-0.0034,-0.0001,642.23,1589.66,1410.36,21.61,553.29,2388.1,9061.71,47.43,521.66,2388.1,8.4642,393,38.67,23.2775 | ||
2,4,-0.0013,0.0002,642.67,1593.12,1408.66,21.61,553.71,2388.12,9047.99,47.65,521.44,2388.16,8.4115,391,38.8,23.3566 | ||
2,5,-0.0018,0.0004,642.77,1589.93,1408.31,21.61,553.21,2388.08,9057.89,47.56,521.32,2388.14,8.3731,393,39.08,23.4748 | ||
2,6,0.001,-0.0002,642.83,1589.25,1406.72,21.61,553.29,2388.15,9053.61,47.58,521.7,2388.1,8.4222,392,38.88,23.2899 | ||
2,7,-0.0011,0.0003,642.69,1591.14,1402.15,21.61,553.16,2388.09,9057.27,47.42,521.31,2388.06,8.43,392,38.92,23.285 | ||
2,8,-0.0031,0.0005,642.41,1590.39,1399.39,21.61,552.75,2388.12,9053.8,47.58,521.46,2388.12,8.4281,393,38.94,23.4308 | ||
2,9,0.0028,0.0004,642.76,1589.31,1416.61,21.61,553.56,2388.1,9053.53,47.54,521.28,2388.1,8.4141,393,38.61,23.2479 | ||
3,1,-0.0037,-0.0004,642.64,1593.48,1402.4,21.61,553.58,2388.07,9049.3,47.41,521.82,2388.08,8.3973,391,38.96,23.3836 | ||
3,2,0.0038,-0.0,641.94,1586.18,1397.07,21.61,553.94,2388.05,9048.85,47.2,521.82,2388.05,8.4227,393,38.94,23.368 | ||
3,3,-0.0014,-0.0004,642.19,1591.6,1407.08,21.61,554.51,2388.11,9049.73,47.43,521.98,2388.06,8.42,393,38.88,23.4072 | ||
3,4,-0.0002,-0.0003,642.66,1588.64,1396.73,21.61,554.21,2388.11,9038.04,47.42,522.19,2388.04,8.446,393,38.97,23.3827 | ||
3,5,-0.0022,0.0004,642.48,1589.61,1408.72,21.61,553.52,2388.09,9046.44,47.47,521.8,2388.07,8.4349,392,38.67,23.3019 | ||
3,6,-0.0017,-0.0004,642.0,1586.4,1397.39,21.61,554.33,2388.03,9060.78,47.45,522.16,2388.06,8.4302,394,39.09,23.4559 | ||
3,7,-0.0011,-0.0005,642.27,1578.13,1407.15,21.61,554.09,2388.04,9052.65,47.26,522.68,2387.99,8.4043,393,38.87,23.3655 | ||
3,8,0.0,-0.0003,642.39,1590.65,1396.75,21.61,553.93,2388.05,9039.78,47.32,522.1,2388.11,8.4106,393,38.95,23.3547 | ||
3,9,-0.0035,-0.0005,642.87,1587.52,1399.3,21.61,554.95,2388.08,9045.12,47.35,522.11,2388.05,8.4274,393,38.88,23.3217 | ||
4,1,-0.0015,0.0003,642.71,1587.27,1405.1,21.61,553.55,2387.99,9063.94,47.4,521.98,2388.0,8.3982,391,39.03,23.4821 | ||
4,2,-0.0004,0.0,642.59,1583.62,1402.26,21.61,555.09,2388.05,9068.08,47.27,521.95,2388.07,8.4185,393,39.14,23.3301 | ||
4,3,-0.0004,0.0003,641.87,1587.99,1412.41,21.61,554.45,2388.02,9064.87,47.44,522.37,2388.06,8.3964,392,38.99,23.3706 | ||
4,4,-0.0,-0.0003,642.2,1590.72,1402.53,21.61,554.32,2388.03,9066.19,47.3,521.94,2388.06,8.3994,392,38.85,23.3627 | ||
4,5,-0.0001,-0.0,642.57,1589.26,1402.45,21.61,554.33,2388.08,9066.68,47.29,521.75,2388.02,8.4047,392,38.89,23.3105 | ||
4,6,-0.0013,0.0003,641.66,1586.94,1403.95,21.61,554.48,2388.02,9067.9,47.37,521.91,2388.03,8.4041,393,39.15,23.3695 | ||
4,7,-0.0011,-0.0002,642.56,1593.65,1406.83,21.61,554.18,2388.06,9064.05,47.41,521.87,2388.04,8.4185,392,38.75,23.3598 | ||
4,8,-0.0019,0.0003,642.74,1594.24,1402.7,21.61,554.19,2388.01,9060.16,47.22,521.81,2388.06,8.4192,392,38.91,23.5189 | ||
4,9,0.0005,0.0002,642.43,1583.03,1398.76,21.61,553.2,2388.05,9056.44,47.42,522.07,2388.03,8.4073,390,38.97,23.3778 | ||
5,1,0.0018,0.0001,642.65,1585.71,1394.7,21.61,554.13,2388.14,9044.2,47.69,521.92,2388.11,8.4409,392,38.89,23.3721 | ||
5,2,-0.0004,0.0003,642.42,1593.58,1400.03,21.61,553.43,2388.08,9057.18,47.44,521.18,2388.1,8.4322,392,38.85,23.2555 | ||
5,3,-0.0021,-0.0005,642.6,1593.09,1410.91,21.61,552.92,2388.09,9055.05,47.45,521.78,2388.09,8.3939,392,38.91,23.3312 | ||
5,4,0.002,0.0,643.16,1589.06,1402.44,21.61,553.71,2388.08,9054.31,47.42,521.86,2388.1,8.4385,394,38.81,23.3245 | ||
5,5,-0.0002,-0.0005,642.07,1585.92,1406.07,21.61,553.36,2388.13,9059.62,47.3,521.3,2388.12,8.4046,394,38.91,23.3379 | ||
5,6,-0.0033,-0.0002,642.55,1593.88,1412.78,21.61,553.5,2388.12,9059.94,47.34,521.6,2388.05,8.45,392,39.04,23.3846 | ||
5,7,-0.003,0.0002,642.35,1587.41,1403.04,21.61,553.45,2388.08,9048.99,47.5,521.76,2388.04,8.4698,392,39.04,23.4062 | ||
5,8,-0.001,0.0002,642.63,1588.18,1407.92,21.61,553.93,2388.08,9050.76,47.49,521.72,2388.05,8.4035,392,38.77,23.4709 | ||
5,9,-0.0025,-0.0,642.71,1572.98,1400.44,21.61,553.78,2388.11,9048.7,47.46,521.76,2388.11,8.4011,392,38.87,23.2853 | ||
6,1,-0.0007,0.0003,642.2,1586.71,1398.93,21.61,554.78,2387.93,9055.53,47.12,522.03,2388.05,8.4242,392,39.12,23.4108 | ||
6,2,-0.0007,0.0001,641.7,1580.85,1403.05,21.61,554.12,2388.0,9067.05,47.22,522.72,2387.97,8.3931,391,39.05,23.4484 | ||
6,3,0.0014,-0.0001,642.05,1577.46,1404.75,21.61,554.38,2387.99,9056.05,47.03,522.66,2388.04,8.4203,392,39.21,23.4586 | ||
6,4,-0.0019,0.0004,642.6,1586.51,1399.61,21.61,553.7,2388.04,9063.07,47.18,522.84,2387.96,8.4187,392,38.94,23.3548 | ||
6,5,-0.0011,0.0002,642.2,1584.74,1401.17,21.61,554.29,2388.04,9069.06,47.15,522.37,2387.99,8.374,394,39.0,23.4887 | ||
6,6,0.0025,0.0005,641.92,1583.24,1399.53,21.61,554.33,2387.96,9056.89,47.21,522.4,2387.97,8.4409,392,39.01,23.4372 | ||
6,7,-0.0003,0.0005,642.34,1586.99,1394.65,21.61,554.63,2388.03,9061.36,47.49,522.45,2388.01,8.3901,392,39.03,23.3065 | ||
6,8,-0.0014,-0.0003,642.14,1583.18,1402.85,21.61,554.72,2387.99,9055.8,47.01,522.85,2387.97,8.4037,393,38.85,23.4653 | ||
6,9,-0.0012,0.0004,642.5,1584.76,1396.58,21.61,554.6,2388.03,9067.66,47.15,522.32,2387.98,8.4054,392,38.88,23.3633 | ||
7,1,0.0021,0.0001,642.26,1588.45,1400.02,21.61,552.94,2388.09,9065.41,47.61,521.18,2388.07,8.43,392,38.81,23.418 | ||
7,2,-0.0009,0.0004,642.59,1590.0,1407.22,21.61,552.5,2388.11,9059.25,47.48,522.08,2388.12,8.4447,394,38.79,23.3135 | ||
7,3,-0.0008,-0.0,642.82,1586.05,1408.64,21.61,553.75,2388.09,9063.08,47.45,521.93,2388.03,8.4202,392,38.87,23.414 | ||
7,4,0.0005,-0.0005,642.24,1591.24,1398.32,21.61,553.56,2388.09,9057.79,47.52,521.85,2388.05,8.4472,393,38.91,23.2512 | ||
7,5,-0.0005,-0.0004,642.32,1580.76,1399.39,21.61,554.21,2388.04,9059.36,47.43,521.6,2388.08,8.4073,391,39.02,23.3361 | ||
7,6,-0.0042,-0.0002,642.43,1582.35,1402.19,21.61,554.3,2388.07,9052.55,47.47,521.32,2388.08,8.4223,392,39.02,23.3396 | ||
7,7,-0.0003,0.0003,642.54,1583.29,1394.98,21.61,553.55,2388.07,9054.04,47.44,521.24,2388.12,8.3992,393,38.75,23.4152 | ||
7,8,0.0046,-0.0004,642.39,1590.11,1400.12,21.61,553.2,2388.03,9058.59,47.39,521.89,2388.05,8.4073,392,38.91,23.2867 | ||
7,9,0.003,-0.0005,642.85,1577.69,1402.84,21.61,553.31,2388.05,9062.32,47.26,521.76,2388.08,8.4612,394,38.9,23.2371 | ||
8,1,-0.0014,0.0004,642.62,1591.75,1411.32,21.61,553.71,2388.07,9052.57,47.55,521.45,2388.1,8.4655,392,38.92,23.2762 | ||
8,2,-0.0004,0.0004,642.46,1591.88,1406.97,21.61,553.44,2388.13,9043.86,47.3,521.67,2388.1,8.4789,392,38.88,23.2764 | ||
8,3,0.0042,0.0,641.78,1585.31,1400.54,21.61,553.67,2388.12,9051.11,47.39,521.46,2388.09,8.435,394,38.82,23.2387 | ||
8,4,0.0027,-0.0003,642.71,1592.71,1410.04,21.61,553.22,2388.13,9050.29,47.38,521.28,2388.14,8.4365,393,38.79,23.4034 | ||
8,5,-0.0001,0.0004,642.09,1587.59,1405.8,21.61,553.73,2388.1,9048.07,47.56,521.67,2388.11,8.4685,391,38.8,23.3085 | ||
8,6,0.0037,0.0001,642.85,1586.86,1405.66,21.61,553.39,2388.11,9051.76,47.5,521.45,2388.12,8.4695,392,38.83,23.2958 | ||
8,7,-0.0003,-0.0001,642.26,1583.78,1417.79,21.61,553.33,2388.06,9047.15,47.54,522.18,2388.11,8.4786,393,38.85,23.3976 | ||
8,8,0.0001,-0.0,642.4,1587.29,1405.13,21.61,553.6,2388.11,9045.31,47.56,522.0,2388.1,8.4239,393,38.93,23.3802 | ||
8,9,0.0044,0.0001,642.72,1589.08,1407.47,21.61,554.43,2388.07,9051.04,47.53,521.41,2388.1,8.4484,392,39.04,23.3733 | ||
9,1,-0.0032,-0.0001,642.51,1579.7,1402.95,21.61,554.54,2388.07,9060.7,47.13,522.42,2388.03,8.4373,391,39.03,23.3501 | ||
9,2,-0.0029,0.0001,642.06,1583.52,1394.46,21.61,553.95,2388.02,9062.06,47.04,521.66,2388.03,8.4357,392,38.99,23.3998 | ||
9,3,-0.0035,-0.0004,641.88,1583.81,1388.81,21.61,554.1,2388.06,9062.63,47.42,522.2,2387.99,8.4127,393,38.94,23.381 | ||
9,4,0.0013,0.0004,641.64,1584.57,1395.56,21.61,554.0,2388.05,9061.06,47.11,522.34,2388.0,8.379,390,39.03,23.4494 | ||
9,5,0.0023,0.0004,642.66,1583.04,1394.77,21.61,554.81,2388.08,9062.0,47.19,522.28,2387.97,8.397,392,38.94,23.31 | ||
9,6,0.0,-0.0004,642.15,1585.54,1398.23,21.61,553.91,2387.99,9066.45,47.53,523.04,2387.98,8.4252,392,38.9,23.2549 | ||
9,7,-0.0007,0.0001,642.17,1592.67,1398.81,21.61,553.88,2388.09,9054.66,47.25,522.31,2388.05,8.3991,393,39.05,23.4584 | ||
9,8,-0.0024,0.0001,641.94,1577.83,1399.59,21.61,554.3,2388.05,9065.07,47.43,522.55,2387.98,8.4123,391,38.99,23.4375 | ||
9,9,-0.0,0.0005,642.07,1590.77,1404.51,21.61,554.58,2387.97,9057.62,47.24,522.33,2388.01,8.4321,395,39.08,23.339 | ||
10,1,0.0019,-0.0002,642.3,1580.93,1404.55,21.61,554.16,2388.0,9073.83,47.25,522.3,2387.97,8.3792,392,39.01,23.4547 | ||
10,2,0.0025,0.0,642.13,1585.29,1394.47,21.61,552.75,2388.03,9070.74,47.03,521.8,2388.03,8.4158,392,38.91,23.3055 | ||
10,3,-0.0011,-0.0005,642.35,1582.27,1396.33,21.61,555.45,2387.94,9068.22,47.22,522.47,2387.99,8.4124,391,39.16,23.3677 | ||
10,4,-0.0009,-0.0004,641.8,1585.23,1398.98,21.61,554.99,2387.98,9068.93,47.07,522.19,2387.99,8.45,391,39.07,23.3832 | ||
10,5,-0.0011,0.0001,641.85,1592.31,1397.87,21.61,554.04,2388.0,9070.44,47.1,521.96,2387.92,8.4068,392,39.03,23.4866 | ||
10,6,0.0019,-0.0005,642.46,1581.0,1400.8,21.61,554.18,2387.99,9070.09,47.26,522.69,2387.95,8.3922,390,39.05,23.5227 | ||
10,7,-0.0042,-0.0003,642.08,1578.64,1405.97,21.61,554.72,2388.09,9066.95,47.21,522.58,2388.05,8.3928,392,38.92,23.3819 | ||
10,8,-0.0018,-0.0005,642.1,1583.29,1402.88,21.61,554.15,2388.0,9074.48,47.08,522.6,2388.04,8.3901,391,38.94,23.4539 | ||
10,9,0.0027,0.0004,642.22,1588.95,1393.89,21.61,554.81,2388.03,9071.66,47.15,522.75,2388.0,8.3943,392,38.94,23.4024 |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
import time | ||
import traceback | ||
import json | ||
import config | ||
import pandas as pd | ||
from publish import publish_msg | ||
from data_help_funcs import get_test_data | ||
import warnings | ||
|
||
warnings.filterwarnings("ignore") | ||
|
||
test_set = get_test_data() | ||
|
||
|
||
def generate(): | ||
while True: | ||
for idx, row in test_set.iterrows(): | ||
try: | ||
formated_msg = {**row} | ||
formated_msg = json.dumps(formated_msg) | ||
print(formated_msg) | ||
publish_msg(formated_msg, config.raw_data_topic) | ||
except Exception as exc: | ||
print(traceback.format_exc()) | ||
print(f"issue {exc}") | ||
|
||
|
||
if __name__ == "__main__": | ||
generate() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
import time | ||
import traceback | ||
import json | ||
import threading | ||
import paho.mqtt.client as mqtt | ||
import pandas as pd | ||
import influxdb_client, os, time | ||
from influxdb_client import InfluxDBClient, Point, WritePrecision | ||
from influxdb_client.client.write_api import SYNCHRONOUS | ||
from influxdb_client.client.write_api import WriteApi | ||
import config | ||
from pred_maintenance_and_anomaly import get_model_dict | ||
from data_help_funcs import rename_sensors | ||
from publish import publish_msg | ||
import warnings | ||
|
||
warnings.filterwarnings("ignore") | ||
|
||
try: | ||
write_client = influxdb_client.InfluxDBClient( | ||
url=config.url, token=config.token, org=config.org | ||
) | ||
except: | ||
print("Failed to init influx conn...") | ||
|
||
model_dict = get_model_dict() | ||
|
||
|
||
def predict_on_msg(client, userdata, message): | ||
print(f"Received message: {message.payload.decode('utf-8')}") | ||
row = json.loads(message.payload.decode("utf-8")) | ||
for k, v in row.items(): | ||
row[k] = float(v) | ||
row["engine_no"] = int(row["engine_no"]) | ||
row = pd.Series(row) | ||
|
||
engine_no = row["engine_no"] | ||
data = model_dict[engine_no].get_scores(row) | ||
row, top_predicitive_features, top_anomalous_features = rename_sensors( | ||
row, data[1], data[3] | ||
) | ||
|
||
formated_msg_orig = { | ||
"Probability of failure within 30 cycles": data[0], | ||
**top_predicitive_features, | ||
"Anomaly score": data[2], | ||
**top_anomalous_features, | ||
**row, | ||
} | ||
formated_msg = json.dumps(formated_msg_orig) | ||
publish_msg(formated_msg, config.predictions_topic) | ||
|
||
print(f"Sent msg: {formated_msg}") | ||
|
||
try: | ||
influx_send_msg(formated_msg_orig, engine_no) | ||
except Exception as exc: | ||
print(f"Failed to send data to influx due to: {exc}") | ||
|
||
|
||
def influx_send_msg(formated_msg_orig, engine_no): | ||
write_api = write_client.write_api(write_options=SYNCHRONOUS) | ||
|
||
for key, value in formated_msg_orig.items(): | ||
point = Point("measurement").tag("engine_no", int(engine_no)).field(key, value) | ||
point2 = Point("measurement2").field(key, value) | ||
write_api.write(bucket=config.bucket, org=config.org, record=point) | ||
write_api.write(bucket=config.bucket, org=config.org, record=point2) | ||
# time.sleep(0.1) # separate points by 1 second | ||
|
||
print("shared") | ||
|
||
|
||
def listen_to_mqtt_topic(): | ||
client = mqtt.Client() | ||
client.connect(config.broker_address, config.broker_port) | ||
client.subscribe(config.raw_data_topic) | ||
|
||
client.on_message = predict_on_msg | ||
|
||
client.loop_start() | ||
time.sleep(30) | ||
client.loop_stop() | ||
return True | ||
|
||
|
||
class MQTTWatcher(threading.Thread): | ||
def __init__(self): | ||
super().__init__() | ||
self._stop_event = threading.Event() | ||
|
||
def run(self): | ||
while not self._stop_event.is_set(): | ||
listen_to_mqtt_topic() | ||
|
||
def stop(self): | ||
self._stop_event.set() | ||
|
||
|
||
def main(): | ||
while True: | ||
mqtt_watcher = MQTTWatcher() | ||
mqtt_watcher.start() | ||
|
||
mqtt_watcher.join(timeout=36) | ||
|
||
if mqtt_watcher.is_alive(): | ||
print("Thread is still running, terminating...") | ||
mqtt_watcher.stop() | ||
else: | ||
print("Thread has finished") | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
Oops, something went wrong.