From b8cdcac7ab80546250b15c01274f6ec782b3d98f Mon Sep 17 00:00:00 2001 From: violetaperezandrade Date: Fri, 19 Jan 2024 19:34:02 -0300 Subject: [PATCH 1/6] dummy --- dummy | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 dummy diff --git a/dummy b/dummy new file mode 100644 index 0000000..e69de29 From 5d7cd9902858b27281893290a1e7ca653c42432d Mon Sep 17 00:00:00 2001 From: violetaperezandrade Date: Fri, 19 Jan 2024 19:50:14 -0300 Subject: [PATCH 2/6] add middleware --- dummy | 0 src/common/middleware.py | 51 ++++++++++++++++++++++++++++++++++++++++ src/main.py | 6 ++++- 3 files changed, 56 insertions(+), 1 deletion(-) delete mode 100644 dummy create mode 100644 src/common/middleware.py diff --git a/dummy b/dummy deleted file mode 100644 index e69de29..0000000 diff --git a/src/common/middleware.py b/src/common/middleware.py new file mode 100644 index 0000000..77cfc30 --- /dev/null +++ b/src/common/middleware.py @@ -0,0 +1,51 @@ +import pika +import os + + +class Middleware: + + def __init__(self): + self._connection = (pika.BlockingConnection + (pika.ConnectionParameters(host=os.environ.get + ("RABBITMQ_HOST")))) + self._channel = self._connection.channel() + self._exit = False + self._remake = False + + def create_queue(self, queue_name): + self._channel.queue_declare(queue=queue_name) + + def _setup_message_consumption(self, queue_name, user_function): + self._channel.basic_consume(queue=queue_name, + on_message_callback=lambda channel, + method, properties, body: + (user_function(body), + channel.basic_ack + (delivery_tag=method.delivery_tag), + self._verify_connection_end())) + self._channel.start_consuming() + + def _verify_connection_end(self): + if self._exit: + self._channel.close() + if self._remake: + self._exit = False + self._channel = self._connection.channel() + + def finish(self, open_new_channel=False): + self._exit = True + self._remake = open_new_channel + + # Work queue methods + def listen_on(self, queue_name, user_function): + self.create_queue(queue_name) + self._channel.basic_qos(prefetch_count=30) + self._setup_message_consumption(queue_name, user_function) + + def send_message(self, queue_name, message): + self._channel.basic_publish(exchange='', + routing_key=queue_name, + body=message) + + def __del__(self): + self._connection.close() \ No newline at end of file diff --git a/src/main.py b/src/main.py index 47b3ec1..5a6b994 100644 --- a/src/main.py +++ b/src/main.py @@ -11,11 +11,15 @@ import time, random import logging +import json +from common.middleware import Middleware from data_packet import generate_data, create_packet, data_has_changed def simulate_packets(config): + middleware = Middleware() + middleware.create_queue("device_plant") last_sent_packet = None while True: try: @@ -25,7 +29,7 @@ def simulate_packets(config): if not current_packet or not data_has_changed(current_packet, last_sent_packet, config["deviations"]): continue - # TODO: Send packet to the RabbitMQ queue + middleware.send_message("device_plant", json.dumps(current_packet)) logging.info(f"Packet sent: {current_packet}") last_sent_packet = current_packet From b589d2ddc10e0e3c3f5b59468cb495a81b8ec475 Mon Sep 17 00:00:00 2001 From: violetaperezandrade Date: Mon, 22 Jan 2024 12:23:10 -0300 Subject: [PATCH 3/6] dockerize --- Dockerfile | 10 ++++++++++ Makefile | 21 +++++++++++++++++++++ docker-compose.yaml | 32 ++++++++++++++++++++++++++++++++ requirements.txt | 1 + src/rabbitmq/Dockerfile | 3 +++ 5 files changed, 67 insertions(+) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 docker-compose.yaml create mode 100644 requirements.txt create mode 100644 src/rabbitmq/Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..35b8645 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.9.7-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src src + +CMD ["python", "src/main.py"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..751dfba --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +default: docker-compose-up + +all: + +docker-image: + docker build -f Dockerfile -t main-app . + docker build -f src/rabbitmq/Dockerfile -t rabbitmq ./src/rabbitmq +.PHONY: docker-image + +docker-compose-up: docker-image + docker-compose -f docker-compose.yaml up -d --build +.PHONY: docker-compose-up + +docker-compose-down: + docker-compose -f docker-compose.yaml stop -t 20 + docker-compose -f docker-compose.yaml down --remove-orphans +.PHONY: docker-compose-down + +docker-compose-logs: + docker-compose -f docker-compose.yaml logs -f +.PHONY: docker-compose-logs \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..26a9854 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,32 @@ +version: '3.9' + +services: + main-app: + build: + context: . + dockerfile: Dockerfile + image: main-app + env_file: + - .env + depends_on: + rabbitmq: + condition: service_healthy + rabbitmq: + build: + context: ./src/rabbitmq + dockerfile: Dockerfile + image: rabbitmq + ports: + - "5672:5672" + - "15672:15672" + healthcheck: + test: + [ + "CMD", + "rabbitmq-diagnostics", + "check_port_connectivity" + ] + interval: 5s + timeout: 3s + retries: 10 + start_period: 50s \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..df7f423 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pika diff --git a/src/rabbitmq/Dockerfile b/src/rabbitmq/Dockerfile new file mode 100644 index 0000000..662c821 --- /dev/null +++ b/src/rabbitmq/Dockerfile @@ -0,0 +1,3 @@ +FROM rabbitmq:3.12-management +RUN apt-get update +RUN apt-get install -y curl \ No newline at end of file From 93a3c45d48442e5813aabf865305c9e32b838dcf Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 29 Jan 2024 22:19:43 -0300 Subject: [PATCH 4/6] address pr comments: -documentate .env file -add running instructions -connect to measurements rabbit instance --- .env.dist | 5 ++++ Makefile | 1 - README.md | 14 ++++++++++ docker-compose.yaml | 28 +++++--------------- requirements.txt | 2 ++ src/common/middleware.py | 10 +++++--- src/data_packet.py | 50 +++++++++++++----------------------- src/main.py | 55 ++++++++++++++++++++++++++++++++-------- src/rabbitmq/Dockerfile | 3 --- src/requirements.txt | 2 -- 10 files changed, 95 insertions(+), 75 deletions(-) create mode 100644 .env.dist delete mode 100644 src/rabbitmq/Dockerfile delete mode 100644 src/requirements.txt diff --git a/.env.dist b/.env.dist new file mode 100644 index 0000000..9b00d58 --- /dev/null +++ b/.env.dist @@ -0,0 +1,5 @@ +RABBITMQ_HOST= +QUEUE_NAME= +LOGGING_LEVEL= +WEATHER_API_KEY= +WEATHER_API_URL= \ No newline at end of file diff --git a/Makefile b/Makefile index 751dfba..2a6b8a7 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,6 @@ all: docker-image: docker build -f Dockerfile -t main-app . - docker build -f src/rabbitmq/Dockerfile -t rabbitmq ./src/rabbitmq .PHONY: docker-image docker-compose-up: docker-image diff --git a/README.md b/README.md index 72bf8c0..77ef73f 100644 --- a/README.md +++ b/README.md @@ -20,3 +20,17 @@ This parameters are fetched from the [OpenWeatherMap API](https://openweathermap ## Commands It would be nice to accept commands from TUI to simulate deviations fixes. Something like ` ` + +## Usage Instructions +The repository includes a **Makefile** that encapsulates various commands used frequently in the project as targets. The targets are executed by invoking: + +* **make \**: +The essential targets to start and stop the system are **docker-compose-up** and **docker-compose-down**, with the remaining targets being useful for debugging and troubleshooting. + +Available targets are: +* **docker-compose-up**: Initializes the development environment (builds docker images for the server and client, initializes the network used by docker, etc.) and starts the containers of the applications that make up the project. +* **docker-compose-down**: Performs a `docker-compose stop` to stop the containers associated with the compose and then performs a `docker-compose down` to destroy all resources associated with the initialized project. It is recommended to execute this command at the end of each run to prevent the host machine's disk from filling up. +* **docker-compose-logs**: Allows viewing the current logs of the project. Use with `grep` to filter messages from a specific application within the compose. +* **docker-image**: Builds the images to be used. This target is used by **docker-compose-up**, so it can be used to test new changes in the images before starting the project. + +Important Note: This service assumes a running instance of RabbitMQ and connects to it. Therefore, to run this service, it is necessary to first have the **measurements** service running. Please make sure to also check [measurements repository](https://github.com/Hanagotchi/measurements). diff --git a/docker-compose.yaml b/docker-compose.yaml index 26a9854..0d1b974 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,25 +8,9 @@ services: image: main-app env_file: - .env - depends_on: - rabbitmq: - condition: service_healthy - rabbitmq: - build: - context: ./src/rabbitmq - dockerfile: Dockerfile - image: rabbitmq - ports: - - "5672:5672" - - "15672:15672" - healthcheck: - test: - [ - "CMD", - "rabbitmq-diagnostics", - "check_port_connectivity" - ] - interval: 5s - timeout: 3s - retries: 10 - start_period: 50s \ No newline at end of file + networks: + - common_network + +networks: + common_network: + external: true diff --git a/requirements.txt b/requirements.txt index df7f423..2cb404a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ pika +requests +python-dotenv \ No newline at end of file diff --git a/src/common/middleware.py b/src/common/middleware.py index 77cfc30..a95d45c 100644 --- a/src/common/middleware.py +++ b/src/common/middleware.py @@ -5,9 +5,10 @@ class Middleware: def __init__(self): - self._connection = (pika.BlockingConnection - (pika.ConnectionParameters(host=os.environ.get - ("RABBITMQ_HOST")))) + rabbitmq_host = os.environ.get("RABBITMQ_HOST", "localhost") + self._connection = pika.BlockingConnection( + pika.ConnectionParameters(host=rabbitmq_host) + ) self._channel = self._connection.channel() self._exit = False self._remake = False @@ -43,9 +44,10 @@ def listen_on(self, queue_name, user_function): self._setup_message_consumption(queue_name, user_function) def send_message(self, queue_name, message): + print(f"Message sending: {message} to {queue_name}") self._channel.basic_publish(exchange='', routing_key=queue_name, body=message) def __del__(self): - self._connection.close() \ No newline at end of file + self._connection.close() diff --git a/src/data_packet.py b/src/data_packet.py index 692c871..a92580f 100644 --- a/src/data_packet.py +++ b/src/data_packet.py @@ -4,9 +4,15 @@ from os import environ from typing import Tuple from datetime import datetime +import logging import math +import uuid +import random load_dotenv() +logging.getLogger("urllib3").setLevel(logging.WARNING) +UUID = str(uuid.uuid4()).replace("-", "") + def fetch_temperature_and_humidity(location: str) -> Tuple[int, int]: @@ -18,7 +24,7 @@ def fetch_temperature_and_humidity(location: str) -> Tuple[int, int]: if not res.ok: raise Exception( - "Could fetch temperature and humidity from weather API" + "Could not fetch temperature and humidity from weather API" ) result = res.json() @@ -65,14 +71,12 @@ def fetch_watering(): return round(watering_simulator(x)) -def create_packet(temperature: int = None, - humidity: int = None, - light: int = None, - watering: int = None): +def create_packet(temperature: float = None, humidity: float = None, + light: float = None, watering: float = None): ''' Creates a data packet with simulated data, validating the data before that. - If all the parameters are empty, returns None. Else, return a dictionary + If all the parameters are empty, returns None. Else, return a dictionary with the data. @@ -81,27 +85,7 @@ def create_packet(temperature: int = None, percentages. - Light has to be positive or 0. ''' - - if temperature is None and humidity is None and light is None and watering is None: - return None - - if humidity < 0 or humidity > 100: - raise Exception( - f"Humidity has to be between 0 and 100. Current value: {humidity}" - ) - - if watering < 0 or watering > 100: - raise Exception( - f"Watering has to be between 0 and 100. Current value: {watering}" - ) - - if light < 0: - raise Exception( - f"Light has to be positive or 0. Current value: {light}" - ) - -def create_packet(temperature: float = None, humidity: float = None, light: float = None, watering: float = None): - if not (temperature and humidity and light and watering): + if not (temperature or humidity or light or watering): return None if humidity < 0 or humidity > 100: @@ -113,12 +97,13 @@ def create_packet(temperature: float = None, humidity: float = None, light: floa if light < 0: raise Exception(f"Light has to be positive or 0. Current value: {light}") - return { "temperature": temperature, "humidity": humidity, "light": light, - "watering": watering + "watering": watering, + "time_stamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "id_device": UUID } @@ -133,15 +118,16 @@ def generate_data(location="Pilar, AR") -> Tuple[int, int, int, int]: ''' temperature, humidity = fetch_temperature_and_humidity(location) + # esto no va a producir cambios instantaneamente, pero es para probar light = fetch_solar_irradiation() watering = fetch_watering() - return temperature, humidity, light, watering + return temperature, humidity, light, watering def data_has_changed(current, last_sent, deviations): ''' - Compares the current packet and the last sent packet, + Compares the current packet and the last sent packet, based in the deviations. If any parameter differs enough from the last sent packet, then the packet @@ -162,7 +148,7 @@ def data_has_changed(current, last_sent, deviations): - deviations: {temperature: float, humidity: float, light: float, watering: float} ''' - + if not last_sent: return True diff --git a/src/main.py b/src/main.py index 9c9e32a..53f8e51 100644 --- a/src/main.py +++ b/src/main.py @@ -13,32 +13,43 @@ import random import logging import json +import os from common.middleware import Middleware from data_packet import generate_data, create_packet, data_has_changed def simulate_packets(config): + print(f"Simulatingg packets") middleware = Middleware() - middleware.create_queue("device_plant") + queue_name = os.environ.get("QUEUE_NAME") + print(f"creatingg queueee: {queue_name}") + middleware.create_queue(queue_name) last_sent_packet = None + current_packet = None while True: try: temperature, humidity, light, watering = generate_data() - current_packet = create_packet(temperature, humidity, light, watering) + current_packet = create_packet(temperature, humidity, light, + watering) + print(f"packet created: {current_packet}") - if not current_packet or not data_has_changed(current_packet, last_sent_packet, config["deviations"]): + if not current_packet or not data_has_changed( + current_packet, + last_sent_packet, + config["deviations"] + ): + print("no se envia") continue - - middleware.send_message("device_plant", json.dumps(current_packet)) + middleware.send_message(queue_name, json.dumps(current_packet)) + print("holaaaa") logging.info(f"Packet sent: {current_packet}") last_sent_packet = current_packet - except Exception as err: - logging.warning(err) + logging.warning(f"{err}") finally: - print(current_packet) + # print(current_packet) time.sleep(config["packet_period"]) @@ -59,8 +70,30 @@ def read_config_file(path): } -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) +def main(): + logging_level = os.environ.get("LOGGING_LEVEL") + print(f"logging level: {logging_level}") + initialize_log(logging_level) config = read_config_file("") - + print(f"config: {config}") simulate_packets(config) + + +def initialize_log(logging_level): + """ + Python custom logging initialization + + Current timestamp is added to be able to identify in docker + compose logs the date when the log has arrived + """ + logging.basicConfig( + format='%(asctime)s %(levelname)-8s %(message)s', + level=logging_level, + datefmt='%Y-%m-%d %H:%M:%S', + ) + logging.getLogger("pika").setLevel(logging.WARNING) + + +if __name__ == '__main__': + print("starting") + main() diff --git a/src/rabbitmq/Dockerfile b/src/rabbitmq/Dockerfile deleted file mode 100644 index 662c821..0000000 --- a/src/rabbitmq/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM rabbitmq:3.12-management -RUN apt-get update -RUN apt-get install -y curl \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt deleted file mode 100644 index 8804ab6..0000000 --- a/src/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -python-dotenv \ No newline at end of file From 730ea9410237ee5f4c5fa93c9ac79b64cf441373 Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 29 Jan 2024 22:21:36 -0300 Subject: [PATCH 5/6] add linter --- .github/workflows/linters.yml | 31 +++++++++++++++++++++++++++++++ tox.ini | 2 ++ 2 files changed, 33 insertions(+) create mode 100644 .github/workflows/linters.yml create mode 100644 tox.ini diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml new file mode 100644 index 0000000..a12f030 --- /dev/null +++ b/.github/workflows/linters.yml @@ -0,0 +1,31 @@ + +name: Linters + +on: 'push' + +jobs: + run-linters: + name: Run linters + runs-on: ubuntu-latest + + steps: + - name: Check out Git repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install Python dependencies + run: pip install black flake8 + + - name: Run linters + uses: wearerequired/lint-action@v1 + with: + black: true + + - name: flake8 Lint + uses: py-actions/flake8@v1.2.0 + with: + max-line-length: "88" \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..1d36346 --- /dev/null +++ b/tox.ini @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 88 \ No newline at end of file From 3e2e76432bb43045f58a6cb7304ae5dc7715d050 Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 29 Jan 2024 22:33:30 -0300 Subject: [PATCH 6/6] fix linter? + remove prints --- src/common/middleware.py | 1 - src/data_packet.py | 22 ++++++++++++++-------- src/main.py | 10 +--------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/common/middleware.py b/src/common/middleware.py index a95d45c..1a48914 100644 --- a/src/common/middleware.py +++ b/src/common/middleware.py @@ -44,7 +44,6 @@ def listen_on(self, queue_name, user_function): self._setup_message_consumption(queue_name, user_function) def send_message(self, queue_name, message): - print(f"Message sending: {message} to {queue_name}") self._channel.basic_publish(exchange='', routing_key=queue_name, body=message) diff --git a/src/data_packet.py b/src/data_packet.py index a92580f..229ab54 100644 --- a/src/data_packet.py +++ b/src/data_packet.py @@ -7,7 +7,6 @@ import logging import math import uuid -import random load_dotenv() logging.getLogger("urllib3").setLevel(logging.WARNING) @@ -89,13 +88,16 @@ def create_packet(temperature: float = None, humidity: float = None, return None if humidity < 0 or humidity > 100: - raise Exception(f"Humidity has to be between 0 and 100. Current value: {humidity}") + raise Exception(f"Humidity has to be between 0 and 100." + f"Current value: {humidity}") if watering < 0 or watering > 100: - raise Exception(f"Watering has to be between 0 and 100. Current value: {watering}") + raise Exception(f"Watering has to be between 0 and 100. " + f"Current value: {watering}") if light < 0: - raise Exception(f"Light has to be positive or 0. Current value: {light}") + raise Exception(f"Light has to be positive or 0. " + f"Current value: {light}") return { "temperature": temperature, @@ -152,10 +154,14 @@ def data_has_changed(current, last_sent, deviations): if not last_sent: return True - if parameter_has_changed(current["temperature"], last_sent["temperature"], deviations["temperature"])\ - or parameter_has_changed(current["humidity"], last_sent["humidity"], deviations["humidity"])\ - or parameter_has_changed(current["light"], last_sent["light"], deviations["light"])\ - or parameter_has_changed(current["watering"], last_sent["watering"], deviations["watering"]): + if parameter_has_changed(current["temperature"], last_sent["temperature"], + deviations["temperature"])\ + or parameter_has_changed(current["humidity"], last_sent["humidity"], + deviations["humidity"])\ + or parameter_has_changed(current["light"], last_sent["light"], + deviations["light"])\ + or parameter_has_changed(current["watering"], last_sent["watering"], + deviations["watering"]): return True return False diff --git a/src/main.py b/src/main.py index 53f8e51..1f627ee 100644 --- a/src/main.py +++ b/src/main.py @@ -20,10 +20,8 @@ def simulate_packets(config): - print(f"Simulatingg packets") middleware = Middleware() queue_name = os.environ.get("QUEUE_NAME") - print(f"creatingg queueee: {queue_name}") middleware.create_queue(queue_name) last_sent_packet = None current_packet = None @@ -32,24 +30,21 @@ def simulate_packets(config): temperature, humidity, light, watering = generate_data() current_packet = create_packet(temperature, humidity, light, watering) - print(f"packet created: {current_packet}") if not current_packet or not data_has_changed( current_packet, last_sent_packet, config["deviations"] ): - print("no se envia") continue middleware.send_message(queue_name, json.dumps(current_packet)) - print("holaaaa") logging.info(f"Packet sent: {current_packet}") last_sent_packet = current_packet except Exception as err: logging.warning(f"{err}") finally: - # print(current_packet) + print(current_packet) time.sleep(config["packet_period"]) @@ -72,10 +67,8 @@ def read_config_file(path): def main(): logging_level = os.environ.get("LOGGING_LEVEL") - print(f"logging level: {logging_level}") initialize_log(logging_level) config = read_config_file("") - print(f"config: {config}") simulate_packets(config) @@ -95,5 +88,4 @@ def initialize_log(logging_level): if __name__ == '__main__': - print("starting") main()