From 4cde87d6940d1f74d032c74d5bc9caaf7b1200c5 Mon Sep 17 00:00:00 2001 From: drorganvidez Date: Tue, 15 Oct 2024 13:50:01 +0200 Subject: [PATCH] feat: Implement statistics module --- app/modules/dataset/repositories.py | 10 +--- app/modules/dataset/services.py | 11 ++-- app/modules/hubfile/routes.py | 4 ++ app/modules/hubfile/services.py | 10 +--- app/modules/public/routes.py | 10 ++-- app/modules/statistics/__init__.py | 3 + app/modules/statistics/assets/scripts.js | 1 + app/modules/statistics/forms.py | 6 ++ app/modules/statistics/models.py | 16 ++++++ app/modules/statistics/repositories.py | 55 +++++++++++++++++++ app/modules/statistics/routes.py | 7 +++ app/modules/statistics/seeders.py | 14 +++++ app/modules/statistics/services.py | 37 +++++++++++++ .../templates/statistics/index.html | 11 ++++ app/modules/statistics/tests/__init__.py | 0 app/modules/statistics/tests/locustfile.py | 21 +++++++ app/modules/statistics/tests/test_selenium.py | 35 ++++++++++++ app/modules/statistics/tests/test_unit.py | 24 ++++++++ migrations/versions/011.py | 35 ++++++++++++ 19 files changed, 284 insertions(+), 26 deletions(-) create mode 100644 app/modules/statistics/__init__.py create mode 100644 app/modules/statistics/assets/scripts.js create mode 100644 app/modules/statistics/forms.py create mode 100644 app/modules/statistics/models.py create mode 100644 app/modules/statistics/repositories.py create mode 100644 app/modules/statistics/routes.py create mode 100644 app/modules/statistics/seeders.py create mode 100644 app/modules/statistics/services.py create mode 100644 app/modules/statistics/templates/statistics/index.html create mode 100644 app/modules/statistics/tests/__init__.py create mode 100644 app/modules/statistics/tests/locustfile.py create mode 100644 app/modules/statistics/tests/test_selenium.py create mode 100644 app/modules/statistics/tests/test_unit.py create mode 100644 migrations/versions/011.py diff --git a/app/modules/dataset/repositories.py b/app/modules/dataset/repositories.py index c835715c..2c15f31b 100644 --- a/app/modules/dataset/repositories.py +++ b/app/modules/dataset/repositories.py @@ -4,7 +4,7 @@ from typing import List, Optional import pytz -from sqlalchemy import desc, func +from sqlalchemy import desc from app.modules.dataset.models import ( Author, @@ -28,10 +28,6 @@ class DSDownloadRecordRepository(BaseRepository): def __init__(self): super().__init__(DSDownloadRecord) - def total_dataset_downloads(self) -> int: - max_id = self.model.query.with_entities(func.max(self.model.id)).scalar() - return max_id if max_id is not None else 0 - def the_record_exists(self, dataset: DataSet, user_cookie: str): return self.model.query.filter_by( user_id=current_user.id if current_user.is_authenticated else None, @@ -60,10 +56,6 @@ class DSViewRecordRepository(BaseRepository): def __init__(self): super().__init__(DSViewRecord) - def total_dataset_views(self) -> int: - max_id = self.model.query.with_entities(func.max(self.model.id)).scalar() - return max_id if max_id is not None else 0 - def the_record_exists(self, dataset: DataSet, user_cookie: str): return self.model.query.filter_by( user_id=current_user.id if current_user.is_authenticated else None, diff --git a/app/modules/dataset/services.py b/app/modules/dataset/services.py index 2ef621e5..dd6606ba 100644 --- a/app/modules/dataset/services.py +++ b/app/modules/dataset/services.py @@ -27,6 +27,7 @@ HubfileRepository, HubfileViewRecordRepository ) +from app.modules.statistics.services import StatisticsService from core.services.BaseService import BaseService logger = logging.getLogger(__name__) @@ -103,12 +104,6 @@ def count_authors(self) -> int: def count_dsmetadata(self) -> int: return self.dsmetadata_repository.count() - def total_dataset_downloads(self) -> int: - return self.dsdownloadrecord_repository.total_dataset_downloads() - - def total_dataset_views(self) -> int: - return self.dsviewrecord_repostory.total_dataset_views() - def update_from_form(self, form: DataSetForm, current_user: User, dataset: DataSet) -> DataSet: main_author = { "name": f"{current_user.profile.surname}, {current_user.profile.name}", @@ -327,6 +322,7 @@ def __init__(self): class DSDownloadRecordService(BaseService): def __init__(self): super().__init__(DSDownloadRecordRepository()) + self.statistics_service = StatisticsService() def the_record_exists(self, dataset: DataSet, user_cookie: str): return self.repository.the_record_exists(dataset, user_cookie) @@ -344,6 +340,7 @@ def create_cookie(self, dataset: DataSet) -> str: if not existing_record: self.create_new_record(dataset=dataset, user_cookie=user_cookie) + self.statistics_service.increment_datasets_downloaded() return user_cookie @@ -362,6 +359,7 @@ def filter_by_doi(self, doi: str) -> Optional[DSMetaData]: class DSViewRecordService(BaseService): def __init__(self): super().__init__(DSViewRecordRepository()) + self.statistics_service = StatisticsService() def the_record_exists(self, dataset: DataSet, user_cookie: str): return self.repository.the_record_exists(dataset, user_cookie) @@ -379,6 +377,7 @@ def create_cookie(self, dataset: DataSet) -> str: if not existing_record: self.create_new_record(dataset=dataset, user_cookie=user_cookie) + self.statistics_service.increment_datasets_viewed() return user_cookie diff --git a/app/modules/hubfile/routes.py b/app/modules/hubfile/routes.py index 38e3897d..dacc48d8 100644 --- a/app/modules/hubfile/routes.py +++ b/app/modules/hubfile/routes.py @@ -8,6 +8,7 @@ from app.modules.hubfile.services import HubfileDownloadRecordService, HubfileService from app import db +from app.modules.statistics.services import StatisticsService hubfile_download_record_service = HubfileDownloadRecordService() @@ -139,6 +140,9 @@ def view_file(file_id): db.session.add(new_view_record) db.session.commit() + statistics_service = StatisticsService() + statistics_service.increment_feature_models_viewed() + # Prepare response response = jsonify({'success': True, 'content': content}) if not request.cookies.get('view_cookie'): diff --git a/app/modules/hubfile/services.py b/app/modules/hubfile/services.py index 478d8a5f..d833fadd 100644 --- a/app/modules/hubfile/services.py +++ b/app/modules/hubfile/services.py @@ -10,6 +10,7 @@ HubfileRepository, HubfileViewRecordRepository ) +from app.modules.statistics.services import StatisticsService from core.services.BaseService import BaseService @@ -40,13 +41,6 @@ def get_path_by_hubfile(self, hubfile: Hubfile) -> str: return path - def total_hubfile_views(self) -> int: - return self.hubfile_view_record_repository.total_hubfile_views() - - def total_hubfile_downloads(self) -> int: - hubfile_download_record_repository = HubfileDownloadRecordRepository() - return hubfile_download_record_repository.total_hubfile_downloads() - def get_by_ids(self, ids: list[int]) -> list[Hubfile]: return self.repository.get_by_ids(ids) @@ -54,6 +48,7 @@ def get_by_ids(self, ids: list[int]) -> list[Hubfile]: class HubfileDownloadRecordService(BaseService): def __init__(self): super().__init__(HubfileDownloadRecordRepository()) + self.statistics_service = StatisticsService() def the_record_exists(self, hubfile: Hubfile, user_cookie: str): return self.repository.the_record_exists(hubfile, user_cookie) @@ -71,5 +66,6 @@ def create_cookie(self, hubfile: Hubfile): if not existing_record: self.create_new_record(hubfile=hubfile, user_cookie=user_cookie) + self.statistics_service.increment_feature_models_downloaded() return user_cookie diff --git a/app/modules/public/routes.py b/app/modules/public/routes.py index 84c45755..1e8109cd 100644 --- a/app/modules/public/routes.py +++ b/app/modules/public/routes.py @@ -5,6 +5,7 @@ from app.modules.featuremodel.services import FeatureModelService from app.modules.public import public_bp from app.modules.dataset.services import DataSetService +from app.modules.statistics.services import StatisticsService logger = logging.getLogger(__name__) @@ -14,18 +15,19 @@ def index(): logger.info("Access index") dataset_service = DataSetService() feature_model_service = FeatureModelService() + statistics_service = StatisticsService() # Statistics: total datasets and feature models datasets_counter = dataset_service.count_synchronized_datasets() feature_models_counter = feature_model_service.count_feature_models() # Statistics: total downloads - total_dataset_downloads = dataset_service.total_dataset_downloads() - total_feature_model_downloads = feature_model_service.total_feature_model_downloads() + total_dataset_downloads = statistics_service.get_datasets_downloaded() + total_feature_model_downloads = statistics_service.get_feature_models_downloaded() # Statistics: total views - total_dataset_views = dataset_service.total_dataset_views() - total_feature_model_views = feature_model_service.total_feature_model_views() + total_dataset_views = statistics_service.get_datasets_viewed() + total_feature_model_views = statistics_service.get_feature_models_viewed() return render_template( "public/index.html", diff --git a/app/modules/statistics/__init__.py b/app/modules/statistics/__init__.py new file mode 100644 index 00000000..6eec2f8a --- /dev/null +++ b/app/modules/statistics/__init__.py @@ -0,0 +1,3 @@ +from core.blueprints.base_blueprint import BaseBlueprint + +statistics_bp = BaseBlueprint('statistics', __name__, template_folder='templates') diff --git a/app/modules/statistics/assets/scripts.js b/app/modules/statistics/assets/scripts.js new file mode 100644 index 00000000..7871c6c1 --- /dev/null +++ b/app/modules/statistics/assets/scripts.js @@ -0,0 +1 @@ +console.log("Hi, I am a script loaded from statistics module"); diff --git a/app/modules/statistics/forms.py b/app/modules/statistics/forms.py new file mode 100644 index 00000000..3c4cbb6f --- /dev/null +++ b/app/modules/statistics/forms.py @@ -0,0 +1,6 @@ +from flask_wtf import FlaskForm +from wtforms import SubmitField + + +class StatisticsForm(FlaskForm): + submit = SubmitField('Save statistics') diff --git a/app/modules/statistics/models.py b/app/modules/statistics/models.py new file mode 100644 index 00000000..54e6b5f9 --- /dev/null +++ b/app/modules/statistics/models.py @@ -0,0 +1,16 @@ +from app import db + + +class Statistics(db.Model): + id = db.Column(db.Integer, primary_key=True) + datasets_viewed = db.Column(db.Integer, default=0) + feature_models_viewed = db.Column(db.Integer, default=0) + datasets_downloaded = db.Column(db.Integer, default=0) + feature_models_downloaded = db.Column(db.Integer, default=0) + + def __repr__(self): + return ( + f'Statistics' + ) diff --git a/app/modules/statistics/repositories.py b/app/modules/statistics/repositories.py new file mode 100644 index 00000000..369b6cb2 --- /dev/null +++ b/app/modules/statistics/repositories.py @@ -0,0 +1,55 @@ +from app.modules.statistics.models import Statistics +from core.repositories.BaseRepository import BaseRepository + + +class StatisticsRepository(BaseRepository): + def __init__(self): + super().__init__(Statistics) + + def get_statistics(self) -> Statistics: + statistics = self.model.query.first() + if statistics is None: + # If no registry exists, create a new registry with default values + statistics = Statistics(datasets_viewed=0, feature_models_viewed=0, + datasets_downloaded=0, feature_models_downloaded=0) + self.session.add(statistics) + self.session.commit() + return statistics + + # Incremental methods + def increment_datasets_viewed(self) -> int: + return self._increment_field('datasets_viewed') + + def increment_feature_models_viewed(self) -> int: + return self._increment_field('feature_models_viewed') + + def increment_datasets_downloaded(self) -> int: + return self._increment_field('datasets_downloaded') + + def increment_feature_models_downloaded(self) -> int: + return self._increment_field('feature_models_downloaded') + + def _increment_field(self, field_name: str) -> int: + statistics = self.get_statistics() + current_value = getattr(statistics, field_name) + new_value = current_value + 1 + setattr(statistics, field_name, new_value) + self.session.commit() + return new_value + + # Consultation methods + def get_datasets_viewed(self) -> int: + statistics = self.get_statistics() + return statistics.datasets_viewed + + def get_feature_models_viewed(self) -> int: + statistics = self.get_statistics() + return statistics.feature_models_viewed + + def get_datasets_downloaded(self) -> int: + statistics = self.get_statistics() + return statistics.datasets_downloaded + + def get_feature_models_downloaded(self) -> int: + statistics = self.get_statistics() + return statistics.feature_models_downloaded diff --git a/app/modules/statistics/routes.py b/app/modules/statistics/routes.py new file mode 100644 index 00000000..5c4435f9 --- /dev/null +++ b/app/modules/statistics/routes.py @@ -0,0 +1,7 @@ +from flask import render_template +from app.modules.statistics import statistics_bp + + +@statistics_bp.route('/statistics', methods=['GET']) +def index(): + return render_template('statistics/index.html') diff --git a/app/modules/statistics/seeders.py b/app/modules/statistics/seeders.py new file mode 100644 index 00000000..de0b3212 --- /dev/null +++ b/app/modules/statistics/seeders.py @@ -0,0 +1,14 @@ +from app.modules.statistics.models import Statistics +from core.seeders.BaseSeeder import BaseSeeder + + +class StatisticsSeeder(BaseSeeder): + + def run(self): + + data = [ + Statistics(datasets_viewed=0, feature_models_viewed=0, + datasets_downloaded=0, feature_models_downloaded=0) + ] + + self.seed(data) diff --git a/app/modules/statistics/services.py b/app/modules/statistics/services.py new file mode 100644 index 00000000..74d04ea6 --- /dev/null +++ b/app/modules/statistics/services.py @@ -0,0 +1,37 @@ +from app.modules.statistics.models import Statistics +from app.modules.statistics.repositories import StatisticsRepository +from core.services.BaseService import BaseService + + +class StatisticsService(BaseService): + def __init__(self): + super().__init__(StatisticsRepository()) + + def get_statistics(self) -> Statistics: + return self.repository.get_statistics() + + # Incremental methods + def increment_datasets_viewed(self) -> int: + return self.repository.increment_datasets_viewed() + + def increment_feature_models_viewed(self) -> int: + return self.repository.increment_feature_models_viewed() + + def increment_datasets_downloaded(self) -> int: + return self.repository.increment_datasets_downloaded() + + def increment_feature_models_downloaded(self) -> int: + return self.repository.increment_feature_models_downloaded() + + # Consultation methods + def get_datasets_viewed(self) -> int: + return self.repository.get_datasets_viewed() + + def get_feature_models_viewed(self) -> int: + return self.repository.get_feature_models_viewed() + + def get_datasets_downloaded(self) -> int: + return self.repository.get_datasets_downloaded() + + def get_feature_models_downloaded(self) -> int: + return self.repository.get_feature_models_downloaded() diff --git a/app/modules/statistics/templates/statistics/index.html b/app/modules/statistics/templates/statistics/index.html new file mode 100644 index 00000000..ff83161f --- /dev/null +++ b/app/modules/statistics/templates/statistics/index.html @@ -0,0 +1,11 @@ +{% extends "base_template.html" %} + +{% block title %}View statistics{% endblock %} + +{% block content %} + +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/app/modules/statistics/tests/__init__.py b/app/modules/statistics/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/modules/statistics/tests/locustfile.py b/app/modules/statistics/tests/locustfile.py new file mode 100644 index 00000000..ad29e5a3 --- /dev/null +++ b/app/modules/statistics/tests/locustfile.py @@ -0,0 +1,21 @@ +from locust import HttpUser, TaskSet, task +from core.environment.host import get_host_for_locust_testing + + +class StatisticsBehavior(TaskSet): + def on_start(self): + self.index() + + @task + def index(self): + response = self.client.get("/statistics") + + if response.status_code != 200: + print(f"Statistics index failed: {response.status_code}") + + +class StatisticsUser(HttpUser): + tasks = [StatisticsBehavior] + min_wait = 5000 + max_wait = 9000 + host = get_host_for_locust_testing() diff --git a/app/modules/statistics/tests/test_selenium.py b/app/modules/statistics/tests/test_selenium.py new file mode 100644 index 00000000..7fbb062c --- /dev/null +++ b/app/modules/statistics/tests/test_selenium.py @@ -0,0 +1,35 @@ +from selenium.common.exceptions import NoSuchElementException +import time + +from core.environment.host import get_host_for_selenium_testing +from core.selenium.common import initialize_driver, close_driver + + +def test_statistics_index(): + + driver = initialize_driver() + + try: + host = get_host_for_selenium_testing() + + # Open the index page + driver.get(f'{host}/statistics') + + # Wait a little while to make sure the page has loaded completely + time.sleep(4) + + try: + + pass + + except NoSuchElementException: + raise AssertionError('Test failed!') + + finally: + + # Close the browser + close_driver(driver) + + +# Call the test function +test_statistics_index() diff --git a/app/modules/statistics/tests/test_unit.py b/app/modules/statistics/tests/test_unit.py new file mode 100644 index 00000000..950421eb --- /dev/null +++ b/app/modules/statistics/tests/test_unit.py @@ -0,0 +1,24 @@ +import pytest + + +@pytest.fixture(scope='module') +def test_client(test_client): + """ + Extends the test_client fixture to add additional specific data for module testing. + """ + with test_client.application.app_context(): + # Add HERE new elements to the database that you want to exist in the test context. + # DO NOT FORGET to use db.session.add() and db.session.commit() to save the data. + pass + + yield test_client + + +def test_sample_assertion(test_client): + """ + Sample test to verify that the test framework and environment are working correctly. + It does not communicate with the Flask application; it only performs a simple assertion to + confirm that the tests in this module can be executed. + """ + greeting = "Hello, World!" + assert greeting == "Hello, World!", "The greeting does not coincide with 'Hello, World!'" diff --git a/migrations/versions/011.py b/migrations/versions/011.py new file mode 100644 index 00000000..9cbc7313 --- /dev/null +++ b/migrations/versions/011.py @@ -0,0 +1,35 @@ +"""add statistics module + +Revision ID: 011 +Revises: 010 +Create Date: 2024-10-15 11:01:19.816245 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '011' +down_revision = '010' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('statistics', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('datasets_viewed', sa.Integer(), nullable=True), + sa.Column('feature_models_viewed', sa.Integer(), nullable=True), + sa.Column('datasets_downloaded', sa.Integer(), nullable=True), + sa.Column('feature_models_downloaded', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('statistics') + # ### end Alembic commands ###