Cleanup time-wise analysis

maciejzj · maciejzj · commit 80813b185a5b · 2025-01-02T14:15:49.000+01:00
diff --git a/README.md b/README.md
@@ -189,7 +189,7 @@ Tools configuration is stored in the `pyproject.toml` file.
 The application is not bound to any specific deployment environment; however,
 AWS is used for running the main instance. The setup for creating AWS
 infrastructure for the application using Terraform and Ansible deployment is
-placed int the `deployment` directory. 
+placed int the `deployment` directory.
 
 ## License
 
diff --git a/it_jobs_meta/__main__.py b/it_jobs_meta/__main__.py
@@ -54,7 +54,6 @@ def main():
                 data_pipeline.run()
 
         case 'dashboard':
-
             data_provider = MongodbDashboardDataProvider.from_config_file(parser.args['mongodb'])
             layout_parameters = LayoutTemplateParameters(navbar_label=parser.args['label'])
             app = DashboardApp(data_provider, layout_parameters)
diff --git a/it_jobs_meta/common/cli.py b/it_jobs_meta/common/cli.py
@@ -5,7 +5,6 @@
 from pathlib import Path
 from typing import Any
 
-from it_jobs_meta.dashboard.data_provision import DashboardDataProvider
 from it_jobs_meta.data_pipeline.data_etl import EtlLoaderImpl
 from it_jobs_meta.data_pipeline.data_lake import DataLakeImpl
 
diff --git a/it_jobs_meta/dashboard/dashboard.py b/it_jobs_meta/dashboard/dashboard.py
@@ -8,7 +8,7 @@
 import dash_bootstrap_components as dbc
 import pandas as pd
 from dash import Input, Output, callback, dcc, html
-from dash.development import base_component as DashComponent
+from dash.development.base_component import Component as DashComponent
 from flask_caching import Cache as AppCache
 from waitress import serve as wsgi_serve
 
@@ -18,10 +18,8 @@
 from it_jobs_meta.dashboard.layout import (
     LayoutDynamicContent,
     LayoutTemplateParameters,
-    make_categories_and_seniorities_graphs_layout,
+    make_graphs_layout,
     make_layout,
-    make_locations_and_remote_graphs_layout,
-    make_salaries_breakdown_graphs_layout,
 )
 
 
@@ -62,18 +60,15 @@ def cache(self) -> AppCache:
         if self._cache is None:
             self._cache = AppCache(
                 self.app.server,
-                config={
-                    'CACHE_TYPE': 'SimpleCache',
-                    'CACHE_THRESHOLD': 2,
-                },
+                config={'CACHE_TYPE': 'SimpleCache', 'CACHE_THRESHOLD': 2},
             )
         return self._cache
 
-    def render_layout(self) -> DashComponent:
+    def make_layout(self) -> DashComponent:
         logging.info('Rendering dashboard')
         logging.info('Attempting to retrieve data')
-        metadata_df =self._data_provider.fetch_metadata()
-        data_df = self._data_provider.fetch_data()
+        metadata_df = self._data_provider.fetch_metadata()
+        data_df = self._data_provider.fetch_data(metadata_df.iloc[-1]['batch_id'])
         logging.info('Data retrieval succeeded')
 
         logging.info('Making layout')
@@ -87,24 +82,19 @@ def register_callbacks(self) -> DashComponent:
         @callback(
             Output('graphs', 'children'),
             Input('batch-slider', 'value'),
-            prevent_initial_callback=True,
+            prevent_initial_call=True,
         )
         def update_graphs_section(value):
             metadata_df = self._data_provider.fetch_metadata()
             data_df = self._data_provider.fetch_data(metadata_df.iloc[value]['batch_id'])
-            dynamic_content = self.make_dynamic_content(metadata_df, data_df)
-            graphs = dynamic_content.graphs
-            return [
-                make_categories_and_seniorities_graphs_layout(graphs),
-                make_locations_and_remote_graphs_layout(graphs),
-                make_salaries_breakdown_graphs_layout(graphs),
-            ]
+            graphs = GraphRegistry.make(data_df)
+            return make_graphs_layout(graphs)
 
     def run(self, with_wsgi=False):
         try:
             render_layout_memoized = self.cache.memoize(
                 timeout=int(self._cache_timeout.total_seconds())
-            )(self.render_layout)
+            )(self.make_layout)
             self.app.layout = render_layout_memoized
             self.register_callbacks()
 
@@ -131,7 +121,7 @@ def main():
     setup_logging()
     layout_params = LayoutTemplateParameters()
     data_provider = MongodbDashboardDataProvider.from_config_file(Path('config/mongodb_config.yml'))
-    app = DashboardApp(layout_params, data_provider, cache_timeout=timedelta(seconds=30))
+    app = DashboardApp(data_provider, layout_params, cache_timeout=timedelta(seconds=5))
     app.run()
 
 
diff --git a/it_jobs_meta/dashboard/dashboard_components.py b/it_jobs_meta/dashboard/dashboard_components.py
@@ -6,7 +6,7 @@
 
 import numpy as np
 import pandas as pd
-from dash import Input, Output, callback, dcc
+from dash import dcc
 from plotly import express as px
 from plotly import graph_objects as go
 from sklearn import preprocessing
@@ -111,8 +111,10 @@ def make_fig(cls, postings_df: pd.DataFrame) -> go.Figure:
         tech_most_freq_df = get_rows_with_n_most_frequent_vals_in_col(
             postings_df, 'technology', cls.N_MOST_FREQ
         )
+        technology_counts = tech_most_freq_df['technology'].value_counts().reset_index()
+        technology_counts.columns = ['technology', 'count']
 
-        fig = px.pie(tech_most_freq_df, names='technology', title=cls.TITLE)
+        fig = px.pie(technology_counts, names='technology', values='count', title=cls.TITLE)
         fig.update_traces(textposition='inside')
         fig = center_title(fig)
         return fig
@@ -125,11 +127,15 @@ class CategoriesPieChart(GraphFigure):
 
     @classmethod
     def make_fig(cls, postings_df: pd.DataFrame) -> go.Figure:
+        # Get the most frequent categories and their counts
         cat_largest_df = get_rows_with_n_most_frequent_vals_in_col(
             postings_df, 'category', cls.N_MOST_FREQ
         )
+        category_counts = cat_largest_df['category'].value_counts().reset_index()
+        category_counts.columns = ['category', 'count']
 
-        fig = px.pie(cat_largest_df, names='category', title=cls.TITLE)
+        # Create a pie chart with count values
+        fig = px.pie(category_counts, names='category', values='count', title=cls.TITLE)
         fig.update_traces(textposition='inside')
         fig = center_title(fig)
         return fig
@@ -193,7 +199,9 @@ class SeniorityPieChart(GraphFigure):
     @classmethod
     def make_fig(cls, postings_df: pd.DataFrame) -> go.Figure:
         postings_df = postings_df.explode('seniority')
-        fig = px.pie(postings_df, names='seniority', title=cls.TITLE)
+        seniority_counts = postings_df['seniority'].value_counts().reset_index()
+        seniority_counts.columns = ['seniority', 'count']
+        fig = px.pie(seniority_counts, values='count', names='seniority', title=cls.TITLE)
         fig = center_title(fig)
         return fig
 
@@ -252,6 +260,7 @@ def make_fig(cls, postings_df) -> go.Figure:
         )
         job_counts = postings_df.groupby('city')['_id'].count()
         salaries = postings_df.groupby('city')[['salary_mean', 'lat', 'lon']].mean()
+        salaries['salary_mean'] = salaries['salary_mean'].round()
         cities_salaries = pd.concat([job_counts.rename('job_counts'), salaries], axis=1)
         cities_salaries = cities_salaries.reset_index()
 
diff --git a/it_jobs_meta/dashboard/data_provision.py b/it_jobs_meta/dashboard/data_provision.py
@@ -1,12 +1,12 @@
 """Data provision and data source for the data dashboard."""
 
 from abc import ABC, abstractmethod
-from enum import Enum, auto
 from pathlib import Path
 from typing import Self
 
 import pandas as pd
 from pymongo import MongoClient
+from pymongo.synchronous.database import Database
 
 from it_jobs_meta.common.utils import load_yaml_as_dict
 
@@ -34,19 +34,29 @@ def from_config_file(cls, config_file_path: Path) -> Self:
         return cls(**load_yaml_as_dict(config_file_path))
 
     def fetch_metadata(self) -> pd.DataFrame:
+        client: MongoClient
         with MongoClient(
             self.host, self.port, username=self.user_name, password=self.password
         ) as client:
-            db = client[self.db_name]
-            return pd.json_normalize(db['metadata'].find())
+            db: Database = client[self.db_name]
+            df = pd.json_normalize(db['metadata'].find().sort('obtained_datetime'))
+            if len(df) == 0:
+                raise ValueError('Found no metadata, dashboard cannot be made')
+            return df
 
     def fetch_data(self, batch_id: str | None = None) -> pd.DataFrame:
+        client: MongoClient
         with MongoClient(
             self.host, self.port, username=self.user_name, password=self.password
         ) as client:
-            db = client[self.db_name]
+            db: Database = client[self.db_name]
             collection = db['postings']
+
             if batch_id is not None:
-                return pd.json_normalize(collection.find({'batch_id': batch_id}))
+                df = pd.json_normalize(collection.find({'batch_id': batch_id}))
             else:
-                return pd.json_normalize(collection.find())
+                df = pd.json_normalize(collection.find())
+
+            if len(df) == 0:
+                raise ValueError('Found no data, dashboard cannot be made')
+            return df
diff --git a/it_jobs_meta/dashboard/layout.py b/it_jobs_meta/dashboard/layout.py
@@ -4,9 +4,8 @@
 from datetime import datetime
 
 import dash_bootstrap_components as dbc
-from dash import Input, Output, callback, dcc, html
-from it_jobs_meta.dashboard.data_provision import DashboardDataProvider
-from dash.development import base_component as DashComponent
+from dash import dcc, html
+from dash.development.base_component import Component as DashComponent
 from plotly import graph_objects as go
 
 from it_jobs_meta.dashboard.dashboard_components import Graph
@@ -105,10 +104,7 @@ def make_about() -> DashComponent:
     return about
 
 
-def make_graphs_layout_header(obtained_datetime: datetime) -> DashComponent:
-    datetime_str = [t.strftime('%-d %B %Y') for t in obtained_datetime]
-    slider_marks = dict(enumerate(datetime_str))
-
+def make_graphs_layout_header() -> DashComponent:
     div = html.Div(
         [
             html.H2('Data', id='data-container'),
@@ -117,13 +113,23 @@ def make_graphs_layout_header(obtained_datetime: datetime) -> DashComponent:
                 is most convenient to explore more complex graphs on larger
                 screens.'''
             ),
-            dcc.Slider(id="batch-slider", step=None, marks=slider_marks),
         ],
         className='text-center mt-5',
     )
     return div
 
 
+def make_timeline_slider(obtained_datetime) -> DashComponent:
+    slider_marks = dict(enumerate([t.strftime('%b %Y') for t in obtained_datetime]))
+    return dcc.Slider(
+        id='batch-slider',
+        className='text-nowrap mx-3 my-5',
+        step=None,
+        marks=slider_marks,
+        value=len(slider_marks) - 1,
+    )
+
+
 def make_categories_and_seniorities_graphs_layout(graphs: dict[Graph, dcc.Graph]) -> DashComponent:
     div = html.Div(
         [
@@ -234,8 +240,7 @@ def make_salaries_breakdown_graphs_layout(graphs: dict[Graph, dcc.Graph]) -> Das
         [
             html.H3('Salaries breakdown', className='mt-4'),
             dbc.Card(
-                graphs[Graph.TECHNOLOGIES_VIOLIN_PLOT],
-                className='mt-4 p-1 border-0 rounded shadow'
+                graphs[Graph.TECHNOLOGIES_VIOLIN_PLOT], className='mt-4 p-1 border-0 rounded shadow'
             ),
             dbc.Card(
                 graphs[Graph.CONTRACT_TYPE_VIOLIN_PLOT],
@@ -246,21 +251,22 @@ def make_salaries_breakdown_graphs_layout(graphs: dict[Graph, dcc.Graph]) -> Das
     return div
 
 
+def make_graphs_layout(graphs: dict[Graph, go.Figure]) -> list[DashComponent]:
+    return [
+        make_categories_and_seniorities_graphs_layout(graphs),
+        make_locations_and_remote_graphs_layout(graphs),
+        make_salaries_breakdown_graphs_layout(graphs),
+    ]
 
-def make_graphs_layout(
+
+def make_data_section_layout(
     obtained_datetime: datetime, graphs: dict[Graph, go.Figure]
 ) -> DashComponent:
     data_section = html.Section(
         [
-            make_graphs_layout_header(obtained_datetime),
-            html.Section(
-                [
-                    make_categories_and_seniorities_graphs_layout(graphs),
-                    make_locations_and_remote_graphs_layout(graphs),
-                    make_salaries_breakdown_graphs_layout(graphs),
-                ],
-                id='graphs',
-            ),
+            make_graphs_layout_header(),
+            make_timeline_slider(obtained_datetime),
+            html.Div(make_graphs_layout(graphs), id='graphs'),
         ],
         id='data-section',
     )
@@ -295,7 +301,7 @@ def make_layout(
                 [
                     make_jumbotron(),
                     make_about(),
-                    make_graphs_layout(
+                    make_data_section_layout(
                         dynamic_content.obtained_datetime,
                         dynamic_content.graphs,
                     ),
diff --git a/it_jobs_meta/data_pipeline/data_etl.py b/it_jobs_meta/data_pipeline/data_etl.py
diff --git a/it_jobs_meta/data_pipeline/data_validation.py b/it_jobs_meta/data_pipeline/data_validation.py
diff --git a/it_jobs_meta/data_pipeline/geolocator.py b/it_jobs_meta/data_pipeline/geolocator.py
