Merge branch 'main' into artmed

# Conflicts:
#	project/source/string_processing/utilities.py

Makefile

@@ -10,8 +10,12 @@ activate:
 	@echo "$$(tput bold)Activating virtual environment...$$(tput sgr0)"
 	poetry shell
 
+## Migration
+migration:
+	poetry run python project/source/manage.py migrate
+
 ## Setup project
-setup: install activate
+setup: install activate migration
 
 test:
 	@echo "$$(tput bold)Running tests...$$(tput sgr0)"
@@ -73,3 +77,6 @@ help:
 
 runui:
 	poetry run python .\project\source\manage.py runserver
+
+run_linux:
+	poetry run python project/source/manage.py runserver

project/source/ml/entity.py

@@ -0,0 +1,32 @@
+from transformers import pipeline
+
+pipe = pipeline(
+    "token-classification",
+    model="viktoroo/sberbank-rubert-base-collection3",
+    aggregation_strategy="simple",
+)
+
+
+def predict_entity(text: str):
+    word_descriptions = pipe.predict(text)
+    res = {
+        "location": [],
+        "person": [],
+        "organization": [],
+    }
+    for word_description in word_descriptions:
+        group = word_description["entity_group"]
+        # word_start = word_description["start"]
+        # word_end = word_description["end"]
+        word = word_description["word"]
+
+        if group == "LOC":
+            res["location"].append(word)
+            pass
+        if group == "PER":
+            res["person"].append(word)
+            pass
+        if group == "ORG":
+            res["organization"].append(word)
+            pass
+    return res

project/source/ml/sentiment.py

@@ -0,0 +1,12 @@
+from transformers import pipeline
+
+sentiment_analysis = pipeline(
+    "text-classification", model="cointegrated/rubert-tiny-sentiment-balanced"
+)
+
+
+def predict_sentiment(text: str):
+    output = sentiment_analysis.predict(text)
+    label = output[0]["label"]
+
+    return label

project/source/static/css/style.css

@@ -3,6 +3,45 @@ body {
   margin-right: 2%;
 }
 
+u {
+  text-decoration: none!important;
+}
+a {
+  text-decoration: none!important;
+}
+
+.tooltip {
+  display: inline-block;
+  border-bottom: 1px dotted black;
+}
+
+/* Tooltip text */
+.tooltip-text {
+  visibility: hidden;
+  position: absolute;
+  z-index: 1;
+  width: auto;
+  color: #000000;
+  font-size: 16px;
+  font-weight: 490;
+  border-radius: 5px;
+  padding: 5px 5px 5px 5px;
+  margin-top: -32px;
+  margin-left: -4px;
+}
+
+.highlighted_text:hover .tooltip-text {
+  visibility: visible;
+}
+
+.table{
+  width: 80%;
+  margin-left: 10%;
+}
+
+table tr:hover td {
+    background-color: rgb(61,65,69); /* Change to the desired background color */
+}
 .form-control {
   width: 80%;
   height: 5em;
@@ -30,3 +69,87 @@ body {
 .form-subtitle {
   font-size: 20px;
 }
+
+.response_container {
+  /*display: flex;*/
+  justify-content: center;
+  font-size: 18px;
+  width: 80%;
+  align-items: center;
+  margin-left: 10%;
+}
+
+.highlighted_text {
+  border-radius: 5px;
+  width: auto;
+  height: fit-content;
+  /*padding: 1px 4px 2px 4px;*/
+  color: #000000;
+  margin-left: 1px;
+  margin-right: 1px;
+  white-space: nowrap;
+}
+
+.phone_number{
+  background-color: rgba(155, 200, 251, 1);
+}
+
+.link{
+  background-color: rgba(237,170,205, 1);
+}
+
+.email{
+  background-color: rgba(213,133,252, 0.8);
+}
+
+.date_time{
+  background-color: rgba(233,209,135, 1);
+
+}
+
+
+.organization{
+  background-color: rgba(255,185,117, 1);
+}
+
+
+.person{
+  background-color: rgba(105, 150, 231, 1);
+}
+
+
+.location{
+  background-color: rgba(245,241,255, 1);
+}
+
+.negative {
+  /*margin-left: 48%;*/
+  text-decoration: darkred underline;
+}
+
+.positive {
+  /*margin-left: 48%;*/
+  text-decoration: lightgreen underline;
+}
+
+.neutral {
+  /*margin-left: 48%;*/
+  text-decoration: cornflowerblue underline;
+}
+
+.button-container {
+  display: flex;
+  justify-content: space-between;
+  width: 80%;
+  align-items: center;
+  margin: 0 auto;
+}
+
+.form-select{
+  width: 50%;
+}
+
+.select-container {
+  flex: 1; /* Fill remaining space */
+  max-width: 200px; /* Set maximum width */
+}

project/source/string_processing/utilities.py

@@ -1,46 +1,51 @@
 import re
+import time
 
+from project.source.ml.entity import predict_entity
 from project.source.string_processing.validator_string import (
     preprocess_str,
     string_validator,
 )
 
 
-def find_start_end(string: str, pattern: str) -> list:
-    """
-    return first and second posinion of pattern in string
-    """
+def find_start_end(string, pattern):
+    print(f"{string=} {pattern=}")
     matches = re.finditer(pattern, string)
     start_end_list = []
     for match in matches:
         start_end_list.append((match.start(), match.end()))
     return start_end_list
 
 
-def highlight_words(text: str) -> str:
-    """
+def highlight_words(text):
+    to_highlight, new_text = string_validator(text)
+    to_highlight_additional = predict_entity(new_text)
+    print(to_highlight)
 
-    :param text:
-    :type text:
-    :return:
-    :rtype:
-    """
-    to_highlight, new_text = string_validator(preprocess_str(text))
     end_tag = "</u></a>"
-    for key, value in to_highlight.items():
+
+    for key, value in dict(to_highlight, **to_highlight_additional).items():
         if key == "Номер":
-            tag = '<a style="background:green"><u>'
+            tag = '<a class="highlighted_text phone_number"><u><span class="tooltip-text phone_number" id="top">Номер</span> '
         elif key == "Ссылки":
-            tag = '<a style="background:yellow"><u>'
+            tag = '<a class="highlighted_text link"><u><span class="tooltip-text link" id="top">Ссылка</span>'
         elif key == "Почта":
-            tag = '<a style="background:red"><u>'
-        else:
-            tag = '<a style="background:white"><u>'
+            tag = '<a class="highlighted_text email"><u><span class="tooltip-text email" id="top">Почта</span>'
+        elif key == "Дата":
+            tag = '<a class="highlighted_text date_time"><u><span class="tooltip-text date_time" id="top">Дата</span>'
+        elif key == "location":
+            tag = '<a class="highlighted_text location"><u><span class="tooltip-text location" id="top">Местоположение</span>'
+        elif key == "person":
+            tag = '<a class="highlighted_text person"><u><span class="tooltip-text person" id="top">ФИО</span>'
+        elif key == "organization":
+            tag = '<a class="highlighted_text organization"><u><span class="tooltip-text organization" id="top">Организация</span>'
         for word in value:
-
-            positions = find_start_end(new_text, word)
+            print(f"{new_text.lower()=} {word.lower()=} ")
+            positions = find_start_end(new_text.lower(), word.lower())
             for i in range(len(positions)):
-                pos = find_start_end(new_text, word)
+
+                time.sleep(1)
+                pos = find_start_end(new_text.lower(), word.lower())
                 new_text = (
                     new_text[: pos[i][0]]
                     + tag

project/source/string_processing/validator_string.py

@@ -88,7 +88,7 @@ def preprocess_str(raw_str: str, demojize: bool = True) -> str:
     return raw_text
 
 
-def string_validator(raw_text: str) -> dict | int:
+def string_validator(raw_text: str):
     """
     Validation string for output on the screen
 
@@ -106,17 +106,13 @@ def string_validator(raw_text: str) -> dict | int:
     }
 
     if not isinstance(raw_text, str):
-        for key in valid_data.keys():
-            valid_data[key] = 0
-        return valid_data
+        return valid_data, raw_text
 
     if len(raw_text) <= 5:
-        for key in valid_data.keys():
-            valid_data[key] = 0
-        return valid_data  # некорректная строка
-    print(f"before {raw_text}")
+        return valid_data, raw_text
+
     raw_text = replace_day(raw_text)
-    print(f"after {raw_text}")
+
     numbers = pattern_number.findall(raw_text)
     for num in numbers:
         num1 = pattern_digit.findall(num)

project/source/templates/dashboard.html

@@ -1,12 +1,15 @@
 {% extends "base.html" %}
 {% block content %}
-<form id="myForm" method="post" >
+<div class="form-container">
+  <form id="myForm" method="post" style="display: flex; align-items: center;">
+    <a class="btn btn-outline-primary" style="width: 100px; margin-right: 10px;" href="/">Назад</a>
     {%csrf_token%}
-  <select name="selected_option" value={{selected}} onchange="this.form.submit()">
-    <option value="Исполнитель" {% if selected == "Исполнитель" %}selected{% endif %}>Исполнитель</option>
-    <option value="Группа тем" {% if selected == "Группа тем" %}selected{% endif %}>Группа тем</option>
-  </select>
-</form>
+    <select class="form-select" name="selected_option" value={{selected}} onchange="this.form.submit()" style="width: 200px;">
+      <option value="Исполнитель" {% if selected == "Исполнитель" %}selected{% endif %}>Исполнитель</option>
+      <option value="Группа тем" {% if selected == "Группа тем" %}selected{% endif %}>Группа тем</option>
+    </select>
+  </form>
+</div>
 <div id="plot_div">
 
     {% autoescape off %}