Merge pull request #49 from sayantikabanik/fix_summarization

LSA applied to pros and cons

analysis_framework/dashboard/base.py

@@ -3,6 +3,8 @@
 import holoviews as hv
 import hvplot.pandas
 from analysis_framework.utils.text_summarization import summarize_text as stext
+#from analysis_framework.utils.text_summarization import summarize_text_lexrank as stext_lexrank
+from analysis_framework.utils.text_summarization import summarize_text_lsa as stext_lsa
 from analysis_framework.utils import burnout_index_calculation
 from analysis_framework.utils import static as rs
 hv.extension('bokeh')
@@ -119,7 +121,7 @@ def pros_widget(location_widget_b, cluster_widget_b, comp_widget_b):
     pros = pn.Column(
         '# Pros',
         pn.layout.Divider(),
-        stext(display_output),
+        stext_lsa(display_output),
         background='#e4ede6', height=500, width=500
     )
     return pros
@@ -137,7 +139,7 @@ def cons_widget(location_widget_b, cluster_widget_b, comp_widget_b):
     cons = pn.Column(
         '# Cons',
         pn.layout.Divider(),
-        stext(display_output),
+        stext_lsa(display_output),
         background='#fac8d3', height=500, width=500
     )
     return cons

analysis_framework/utils/static.py

@@ -59,7 +59,7 @@
 
 # Text summarization
 # ---------------------------------
-sentence_count=2
+sentence_count=3
 bonus_words_list = ['work life balance', 'job security', 'work environment',
                     'career growth', 'salary', 'culture', 'career' , 'pay',
                     'improvement', 'manager', 'employees', 'company',

analysis_framework/utils/text_summarization.py

@@ -3,6 +3,8 @@
 from sumy.nlp.stemmers import Stemmer
 from sumy.utils import get_stop_words 
 from sumy.summarizers.edmundson import EdmundsonSummarizer
+from sumy.summarizers.lex_rank import LexRankSummarizer
+from sumy.summarizers.lsa import LsaSummarizer
 from analysis_framework.utils import static as rs
 
 # Uncomment below imports if not pre-installed
@@ -22,5 +24,16 @@ def summarize_text(text, language='english'):
     for sentence in summary:
         count += 1
         sentence_list.append(str(f'{count}. {sentence}'))
-    output = "\n".join(sentence_list)
+    output = "\n\n".join(sentence_list)
+    return output
+
+def summarize_text_lsa(text, language='english'):
+    summarizer = LsaSummarizer(Stemmer(language))
+    sentence_list=[]
+    count=0
+    summary = summarizer(PlaintextParser(text, Tokenizer(language)).document, rs.sentence_count)
+    for sentence in summary:
+        count += 1
+        sentence_list.append(str(f'{count}. {sentence}'))
+    output = "\n\n".join(sentence_list)
     return output