Merge branch 'add-reponse-times-bugzilla' of 'https://github.com/sduenas/grimoirelab-elk/'

Merges #1171 
Closes #1171

Author: jjmerchante

grimoire_elk/enriched/bugzilla.py

@@ -195,15 +195,29 @@ def get_rich_item(self, item):
 
         # Add extra JSON fields used in Kibana (enriched fields)
         eitem['comments'] = 0
+        eitem['last_comment_date'] = None
         eitem['url'] = None
 
         if 'long_desc' in issue:
             eitem['comments'] = len(issue['long_desc'])
+
+            last_comment_date = None
+
+            if eitem['comments'] > 1:
+                last_comment = issue['long_desc'][-1]
+                last_comment_date = str_to_datetime(last_comment['bug_when'][0]['__text__'])
+                last_comment_date = last_comment_date.isoformat()
+
+            eitem['last_comment_date'] = last_comment_date
+
         eitem['url'] = item['origin'] + "/show_bug.cgi?id=" + issue['bug_id'][0]['__text__']
         eitem['resolution_days'] = \
             get_time_diff_days(eitem['creation_date'], eitem['delta_ts'])
         eitem['timeopen_days'] = \
             get_time_diff_days(eitem['creation_date'], datetime_utcnow().replace(tzinfo=None))
+        eitem['time_to_first_attention'] = \
+            get_time_diff_days(eitem['creation_date'],
+                               self.get_time_to_first_attention(issue))
 
         if self.sortinghat:
             eitem.update(self.get_item_sh(item, self.roles))
@@ -216,3 +230,29 @@ def get_rich_item(self, item):
         self.add_repository_labels(eitem)
         self.add_metadata_filter_raw(eitem)
         return eitem
+
+    def get_time_to_first_attention(self, item):
+        """Set the time to first attention.
+
+        This date is defined as the first date at which a comment by someone
+        other than the user who created the issue.
+        """
+        if 'long_desc' not in item:
+            return None
+
+        comment_dates = []
+        reporter = item['reporter'][0]['__text__']
+
+        # First comment is the description of the issue
+        # Real comments start at the second position (index 1)
+        for comment in item['long_desc'][1:]:
+            user = comment['who'][0]['__text__']
+            if user == reporter:
+                continue
+            dt = str_to_datetime(comment['bug_when'][0]['__text__']).replace(tzinfo=None)
+            comment_dates.append(dt)
+
+        if comment_dates:
+            return min(comment_dates)
+        else:
+            return None

grimoire_elk/enriched/bugzillarest.py

@@ -136,17 +136,29 @@ def get_rich_item(self, item):
 
         # Add extra JSON fields used in Kibana (enriched fields)
         eitem['comments'] = 0
+        eitem['last_comment_date'] = None
         eitem['number_of_comments'] = 0
         eitem['time_to_last_update_days'] = None
+        eitem['time_to_first_attention'] = None
         eitem['url'] = None
 
         # Add the field to know if the ticket is open
         eitem['is_open'] = issue.get('is_open', None)
 
         if 'long_desc' in issue:
             eitem['number_of_comments'] = len(issue['long_desc'])
+
         if 'comments' in issue:
             eitem['comments'] = len(issue['comments'])
+
+            last_comment_date = None
+
+            if eitem['comments'] > 1:
+                last_comment_date = str_to_datetime(issue['comments'][-1]['time'])
+                last_comment_date = last_comment_date.isoformat()
+
+            eitem['last_comment_date'] = last_comment_date
+
         eitem['url'] = item['origin'] + "/show_bug.cgi?id=" + str(issue['id'])
         eitem['time_to_last_update_days'] = \
             get_time_diff_days(eitem['creation_ts'], eitem['delta_ts'])
@@ -155,6 +167,9 @@ def get_rich_item(self, item):
         if 'is_open' in issue and not issue['is_open']:
             eitem['timeopen_days'] = eitem['time_to_last_update_days']
 
+        eitem['time_to_first_attention'] = \
+            get_time_diff_days(eitem['creation_ts'], self.get_time_to_first_attention(issue))
+
         eitem['changes'] = 0
         for history in issue['history']:
             if 'changes' in history:
@@ -177,3 +192,28 @@ def get_rich_item(self, item):
         self.add_repository_labels(eitem)
         self.add_metadata_filter_raw(eitem)
         return eitem
+
+    def get_time_to_first_attention(self, item):
+        """Set the time to first attention.
+
+        This date is defined as the first date at which a comment by someone
+        other than the user who created the issue.
+        """
+        if 'comments' not in item:
+            return None
+
+        comment_dates = []
+        creator = item['creator']
+
+        # First comment is the description of the issue
+        # Real comments start at the second position (index 1)
+        for comment in item['comments'][1:]:
+            user = comment['creator']
+            if user == creator:
+                continue
+            comment_dates.append(str_to_datetime(comment['time']).replace(tzinfo=None))
+
+        if comment_dates:
+            return min(comment_dates)
+        else:
+            return None

releases/unreleased/new-reponse-times-on-bugzilla-items.yml

@@ -0,0 +1,15 @@
+---
+title: New reponse times on bugzilla items
+category: added
+author: Santiago Dueñas <sduenas@bitergia.com>
+issue: null
+notes: >
+  The Bugzilla enriched items include two new
+  fields to track response times on these type
+  of repositories.
+  The field `time_to_first_attention` is the
+  the time expressed in days between the ticket
+  creation and the first comment from a contributor
+  different from the author creating the bug.
+  The field `last_comment_date` is the date of
+  the last comment posted in the bug.

schema/bugzilla.csv

@@ -58,6 +58,7 @@ is_bugzillarest_bugrest,long
 is_open,boolean
 keywords,keyword
 labels,list
+last_comment_date,date
 main_description,keyword
 main_description_analyzed,string,false
 metadata__enriched_on,date
@@ -110,6 +111,7 @@ severity,keyword
 status,keyword
 tag,keyword
 timeopen_days,float
+time_to_first_attention,float
 url,keyword
 uuid,keyword
 whiteboard,keyword

tests/data/bugzilla.json

@@ -181,8 +181,8 @@
         ],
         "reporter": [
             {
-                "__text__": "rocapal@example.org",
-                "name": "Roberto Calvo"
+                "__text__": "sduenas@example.org",
+                "name": "Santiago Due\u00f1as"
             }
         ],
         "reporter_accessible": [
@@ -507,8 +507,8 @@
         ],
         "reporter": [
             {
-                "__text__": "rocapal@example.org",
-                "name": "Roberto Calvo"
+                "__text__": "sduenas@example.org",
+                "name": "Santiago Due\u00f1as"
             }
         ],
         "reporter_accessible": [
@@ -668,8 +668,8 @@
                 ],
                 "who": [
                     {
-                        "__text__": "sduenas@example.org",
-                        "name": "Santiago Due\u00f1as"
+                        "__text__": "dizquierdo@example.org",
+                        "name": "Daniel Izquierdo Cortazar"
                     }
                 ]
             }
@@ -1026,8 +1026,8 @@
         ],
         "reporter": [
             {
-                "__text__": "rocapal@example.com",
-                "name": "Roberto Calvo"
+                "__text__": "sduenas@example.org",
+                "name": "Santiago Due\u00f1as"
             }
         ],
         "reporter_accessible": [
@@ -1251,8 +1251,8 @@
         ],
         "reporter": [
             {
-                "__text__": "rocapal@example.org",
-                "name": "Roberto Calvo"
+                "__text__": "sduenas@example.org",
+                "name": "Santiago Due\u00f1as"
             }
         ],
         "reporter_accessible": [
@@ -1446,8 +1446,8 @@
         ],
         "reporter": [
             {
-                "__text__": "dizquierdo@example.org",
-                "name": "Daniel Izquierdo Cortazar"
+                "__text__": "sduenas@example.org",
+                "name": "Santiago Due\u00f1as"
             }
         ],
         "reporter_accessible": [
@@ -1661,7 +1661,7 @@
         ],
         "creation_ts": [
             {
-                "__text__": "2015-05-23 06:06:06 +0200"
+                "__text__": "2013-06-25 11:49:36 +0200"
             }
         ],
         "delta_ts": [
@@ -1720,7 +1720,7 @@
                 ],
                 "bug_when": [
                     {
-                        "__text__": "2013-06-25 11:55:46 +0200"
+                        "__text__": "2013-07-01 12:00:00 +0200"
                     }
                 ],
                 "commentid": [
@@ -1740,6 +1740,96 @@
                         "name": "Santiago Due\u00f1as"
                     }
                 ]
+            },
+            {
+                "__text__": "\n            ",
+                "attachid": [
+                    {
+                        "__text__": "172"
+                    }
+                ],
+                "bug_when": [
+                    {
+                        "__text__": "2014-06-25 11:55:46 +0200"
+                    }
+                ],
+                "commentid": [
+                    {
+                        "__text__": "1086"
+                    }
+                ],
+                "isprivate": "0",
+                "thetext": [
+                    {
+                        "__text__": "This patch is great"
+                    }
+                ],
+                "who": [
+                    {
+                        "__text__": "lcanas@example.org",
+                        "name": "Luis Ca\u00f1as"
+                    }
+                ]
+            },
+            {
+                "__text__": "\n            ",
+                "attachid": [
+                    {
+                        "__text__": "172"
+                    }
+                ],
+                "bug_when": [
+                    {
+                        "__text__": "2014-06-25 11:55:46 +0200"
+                    }
+                ],
+                "commentid": [
+                    {
+                        "__text__": "1086"
+                    }
+                ],
+                "isprivate": "0",
+                "thetext": [
+                    {
+                        "__text__": "Thanks!"
+                    }
+                ],
+                "who": [
+                    {
+                        "__text__": "sduenas@example.org",
+                        "name": "Santiago Due\u00f1as"
+                    }
+                ]
+            },
+            {
+                "__text__": "\n            ",
+                "attachid": [
+                    {
+                        "__text__": "172"
+                    }
+                ],
+                "bug_when": [
+                    {
+                        "__text__": "2014-08-01 11:55:46 +0200"
+                    }
+                ],
+                "commentid": [
+                    {
+                        "__text__": "1086"
+                    }
+                ],
+                "isprivate": "0",
+                "thetext": [
+                    {
+                        "__text__": "Applied"
+                    }
+                ],
+                "who": [
+                    {
+                        "__text__": "sduenas@example.org",
+                        "name": "Santiago Due\u00f1as"
+                    }
+                ]
             }
         ],
         "op_sys": [
@@ -1770,8 +1860,8 @@
         ],
         "reporter": [
             {
-                "__text__": "carlosgc@example.com",
-                "name": "Carlos Garcia Campos"
+                "__text__": "sduenas@example.org",
+                "name": "Santiago Due\u00f1as"
             }
         ],
         "reporter_accessible": [

tests/test_bugzilla.py

@@ -19,6 +19,7 @@
 #     Alvaro del Castillo <acs@bitergia.com>
 #     Valerio Cosentino <valcos@bitergia.com>
 #
+
 import logging
 import unittest
 
@@ -93,6 +94,38 @@ def test_enrich_keywords(self):
             eitem = enrich_backend.get_rich_item(item)
             self.assertEqual(eitem['keywords'], [])
 
+    def test_time_to_first_attention(self):
+        """Test whether time_to_first_attention is calculated"""
+
+        self._test_raw_to_enrich()
+        enrich_backend = self.connectors[self.connector][2]()
+
+        expected = [None, None, None, None, None, None, 365]
+
+        for index in range(0, len(self.items)):
+            eitem = enrich_backend.get_rich_item(self.items[index])
+            self.assertEqual(eitem['time_to_first_attention'], expected[index])
+
+    def test_last_comment_date(self):
+        """Test whether last_comment_date is added to the enriched item"""
+
+        self._test_raw_to_enrich()
+        enrich_backend = self.connectors[self.connector][2]()
+
+        expected = [
+            "2013-06-25T11:55:46+02:00",
+            "2013-06-25T11:55:46+02:00",
+            None,
+            "2013-06-25T11:55:46+02:00",
+            "2013-06-25T11:55:46+02:00",
+            None,
+            "2014-08-01T11:55:46+02:00"
+        ]
+
+        for index in range(0, len(self.items)):
+            eitem = enrich_backend.get_rich_item(self.items[index])
+            self.assertEqual(eitem['last_comment_date'], expected[index])
+
     def test_raw_to_enrich_sorting_hat(self):
         """Test enrich with SortingHat"""