bugfixes (#1019)

* specify the quoting for csv exports

this fixes an issue where numbers that should be
treated as strings are not quoted and strings that
contain other characters that could be delimiters
are not quoted properly.

* added an extra check for registered voters if it
is a form field to confirm that the value is not
a null value.

Author: takinbo

apollo/locations/services.py

@@ -74,7 +74,7 @@ def export_list(self, query):
 
         output = StringIO()
         output.write(constants.BOM_UTF8_STR)
-        writer = csv.writer(output)
+        writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
         writer.writerow(headers)
         yield output.getvalue()
         output.close()
@@ -121,7 +121,7 @@ def export_list(self, query):
                 record.append(row[index])
 
             output = StringIO()
-            writer = csv.writer(output)
+            writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
             writer.writerow(record)
             yield output.getvalue()
             output.close()

apollo/messaging/services.py

@@ -13,8 +13,7 @@
 class MessageService(Service):
     __model__ = Message
 
-    def log_message(self, event, direction, text, recipient='', sender='',
-                    timestamp=None, message_type='SMS'):
+    def log_message(self, event, direction, text, recipient="", sender="", timestamp=None, message_type="SMS"):
         if timestamp:
             try:
                 msg_time = datetime.utcfromtimestamp(timestamp)
@@ -24,38 +23,38 @@ def log_message(self, event, direction, text, recipient='', sender='',
             msg_time = datetime.utcnow()
 
         return self.create(
-            direction=direction, recipient=recipient, sender=sender, text=text,
-            deployment_id=event.deployment_id, event=event, received=msg_time,
-            message_type=message_type)
+            direction=direction,
+            recipient=recipient,
+            sender=sender,
+            text=text,
+            deployment_id=event.deployment_id,
+            event=event,
+            received=msg_time,
+            message_type=message_type,
+        )
 
     def export_list(self, query):
-        headers = [
-            _('Mobile'), _('Text'), _('Direction'), _('Created'),
-            _('Delivered'), _('Type')
-        ]
+        headers = [_("Mobile"), _("Text"), _("Direction"), _("Created"), _("Delivered"), _("Type")]
         output = StringIO()
         output.write(constants.BOM_UTF8_STR)
-        writer = csv.writer(output)
+        writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
         writer.writerow([str(i) for i in headers])
         yield output.getvalue()
         output.close()
 
         for message in query:
             # limit to three numbers for export and pad if less than three
             record = [
-                message.sender if message.direction == 'IN'
-                else message.recipient,
+                message.sender if message.direction == "IN" else message.recipient,
                 message.text,
                 message.direction.code,
-                message.received.strftime('%Y-%m-%d %H:%M:%S')
-                if message.received else '',
-                message.delivered.strftime('%Y-%m-%d %H:%M:%S')
-                if message.delivered else '',
-                message.message_type.value
+                message.received.strftime("%Y-%m-%d %H:%M:%S") if message.received else "",
+                message.delivered.strftime("%Y-%m-%d %H:%M:%S") if message.delivered else "",
+                message.message_type.value,
             ]
 
             output = StringIO()
-            writer = csv.writer(output)
+            writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
             writer.writerow([str(i) for i in record])
             yield output.getvalue()
             output.close()

apollo/participants/services.py

@@ -53,7 +53,7 @@ def export_list(self, query):
         # TODO: extra fields missing
         output_buffer = StringIO()
         output_buffer.write(constants.BOM_UTF8_STR)
-        writer = csv.writer(output_buffer)
+        writer = csv.writer(output_buffer, quoting=csv.QUOTE_NONNUMERIC)
 
         writer.writerow(headers)
         yield output_buffer.getvalue()
@@ -97,7 +97,7 @@ def export_list(self, query):
 
             # TODO: process extra fields here
             output_buffer = StringIO()
-            writer = csv.writer(output_buffer)
+            writer = csv.writer(output_buffer, quoting=csv.QUOTE_NONNUMERIC)
             writer.writerow(record)
             yield output_buffer.getvalue()
             output_buffer.close()

apollo/result_analysis/results.py

@@ -61,6 +61,7 @@ def _margin_of_error(dataframe, numerator, denominator, cv=1.96):
     return moe
 
 
+# TODO: use proper field terminology for the variable names in this function (field, field_label, etc.)
 def voting_results(form_id, location_id=None):
     """Compute voting results."""
     event = g.event
@@ -131,6 +132,7 @@ def voting_results(form_id, location_id=None):
 
     dataset = make_submission_dataframe(filter_set.qs, form, excluded_tags=excluded_fields)
 
+    # Check to replace records that have the configured null value with nan
     for result_field in result_fields:
         null_value_orig = result_field.get("null_value")
         if null_value_orig is None:
@@ -142,6 +144,17 @@ def voting_results(form_id, location_id=None):
 
         dataset[result_field["tag"]] = dataset[result_field["tag"]].replace(null_value, np.nan)
 
+    # Check to replace records that have the configured null value for registered voters
+    # (if the column is derived from a reported field) with nan
+    if form.registered_voters_tag:
+        try:
+            _null_value = form.get_field_by_tag(form.registered_voters_tag).get("null_value")
+            if null_value is not None:
+                null_value = int(_null_value)
+                dataset[registered_voters_field] = dataset[registered_voters_field].replace(null_value, np.nan)
+        except (TypeError, ValueError):
+            pass
+
     if not dataset.empty:
         # compute and store reporting status
         dataset["reported"] = (

apollo/submissions/aggregation.py

@@ -124,7 +124,7 @@ def aggregate_dataset(query, form, stream=False):
     if stream:
         output_stream = StringIO()
         output_stream.write(codecs.BOM_UTF8.decode("utf-8"))
-        writer = csv.writer(output_stream)
+        writer = csv.writer(output_stream, quoting=csv.QUOTE_NONNUMERIC)
         writer.writerow(headers)
 
         yield output_stream.getvalue()
@@ -179,7 +179,7 @@ def aggregate_dataset(query, form, stream=False):
 
             if stream:
                 output_stream = StringIO()
-                writer = csv.writer(output_stream)
+                writer = csv.writer(output_stream, quoting=csv.QUOTE_NONNUMERIC)
                 writer.writerow(current_row)
 
                 yield output_stream.getvalue()

apollo/submissions/services.py

@@ -113,7 +113,7 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False):
 
         output = StringIO()
         output.write(constants.BOM_UTF8_STR)
-        writer = csv.writer(output)
+        writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
         writer.writerow(dataset_headers)
         yield output.getvalue()
         output.close()
@@ -215,7 +215,7 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False):
                 ]
 
             output = StringIO()
-            writer = csv.writer(output)
+            writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
             writer.writerow(record)
             yield output.getvalue()
             output.close()