add unit support for null and blank votes in tally-pipes tests and ad…

…d one test with those (#88)

test/desborda_test.py

@@ -34,15 +34,15 @@ def remove_spaces(in_str):
 
 def has_input_format(in_str):
     # example: "A1f, B2m \nB3f"
-    m = re.fullmatch(r"((\s*[A-Z][0-9]+[fm]\s*,)*\s*[A-Z][0-9]+[fm]\s*\n)*(\s*[A-Z][0-9]+[fm]\s*,)*\s*[A-Z][0-9]+[fm]\s*\n?", in_str)
+    m = re.fullmatch(r"((\s*(<blank>|<null>|[A-Z][0-9]+[fm])\s*,)*\s*(<blank>|<null>|[A-Z][0-9]+[fm])\s*\n)*(\s*(<blank>|<null>|[A-Z][0-9]+[fm])\s*,)*\s*(<blank>|<null>|[A-Z][0-9]+[fm])\s*\n?", in_str)
     return m is not None
 
 def has_output_format(out_str):
     # example: "A1f, 1,  3\n B33m, 4"
     m = re.fullmatch(r"(\s*[A-Z][0-9]+[fm]\s*,\s*[0-9]+\s*\n)*\s*[A-Z][0-9]+[fm]\s*,\s*[0-9]+\s*\n?", out_str)
     return m is not None
 
-def encode_valid_ballot(
+def encode_ballot(
     text_ballot, 
     question
 ):
@@ -52,6 +52,7 @@ def encode_valid_ballot(
     ballot_question = copy.deepcopy(question)
     normal_choices_dict = dict()
     write_in_choices_dict = dict()
+    is_invalid_vote_flag = False
     for choice_index, text_choice in enumerate(text_ballot):
         if '#' in text_choice:
             answer_id, answer_text = text_choice.split('#')
@@ -60,13 +61,16 @@ def encode_valid_ballot(
             answer_id = None
             answer_text = text_choice
         if answer_id is None:
-            if answer_text in normal_choices_dict:
-                normal_choices_dict[answer_text]['count'] += 1
-            else:
-                normal_choices_dict[answer_text] = dict(
-                    count=1,
-                    choice_index=choice_index
-                )
+            if answer_text not in ['<blank>', '<null>']:
+                if answer_text in normal_choices_dict:
+                    normal_choices_dict[answer_text]['count'] += 1
+                else:
+                    normal_choices_dict[answer_text] = dict(
+                        count=1,
+                        choice_index=choice_index
+                    )
+            elif answer_text == '<null>':
+                is_invalid_vote_flag = True
         else:
             if answer_id in write_in_choices_dict:
                 write_in_choices_dict[answer_id]['count'] += 1
@@ -94,6 +98,9 @@ def encode_valid_ballot(
 
     ballot_encoder = NVotesCodec(ballot_question)
     raw_ballot = ballot_encoder.encode_raw_ballot()
+    # if it's an invalid ballot, set the invalid ballot flag
+    if is_invalid_vote_flag:
+        raw_ballot['choices'][0] = 1
     int_ballot = ballot_encoder.encode_to_int(raw_ballot)
     return str(int_ballot + 1)
 
@@ -253,6 +260,8 @@ def create_desborda_test(
             if len_ballot > max_num:
                 max_num = len_ballot
             for candidate in ballot:
+                if candidate in ['<blank>', '<null>']:
+                    continue
                 team = candidate[:1]
                 female = "f" == candidate[-1]
                 if team not in teams:
@@ -347,7 +356,7 @@ def create_desborda_test(
         counter += 1
         if counter % 1000 == 0:
             print("%i votes encoded" % counter)
-        encoded_ballot = encode_valid_ballot(
+        encoded_ballot = encode_ballot(
             text_ballot=ballot,
             question=question
         )

test/output_tests/test_output_1

@@ -1,4 +1,6 @@
 Votes: borda output test1, with gender transpose from urls to answer
+<blank>
+<null>
 A1f,B1m
 A1f,B1m,D1m
 C1f,A1f,D1m
@@ -9,11 +11,15 @@ B1m, 4
 D1m, 2
 
 Ballots CSV:
+0,BLANK_VOTE
+0,NULL_VOTE
 0,"0. A1f","1. B1m"
 0,"0. A1f","1. B1m","2. D1m"
 0,"3. C1f","0. A1f","2. D1m"
 
 Ballots JSON:
+{"answer_total_votes_percentage": "over-total-valid-votes", "answers": [{"Gender": "https://sequentech.io/api/gender/M", "category": "A", "details": "A1f", "id": 0, "text": "A1f", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "category": "B", "details": "B1m", "id": 1, "text": "B1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "category": "D", "details": "D1m", "id": 2, "text": "D1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/M", "category": "C", "details": "C1f", "id": 3, "text": "C1f", "total_count": 0, "urls": []}], "ballot_flags": {"blank_vote": true, "implicit_null_vote": false, "null_vote": false}, "description": "Desborda question", "election_index": 0, "layout": "simple", "max": 3, "min": 0, "num_winners": 3, "question_num": 0, "randomize_answer_order": true, "tally_type": "borda", "title": "Desborda question"}
+{"answer_total_votes_percentage": "over-total-valid-votes", "answers": [{"Gender": "https://sequentech.io/api/gender/M", "category": "A", "details": "A1f", "id": 0, "text": "A1f", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "category": "B", "details": "B1m", "id": 1, "text": "B1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "category": "D", "details": "D1m", "id": 2, "text": "D1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/M", "category": "C", "details": "C1f", "id": 3, "text": "C1f", "total_count": 0, "urls": []}], "ballot_flags": {"blank_vote": false, "implicit_null_vote": false, "null_vote": true}, "description": "Desborda question", "election_index": 0, "layout": "simple", "max": 3, "min": 0, "num_winners": 3, "question_num": 0, "randomize_answer_order": true, "tally_type": "borda", "title": "Desborda question"}
 {"answer_total_votes_percentage": "over-total-valid-votes", "answers": [{"Gender": "https://sequentech.io/api/gender/M", "ballot_marks": 0, "category": "A", "details": "A1f", "id": 0, "text": "A1f", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "ballot_marks": 1, "category": "B", "details": "B1m", "id": 1, "text": "B1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "category": "D", "details": "D1m", "id": 2, "text": "D1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/M", "category": "C", "details": "C1f", "id": 3, "text": "C1f", "total_count": 0, "urls": []}], "ballot_flags": {"blank_vote": false, "implicit_null_vote": false, "null_vote": false}, "description": "Desborda question", "election_index": 0, "layout": "simple", "max": 3, "min": 0, "num_winners": 3, "question_num": 0, "randomize_answer_order": true, "tally_type": "borda", "title": "Desborda question"}
 {"answer_total_votes_percentage": "over-total-valid-votes", "answers": [{"Gender": "https://sequentech.io/api/gender/M", "ballot_marks": 0, "category": "A", "details": "A1f", "id": 0, "text": "A1f", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "ballot_marks": 1, "category": "B", "details": "B1m", "id": 1, "text": "B1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "ballot_marks": 2, "category": "D", "details": "D1m", "id": 2, "text": "D1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/M", "category": "C", "details": "C1f", "id": 3, "text": "C1f", "total_count": 0, "urls": []}], "ballot_flags": {"blank_vote": false, "implicit_null_vote": false, "null_vote": false}, "description": "Desborda question", "election_index": 0, "layout": "simple", "max": 3, "min": 0, "num_winners": 3, "question_num": 0, "randomize_answer_order": true, "tally_type": "borda", "title": "Desborda question"}
 {"answer_total_votes_percentage": "over-total-valid-votes", "answers": [{"Gender": "https://sequentech.io/api/gender/M", "ballot_marks": 1, "category": "A", "details": "A1f", "id": 0, "text": "A1f", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "category": "B", "details": "B1m", "id": 1, "text": "B1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/H", "ballot_marks": 2, "category": "D", "details": "D1m", "id": 2, "text": "D1m", "total_count": 0, "urls": []}, {"Gender": "https://sequentech.io/api/gender/M", "ballot_marks": 0, "category": "C", "details": "C1f", "id": 3, "text": "C1f", "total_count": 0, "urls": []}], "ballot_flags": {"blank_vote": false, "implicit_null_vote": false, "null_vote": false}, "description": "Desborda question", "election_index": 0, "layout": "simple", "max": 3, "min": 0, "num_winners": 3, "question_num": 0, "randomize_answer_order": true, "tally_type": "borda", "title": "Desborda question"}