Next into master (#43)

* add desborda 2

* add test, fix multiline commands

* add desborda 2 test, fix bugs

* add more tests, enable testing max separately from num_winners

* fix bug, add test

* style corrections

* add desborda2 test

* remove desborda/desborda2 from default pipes whitelist

* add desborda2 to pdf,sort,pretty_print pipes

* total valid points for bordas

* fix pretty_print for borda

* fix women names

* add desborda2 draws test, fix pretty print

* fix

* fix pdf

* fix ties test, add multi_women test

* fix help comments

* fix desborda2 minorities duplication

* sort final winners by points

* add test to desborda2, fix identation issue

* adding desborda3

* add withdraw candidates

* fixes

* fixes

* add desborda3 to sort

* desborda3 fixes

* disable by default some pipes

* fix pretty print

* add desborda3 tests

* add desborda3 and borda tests

* set woman_names to None by default

* Work in progress: add desborda4

* WIP: add bulk of desborda4 algorithmic code

* parity rules

* fix desborda4

* refactor desborda

* fix desborda 1 & 2

* change >= to > strict, fix most tests, for desborda 1

* remove desborda4

* add some rules and info

* add test 11 for desborda2, which checks that error found in algo has been fixed

* refactor

* Go next (#41)

* release go-v1

* change version

* remove openstv dependency

* release 103111.2

* change version to go v3

* change version to go v4

* go v5

* change version to go v6

* change version to go v7

* change version to go v8

* Next (#37)

* adding desborda3

* add withdraw candidates

* fixes

* fixes

* add desborda3 to sort

* desborda3 fixes

* disable by default some pipes

* fix pretty print

* add desborda3 tests

* add desborda3 and borda tests

* set woman_names to None by default

* Work in progress: add desborda4

* WIP: add bulk of desborda4 algorithmic code

* parity rules

* fix desborda4

* refactor desborda

* fix desborda 1 & 2

* change >= to > strict, fix most tests, for desborda 1

* remove desborda4

* add some rules and info

* add test 11 for desborda2, which checks that error found in algo has been fixed

* refactor

* implementing support for counting tally sheets (#38)

* adding more unit tests and implementation of desborda4 (#39)

* adding more unit tests and implementation of desborda4

* fixing parity as requested by Leo

* fixing algorithm

* fix multiple minority teams

* add missing return

* Configurable bordas max points (#40)

* adding more unit tests and implementation of desborda4

* fixing parity as requested by Leo

* fixing algorithm

* fix multiple minority teams

* add missing return

* configurable bordas max points

* update requests version for security reasons

* using pip >= 10

* allowing pip 9 too

* depending on agora-tally master

Co-authored-by: Félix Robles <felrobelv@gmail.com>

README.md

@@ -16,7 +16,7 @@ Just execute this (no stable release yet):
 
 # Usage
 
-    $ agora-results --tally tally.tar.gz --config agora_tongo.test_config
+    $ agora-results --tally tally.tar.gz --config agora_results.test_config
 
 Or the same shorter:
 
@@ -33,6 +33,12 @@ Configuration file specifies the pipeline of functions to be applied to the resu
         ]
     ]
 
+# Testing
+
+Execute the unit tests with:
+
+    $ python3 -m unittest
+
 # Available pipes
 
 Available pipes are documented in python, in the agora_results/pipes directory.
@@ -78,4 +84,4 @@ alternatively found in <http://www.gnu.org/licenses/>.
 External libraries
 This program distributes libraries from external sources. If you follow the
 compilation process you'll download these libraries and their respective
-licenses, which are compatible with our licensing.
+licenses, which are compatible with our licensing.

agora-results

@@ -42,7 +42,10 @@ DEFAULT_PIPELINE = [
 # By default we only allow the most used pipes to reduce default attack surface
 # NOTE: keep the list sorted
 DEFAULT_PIPES_WHITELIST = [
-    #"agora_results.pipes.desborda.podemos_desborda"
+    #"agora_results.pipes.desborda.podemos_desborda",
+    #"agora_results.pipes.desborda2.podemos_desborda2",
+    #"agora_results.pipes.desborda3.podemos_desborda3",
+    #"agora_results.pipes.desborda4.podemos_desborda4",
     #"agora_results.pipes.duplicate_questions.duplicate_questions",
     "agora_results.pipes.modifications.apply_modifications",
     #"agora_results.pipes.multipart.make_multipart",
@@ -55,8 +58,9 @@ DEFAULT_PIPES_WHITELIST = [
     #"agora_results.pipes.multipart.append_ballots",
     "agora_results.pipes.parity.proportion_rounded",
     "agora_results.pipes.parity.parity_zip_non_iterative",
-    #"agora_results.pipes.parity.reorder_winners",
-    #"agora_results.pipes.parity.podemos_parity_loreg_zip_non_iterative",
+    "agora_results.pipes.parity.reorder_winners",
+    "agora_results.pipes.parity.podemos_parity_loreg_zip_non_iterative",
+    "agora_results.pipes.parity.podemos_parity2_loreg_zip_non_iterative",
     #"agora_results.pipes.podemos.podemos_proportion_rounded_and_duplicates",
     #"agora_results.pipes.pretty_print.pretty_print_stv_winners",
     "agora_results.pipes.pretty_print.pretty_print_not_iterative",
@@ -66,6 +70,8 @@ DEFAULT_PIPES_WHITELIST = [
     "agora_results.pipes.sort.sort_non_iterative",
     #"agora_results.pipes.stv_tiebreak.stv_first_round_tiebreak",
     #"agora_results.pipes.pdf.configure_pdf",
+    "agora_results.pipes.withdraw_candidates.withdraw_candidates",
+    #"agora_results.pipes.ballot_boxes.count_tally_sheets"
 ]
 
 def extract_tally(fpath):
@@ -81,7 +87,7 @@ def extract_tally(fpath):
 def print_csv(data, separator, output_func=print):
     counts = data['results']['questions']
     for question, i in zip(counts, range(len(counts))):
-        if question['tally_type'] not in ["plurality-at-large", "desborda", "borda", "borda-nauru"] or\
+        if question['tally_type'] not in ["plurality-at-large", "desborda", "desborda2", "desborda3", "borda", "borda-nauru"] or\
            question.get('no-tally', False):
             continue
 

agora_results/pipes/ballot_boxes.py

@@ -0,0 +1,222 @@
+# -*- coding:utf-8 -*-
+
+# This file is part of agora-results.
+# Copyright (C) 2014-2016  Agora Voting SL <agora@agoravoting.com>
+
+# agora-results is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License.
+
+# agora-results  is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with agora-results.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import json
+
+def _initialize_question(question, override):
+    '''
+    Initialize question results to zero if any
+    '''
+    if 'winners' not in question or override:
+        question['winners'] = []
+    if 'totals' not in question or override:
+        question['totals'] = dict(
+            blank_votes=0,
+            null_votes=0,
+            valid_votes=0
+        )
+    for answer in question['answers']:
+        if "total_count" not in answer:
+            answer['total_count'] = 0
+
+def _verify_tally_sheet(tally_sheet, questions, tally_index):
+    '''
+    Verify that the tally sheets conform with the list of questions and answers
+    of this election
+    '''
+
+    assert\
+        'num_votes' in tally_sheet,\
+        'sheet %d: no num_votes' % tally_index
+    assert\
+        isinstance(tally_sheet['num_votes'], int),\
+        'sheet %d: num_votes is not an integer' % tally_index
+    assert\
+        tally_sheet['num_votes'] >= 0,\
+          'sheet %d: num_votes is negative' % tally_index
+
+    assert\
+      'questions' in tally_sheet,\
+      'sheet %d: tally_sheet has no questions' % tally_index
+    assert\
+      isinstance(tally_sheet['questions'], list),\
+      'sheet %d: tally_sheet questions is not a list' % tally_index
+    assert\
+      len(tally_sheet['questions']) == len(questions),\
+      'sheet %d: tally_sheet has invalid number of questions' % tally_index
+
+    for qindex, question in enumerate(questions):
+        sheet_question = tally_sheet['questions'][qindex]
+
+        assert\
+            'title' in sheet_question,\
+            'sheet %d, question %d: no title' % (tally_index, qindex)
+        assert\
+            isinstance(sheet_question['title'], str),\
+            'sheet %d, question %d: title is not a string' % (tally_index, qindex)
+        assert\
+            question['title'] == sheet_question['title'],\
+            'sheet %d, question %d: invalid title' % (tally_index, qindex)
+
+        assert\
+            'tally_type' in sheet_question,\
+            'sheet %d, question %d: no tally_type' % (tally_index, qindex)
+        assert\
+            isinstance(sheet_question['tally_type'], str),\
+            'sheet %d, question %d: tally_type is not a string' % (tally_index, qindex)
+        assert\
+            sheet_question['tally_type'] == question['tally_type'],\
+            'sheet %d, question %d: tally_type is not a string' % (tally_index, qindex)
+        assert\
+            sheet_question['tally_type'] in ['plurality-at-large'],\
+            'sheet %d, question %d: tally_type is not allowed' % (tally_index, qindex)
+
+        assert\
+            'blank_votes' in sheet_question,\
+            'sheet %d, question %d: no blank_votes' % (tally_index, qindex)
+        assert\
+            isinstance(sheet_question['blank_votes'], int),\
+            'sheet %d, question %d: blank_votes is not an integer' % (tally_index, qindex)
+        assert\
+            sheet_question['blank_votes'] >= 0,\
+            'sheet %d, question %d: blank_votes is negative' % (tally_index, qindex)
+
+        assert\
+            'null_votes' in sheet_question,\
+            'sheet %d, question %d: no null_votes' % (tally_index, qindex)
+        assert\
+            isinstance(sheet_question['null_votes'], int),\
+            'sheet %d, question %d: null_votes is not an integer' % (tally_index, qindex)
+        assert\
+            sheet_question['null_votes'] >= 0,\
+            'sheet %d, question %d: null_votes is negative' % (tally_index, qindex)
+
+        assert\
+            'answers' in sheet_question,\
+            'sheet %d, question %d: no answers' % (tally_index, qindex)
+        assert\
+            isinstance(sheet_question['answers'], list),\
+            'sheet %d, question %d: question answers is not a list' % (tally_index, qindex)
+        assert\
+            len(sheet_question['answers']) == len(question['answers']),\
+            'sheet %d, question %d: invalid number of answers' % (tally_index, qindex)
+
+        sheet_answers = dict([
+          (answer['text'], answer)
+          for answer in sheet_question['answers']
+        ])
+
+        answers = dict([
+          (answer['text'], answer)
+          for answer in question['answers']
+        ])
+
+        assert\
+            set(sheet_answers.keys()) == set(answers.keys()),\
+            'not the same set of answers'
+
+        for aindex, answer in enumerate(question['answers']):
+            text = answer['text']
+            sheet_answer = sheet_answers[text]
+
+            assert\
+                'text' in sheet_answer,\
+                'sheet %d, question %d, answer %d: no text' % (tally_index, qindex, aindex)
+            assert\
+                isinstance(sheet_answer['text'], str),\
+                'sheet %d, question %d, answer %d: text is not a string' % (tally_index, qindex, aindex)
+            assert\
+                sheet_answer['text'] == answer['text'],\
+                'sheet %d, question %d, answer %d: text is not valid "%s" != "%s"' % (
+                    tally_index,
+                    qindex,
+                    aindex,
+                    sheet_answer['text'],
+                    answer['text']
+                )
+
+            assert\
+                'num_votes' in sheet_answer,\
+                'sheet %d, question %d, answer %d: no num_votes' % (tally_index, qindex, aindex)
+            assert\
+                isinstance(sheet_answer['num_votes'], int),\
+                'sheet %d, question %d, answer %d: num_votes is not an int' % (tally_index, qindex, aindex)
+            assert\
+                sheet_answer['num_votes'] >= 0,\
+                'sheet %d, question %d, answer %d: num_votes is negative' % (tally_index, qindex, aindex)
+
+        assert\
+            sum([
+                sum([
+                    answer['num_votes']
+                    for answer in sheet_question['answers']
+                ]),
+                sheet_question['blank_votes'],
+                sheet_question['null_votes']
+            ]) == tally_sheet['num_votes'],\
+            'sheet %d, question %d: number of votes does not match' % (tally_index, qindex)
+
+def _sum_tally_sheet_numbers(tally_sheet, results, tally_index):
+    '''
+    Adds the results of the tally sheet
+    '''
+    questions = results['questions']
+    results['total_votes'] += tally_sheet['num_votes']
+
+    for qindex, question in enumerate(questions):
+        sheet_question = tally_sheet['questions'][qindex]
+        question['totals']['blank_votes'] += sheet_question['blank_votes']
+        question['totals']['null_votes'] += sheet_question['null_votes']
+
+        sheet_answers = dict([
+          (answer['text'], answer)
+          for answer in sheet_question['answers']
+        ])
+
+        for aindex, answer in enumerate(question['answers']):
+            text = answer['text']
+            sheet_answer = sheet_answers[text]
+            answer['total_count'] += sheet_answer['num_votes']
+            question['totals']['valid_votes'] += sheet_answer['num_votes']
+
+# given a list of tally_sheets, add them to the electoral results
+def count_tally_sheets(data_list, tally_sheets, override=False):
+    data = data_list[0]
+
+    if override or 'results' not in data:
+        questions_path = os.path.join(data['extract_dir'], "questions_json")
+        with open(questions_path, 'r', encoding="utf-8") as f:
+            questions = json.loads(f.read())
+        data['results'] = dict(
+            questions=questions,
+            total_votes=0
+        )
+    else:
+        questions = data['results']['questions']
+
+    # initialize
+    for question in questions:
+        _initialize_question(question, override=override)
+
+    # check ballot_box_list
+    for tally_index, tally_sheet in enumerate(tally_sheets):
+        _verify_tally_sheet(tally_sheet, questions, tally_index)
+
+    # add the numbers of each tally sheet to the electoral results
+    for tally_index, tally_sheet in enumerate(tally_sheets):
+        _sum_tally_sheet_numbers(tally_sheet, data['results'], tally_index)