Merge pull request #533 from baobabsoluciones/release/v1.1.0

Release/v1.1.0

.github/workflows/test_cornflow_client.yml

@@ -44,7 +44,7 @@ jobs:
         pip install .
     - name: Test
       run: |
-        coverage run --source=./cornflow_client/ -m unittest discover -s cornflow_client/tests/unit
+        coverage run --source=./cornflow_client/ --rcfile=./.coveragerc -m unittest discover -s cornflow_client/tests/unit
         coverage report -m
   
   integration:
@@ -140,11 +140,11 @@ jobs:
           CORNFLOW_SERVICE_USER: airflow
       - name: Run unit tests
         run: |
-          coverage run --source=./cornflow_client/ -m unittest discover -s cornflow_client/tests/unit
+          coverage run --source=./cornflow_client/ --rcfile=./.coveragerc -m unittest discover -s cornflow_client/tests/unit
           coverage report -m
       - name: Run integration tests
         run: |
-          coverage run -a --source=./cornflow_client/ -m unittest discover -s cornflow_client/tests/integration
+          coverage run -a --source=./cornflow_client/ --rcfile=./.coveragerc -m unittest discover -s cornflow_client/tests/integration
           coverage report -m
           coverage xml
       - name: Upload coverage to codecov

.github/workflows/test_cornflow_dags.yml

@@ -76,11 +76,11 @@ jobs:
         AIRFLOW_CONN_CF_URI: cornflow://airflow_test@admin.com:airflow_test_password@localhost:5000
     - name: Run unit tests
       run: |
-        coverage run --source=. -m unittest tests/test_dags.py
+        coverage run --source=. --rcfile=./.coveragerc -m unittest tests/test_dags.py
         coverage report -m
     - name: Run integration tests
       run: |
-        coverage run -a --source=. -m unittest tests/test_dags_af.py
+        coverage run -a --source=. --rcfile=./.coveragerc -m unittest tests/test_dags_af.py
         coverage report -m
         coverage xml
     - name: Upload coverage to codecov

.github/workflows/test_cornflow_server.yml

@@ -103,7 +103,7 @@ jobs:
         AIRFLOW_CONN_CF_URI: http://airflow:Airflow_test_password1@localhost:5050
     - name: Run unit tests
       run: | 
-        coverage run --source=./cornflow/ --omit="*/tests/data/*" -m unittest discover -s cornflow/tests/unit
+        coverage run --source=./cornflow/ --rcfile=./.coveragerc -m unittest discover -s cornflow/tests/unit
         coverage report -m
       env:
         FLASK_ENV: testing
@@ -115,7 +115,7 @@ jobs:
         CF_ALARMS_ENDPOINT: 1
     - name: Run ldap unit tests
       run: | 
-        coverage run -a --source=./cornflow/ -m unittest discover -s cornflow/tests/ldap
+        coverage run -a --source=./cornflow/ --rcfile=./.coveragerc -m unittest discover -s cornflow/tests/ldap
         coverage report -m
       env:
         FLASK_ENV: testing
@@ -143,7 +143,7 @@ jobs:
         CORNFLOW_SERVICE_USER: cornflow
     - name: Run postgres unit tests
       run: | 
-        coverage run -a --source=./cornflow/ -m unittest cornflow/tests/unit/test_commands.py
+        coverage run -a --source=./cornflow/ --rcfile=./.coveragerc -m unittest cornflow/tests/unit/test_commands.py
         coverage report -m
       env:
         FLASK_ENV: testing
@@ -153,7 +153,7 @@ jobs:
         CF_ALARMS_ENDPOINT: 1
     - name: Run integration tests
       run: |
-        coverage run -a --source=./cornflow/ -m unittest discover -s cornflow/tests/integration
+        coverage run -a --source=./cornflow/ --rcfile=./.coveragerc -m unittest discover -s cornflow/tests/integration
         coverage report -m
         coverage xml
       env:

SECURITY.md

@@ -0,0 +1,19 @@
+# Security Policy
+
+This document contains information on how to report security vulnerabilities in **cornflow** and how security issues reported to the **cornflow development team** are handled.
+
+## Please do not file GitHub issues fo security vulnerabilities as they are public!
+
+The cornflow development team takes security issues very seriously. If you have any concern around cornflow security or belive you have uncovered a vulnerability, we suggest that you get in touch via the e-mail addresses cornflow@baobabsoluciones.es and seguridad@baobabsoluciones.es
+
+Before sending the report, however, please read the following guideslines first. The guidleines should answer the most common questions you might have about reporting vulnerabilities.
+
+## What should be and should NOT be reported?
+
+**Only** use the security e-mail address to report undisclosed security vulnerabilities in cornflow and to manage the process of fixing such vulnerabilities. We do not accept regular bug reports or other security-related queries at this address. We will ignore mail sent to this address that does not relate to an undisclosed security problem in the cornflow project. Please follow regular communication channels for inquiries, questions and other discussions related to the process or issues.
+
+Specifically, we will ignore results of security scans that contain a list of dependencies of cornflow with dependencies in cornflow docker reference image.
+
+## How to report the issue?
+
+Please send one plain-text email for each vulnerability you are reporting including an explanation of how it affects cornflow security. We may ask that you resubmit your report if you send it as an image, movie, HTML, or PDF attachment when you could as easily describe it with plain text.

cornflow-dags/.coveragerc

@@ -0,0 +1,7 @@
+[run]
+omit =
+    */tests/*
+
+[report]
+exclude_also =
+    raise NotImplementedError

cornflow-dags/DAG/bar_cutting/__init__.py

@@ -20,24 +20,18 @@ class BarCutting(ApplicationCore):
 
     @property
     def test_cases(self) -> List[Union[Dict, Tuple[Dict, Dict]]]:
-        options_instance = ["data/example_instance_1.json"]
-
-        options_solution = ["data/example_solution_1.json"]
+        data_1 = load_json(
+            os.path.join(os.path.dirname(__file__), "data/example_instance_1.json")
+        )
+        data_out_1 = load_json(
+            os.path.join(os.path.dirname(__file__), "data/example_solution_1.json")
+        )
 
         return [
-            (
-                load_json(
-                    os.path.join(
-                        os.path.dirname(__file__),
-                        options_instance[i],
-                    )
-                ),
-                load_json(
-                    os.path.join(
-                        os.path.dirname(__file__),
-                        options_solution[i],
-                    )
-                ),
-            )
-            for i in range(len(options_instance))
+            {
+                "name": "Base case",
+                "instance": data_1,
+                "solution": data_out_1,
+                "description": "Base instance and solution",
+            }
         ]

cornflow-dags/DAG/bar_cutting/schemas/config.json

@@ -4,7 +4,7 @@
     "properties": {
         "solver": {
 			"type": "string",
-			"enum": ["mip", "CG", "mip.cbc", "CG.cbc"],
+			"enum": ["mip", "CG", "mip.cbc", "mip.gurobi", "CG.cbc", "CG.gurobi"],
 			"default": "mip.cbc"
 			},
         "timeLimit": {

cornflow-dags/DAG/bar_cutting/solvers/column_generation.py

@@ -204,13 +204,14 @@ def solve(self, config):
             _, solver_name = solver_name.split(".")
 
         SOLVER_PARAMETERS = dict(
-            sec=config.get("timeLimit", 360),
-            allow=config.get("gapAbs", 1),
-            ratio=config.get("gaPRel", 0.01),
-            tee=config.get("msg", 1),
+            time_limit=config.get("timeLimit", 360),
+            abs_gap=config.get("abs_gap", 1),
+            rel_gap=config.get("rel_gap", 0.01),
+            solver=solver_name
         )
+        SOLVER_PARAMETERS = self.get_solver_config(SOLVER_PARAMETERS)
 
-        opt = SolverFactory(solver_name)
+        opt = SolverFactory(solver_name, tee=config.get("msg", 1))
         opt.options.update(SOLVER_PARAMETERS)
 
         more_patterns_bar1 = True

cornflow-dags/DAG/bar_cutting/solvers/mip_model.py

@@ -140,13 +140,14 @@ def solve(self, config):
             _, solver_name = solver_name.split(".")
 
         SOLVER_PARAMETERS = dict(
-            sec=config.get("timeLimit", 360),
-            allow=config.get("gapAbs", 1),
-            ratio=config.get("gaPRel", 0.01),
-            tee=config.get("msg", 1),
+            time_limit=config.get("timeLimit", 360),
+            abs_gap=config.get("abs_gap", 1),
+            rel_gap=config.get("rel_gap", 0.01),
+            solver=solver_name
         )
+        SOLVER_PARAMETERS = self.get_solver_config(SOLVER_PARAMETERS)
 
-        opt = SolverFactory(solver_name)
+        opt = SolverFactory(solver_name, tee=config.get("msg", 1))
         opt.options.update(SOLVER_PARAMETERS)
         results = opt.solve(model_instance)
 

cornflow-dags/DAG/dag_timer.py

@@ -46,4 +46,4 @@ class Timer(ApplicationCore):
 
     @property
     def test_cases(self):
-        return [{}]
+        return []

cornflow-dags/DAG/facility_location/__init__.py

@@ -18,4 +18,10 @@ def test_cases(self):
         data = load_json(
             os.path.join(os.path.dirname(__file__), "data/input_data_test1.json")
         )
-        return [data]
+        return [
+            {
+                "name": "Base case",
+                "instance": data,
+                "description": "Base example instance",
+            }
+        ]

cornflow-dags/DAG/facility_location/schemas/config.json

@@ -4,7 +4,7 @@
     "properties": {
         "solver": {
 			"type": "string",
-			"enum": ["Pyomo", "Pyomo.cbc"],
+			"enum": ["Pyomo", "Pyomo.cbc", "Pyomo.gurobi"],
 			"default": "Pyomo.cbc"
 			},
         "timeLimit": {

cornflow-dags/DAG/facility_location/solvers/PyomoSolver.py

@@ -338,8 +338,8 @@ def solve(self, config):
             _, solver_name = solver_name.split(".")
 
         opt = SolverFactory(solver_name)
-        opt.options.update(config)
-        results = opt.solve(model_instance)
+        opt.options.update(self.get_solver_config(config))
+        results = opt.solve(model_instance, tee=config.get("msg", True))
 
         status = results.solver.status
         termination_condition = PYOMO_STOP_MAPPING[results.solver.termination_condition]

cornflow-dags/DAG/graph_coloring/__init__.py

@@ -33,6 +33,16 @@ def read_file(filePath):
             return dict(pairs=pairs)
 
         file_dir = os.path.join(os.path.dirname(__file__), "data")
-        files = os.listdir(file_dir)
-        test_files = pt.TupList(files).vfilter(lambda v: v.startswith("gc_"))
-        return [read_file(os.path.join(file_dir, fileName)) for fileName in test_files]
+
+        return [
+            {
+                "name": "gc_4_1",
+                "instance": read_file(os.path.join(file_dir, "gc_4_1")),
+                "description": "Example data with 4 pairs",
+            },
+            {
+                "name": "gc_50_1",
+                "instance": read_file(os.path.join(file_dir, "gc_50_1")),
+                "description": "Example data with 50 pairs",
+            },
+        ]

cornflow-dags/DAG/knapsack/__init__.py

@@ -1,4 +1,4 @@
-from typing import  Union, Type
+from typing import Union, Type
 from cornflow_client import ApplicationCore, get_empty_schema
 import os
 from .core import Instance, Solution, Experiment
@@ -17,7 +17,8 @@ class Knapsack(ApplicationCore):
         MIP=MIPSolver,
     )
     schema = get_empty_schema(
-        properties=dict(timeLimit=dict(type="number")), solvers=list(solvers.keys()) + ["MIP.cbc"],
+        properties=dict(timeLimit=dict(type="number")),
+        solvers=list(solvers.keys()) + ["MIP.cbc", "MIP.gurobi"],
     )
 
     def get_solver_name(self, data, conf):
@@ -39,10 +40,16 @@ def solve(self, data, conf, solution_data=None):
     @property
     def test_cases(self):
         cwd = os.path.dirname(os.path.realpath(__file__))
-        path = os.path.join(cwd, "Data", "ks_4_0")
+        path = os.path.join(cwd, "data", "ks_4_0")
 
         data = Instance.from_file(path).to_dict()
-        return [data]
+        return [
+            {
+                "name": "ks_4_0",
+                "instance": data,
+                "description": "Example data with 4 items",
+            }
+        ]
 
     def get_solver(self, name: str = "default") -> Union[Type[Experiment], None]:
         if "." in name:

cornflow-dags/DAG/knapsack/solvers/MIPSolver.py

@@ -79,8 +79,8 @@ def solve(self, config):
             _, solver_name = solver_name.split(".")
 
         opt = SolverFactory(solver_name)
-        opt.options.update(config)
-        results = opt.solve(model_instance)
+        opt.options.update(self.get_solver_config(config))
+        results = opt.solve(model_instance, tee=config.get("msg", True))
 
         status = results.solver.status
         termination_condition = PYOMO_STOP_MAPPING[results.solver.termination_condition]

cornflow-dags/DAG/roadef/__init__.py

@@ -19,6 +19,12 @@ def test_cases(self):
         _get_file = lambda name: os.path.join(cwd, "data", name)
         _get_instance = lambda fn: Instance.from_json(_get_file(fn)).to_dict()
         _get_solution = lambda fn: Solution.from_json(_get_file(fn)).to_dict()
-        return TupList(
-            [("example_instance_filtered.json", "example_solution_filtered.json")]
-        ).vapply(lambda v: (_get_instance(v[0]), _get_solution(v[1])))
+
+        return [
+            {
+                "name": "Base case",
+                "instance": _get_instance("example_instance_filtered.json"),
+                "solution": _get_solution("example_solution_filtered.json"),
+                "description": "Example data",
+            }
+        ]

cornflow-dags/DAG/roadef/schemas/config.json

@@ -4,7 +4,17 @@
     "solver":{
       "type": "string",
       "default": "Greedy",
-      "enum": ["Greedy", "MIPModel", "PeriodicMIPModel", "MIPModel.PULP_CBC_CMD", "PeriodicMIPModel.PULP_CBC_CMD"]
+      "enum": [
+        "Greedy",
+        "MIPModel",
+        "PeriodicMIPModel",
+        "MIPModel.PULP_CBC_CMD",
+        "PeriodicMIPModel.PULP_CBC_CMD",
+        "MIPModel.GUROBI_CMD",
+        "PeriodicMIPModel.GUROBI_CMD",
+        "MIPModel.GUROBI",
+        "PeriodicMIPModel.GUROBI"
+      ]
     },
     "timeLimit":{
       "type": "number"

cornflow-dags/DAG/roadef/solvers/MIPModel.py

@@ -88,21 +88,23 @@ def solve(self, config=None):
         solver_name = self.get_solver(config)
         config_first = dict(
             solver=solver_name,
-            gapRel=0.1,
+            rel_gap=0.1,
             timeLimit=min(200.0, self._get_remaining_time()),
             msg=self.print_log,
         )
+        config_first = self.get_solver_config(config_first, lib="pulp")
 
         def config_iteration(self, warm_start):
-            return dict(
+            conf = dict(
                 solver=solver_name,
-                gapRel=0.05,
+                rel_gap=0.05,
                 timeLimit=min(100.0, self._get_remaining_time()),
                 msg=self.print_log,
                 warmStart=warm_start,
             )
+            return self.get_solver_config(conf, lib="pulp")
 
-        solver = pl.getSolver(**config_first)
+        solver = pl.getSolver(solver=solver_name, **config_first)
         used_routes, previous_value = self.solve_one_iteration(
             solver, used_routes, previous_value, current_round
         )
@@ -111,7 +113,7 @@ def config_iteration(self, warm_start):
             self.print_in_console(
                 f"=================== ROUND {current_round} ========================"
             )
-            solver = pl.getSolver(**config_iteration(self, current_round != 1))
+            solver = pl.getSolver(solver=solver_name, **config_iteration(self, current_round != 1))
 
             used_routes, previous_value = self.solve_one_iteration(
                 solver, used_routes, previous_value, current_round