Merge pull request #11 from edx/bmedx/coverage_reporting

Add Django coverage reporting and threshold enforcement

.annotations_sample

@@ -1,6 +1,7 @@
 source_path: ../
 report_path: reports
 safelist_path: .annotation_safe_list.yml
+coverage_target: 50.0
 annotations:
     ".. no_pii::":
     ".. ignored::":

code_annotations/__init__.py

@@ -4,4 +4,4 @@
 
 from __future__ import absolute_import, unicode_literals
 
-__version__ = '0.1.0'
+__version__ = '0.2.0'

code_annotations/base.py

@@ -48,6 +48,18 @@ def __init__(self, config_file_path, report_path_override, verbosity, source_pat
         self.safelist_path = raw_config['safelist_path']
         self.extensions = raw_config['extensions']
 
+        try:
+            self.coverage_target = float(raw_config['coverage_target'])
+        except (TypeError, ValueError):
+            raise ConfigurationException(
+                'Coverage target must be a number between 0 and 100 not "{}".'.format(raw_config['coverage_target'])
+            )
+
+        if self.coverage_target < 0.0 or self.coverage_target > 100.0:
+            raise ConfigurationException(
+                'Invalid coverage target. {} is not between 0 and 100.'.format(self.coverage_target)
+            )
+
         self.verbosity = verbosity
         self.echo.set_verbosity(verbosity)
 
@@ -60,9 +72,6 @@ def __init__(self, config_file_path, report_path_override, verbosity, source_pat
         self._configure_annotations(raw_config)
         self._configure_extensions()
 
-        self.echo.echo_v("Configuration:")
-        self.echo.echo_v(str(self))
-
     def _check_raw_config_keys(self, raw_config):
         """
         Validate that all required keys exist in the configuration file.
@@ -74,7 +83,7 @@ def _check_raw_config_keys(self, raw_config):
             ConfigurationException on any missing keys
         """
         errors = []
-        for k in ('report_path', 'source_path', 'safelist_path', 'annotations', 'extensions'):
+        for k in ('report_path', 'source_path', 'safelist_path', 'annotations', 'extensions', 'coverage_target'):
             if k not in raw_config:
                 errors.append(k)
 
@@ -365,6 +374,9 @@ def check_results(self, all_results):
 
         Args:
             all_results: Dict of annotations found in search()
+
+        Returns:
+            Boolean indicating whether or not any errors were found
         """
         if self.config.verbosity >= 2:
             pprint.pprint(all_results, indent=3)
@@ -437,6 +449,8 @@ def check_results(self, all_results):
             if current_group:
                 self.errors.append('File finished with an incomplete group {}!'.format(current_group))
 
+        return False if self.errors else True
+
     def _add_annotation_error(self, annotation, message):
         """
         Add an error message to self.errors, formatted nicely.

code_annotations/cli.py

@@ -2,6 +2,7 @@
 Command line interface for code annotation tools.
 """
 import datetime
+import traceback
 
 import click
 
@@ -42,7 +43,17 @@ def entry_point():
 @click.option('-v', '--verbosity', count=True, help='Verbosity level (-v through -vvv)')
 @click.option('--lint/--no_lint', help='Enable or disable linting checks', default=False, show_default=True)
 @click.option('--report/--no_report', help='Enable or disable writing the report', default=False, show_default=True)
-def django_find_annotations(config_file, seed_safelist, list_local_models, report_path, verbosity, lint, report):
+@click.option('--coverage/--no_coverage', help='Enable or disable coverage checks', default=False, show_default=True)
+def django_find_annotations(
+        config_file,
+        seed_safelist,
+        list_local_models,
+        report_path,
+        verbosity,
+        lint,
+        report,
+        coverage
+):
     """
     Subcommand for dealing with annotations in Django models.
     """
@@ -57,24 +68,29 @@ def django_find_annotations(config_file, seed_safelist, list_local_models, repor
         if list_local_models:
             searcher.list_local_models()
 
-        if lint or report:
+        if lint or report or coverage:
             annotated_models = searcher.search()
 
             if lint:
                 click.echo("Performing linting checks...")
 
                 # Check grouping and choices
-                searcher.check_results(annotated_models)
-
-                # If there are any errors, do not generate the report
-                if searcher.errors:
+                if not searcher.check_results(annotated_models):
                     click.secho("\nSearch failed due to linting errors!", fg="red")
                     click.secho("{} errors:".format(len(searcher.errors)), fg="red")
                     click.secho("---------------------------------", fg="red")
                     click.echo("\n".join(searcher.errors))
+                    # If there are any errors, do not continue
                     exit(-1)
                 click.echo("Linting passed without errors.")
 
+            if coverage:
+                if not searcher.check_coverage():
+                    # If there are any errors, do not continue
+                    exit(-1)
+
+                click.echo("Coverage passed without errors.")
+
             if report:
                 searcher.report(annotated_models)
 
@@ -85,7 +101,9 @@ def django_find_annotations(config_file, seed_safelist, list_local_models, repor
 
             elapsed = datetime.datetime.now() - start_time
             click.echo("Search found {} annotations in {}.".format(annotation_count, elapsed))
+
     except Exception as exc:  # pylint: disable=broad-except
+        click.echo(traceback.print_exc())
         fail(str(exc))
 
 
@@ -151,4 +169,5 @@ def static_find_annotations(config_file, source_path, report_path, verbosity, li
         click.echo("Search found {} annotations in {}.".format(annotation_count, elapsed))
 
     except Exception as exc:  # pylint: disable=broad-except
+        click.echo(traceback.print_exc())
         fail(str(exc))