Add check for misplaced development dependencies (#51)

* Added a check for development dependencies
* Added fixture to speed up the CLI unit tests
* changed --ignore-directories to --exclude
* Updated the documentation

Author: Florian Maas

README.md

@@ -8,11 +8,12 @@
 
 ---
 
-_deptry_ is a command line tool to check for issues with dependencies in a poetry managed Python project. It checks for three types of issues:
+_deptry_ is a command line tool to check for issues with dependencies in a poetry managed Python project. It checks for four types of issues:
 
 - Obsolete dependencies: Dependencies which are added to your project's dependencies, but which are not used within the codebase.
-- Transitive dependencies: Packages from which code is imported, but the package (A) itself is not in your projects dependencies. Instead, another package (B) is in your list of dependencies, which depends on (A). Package (A) should be added to your project's list of dependencies.
 - Missing dependencies: Modules that are imported within your project, but no corresponding package is found in the environment.
+- Transitive dependencies: Packages from which code is imported, but the package (A) itself is not in your projects dependencies. Instead, another package (B) is in your list of dependencies, which depends on (A). Package (A) should be added to your project's list of dependencies.
+- Misplaced dependencies: Development dependencies that should be included as regular dependencies.
 
 _deptry_ detects these issue by scanning the imported modules within all Python files in 
 a directory and it's subdirectories, and comparing those to the dependencies listed in _pyproject.toml_.

deptry/cli.py

@@ -33,6 +33,11 @@
     is_flag=True,
     help="Boolean flag to specify if deptry should skip scanning the project for transitive dependencies.",
 )
+@click.option(
+    "--skip-misplaced-dev",
+    is_flag=True,
+    help="Boolean flag to specify if deptry should skip scanning the project for development dependencies that should be regular dependencies.",
+)
 @click.option(
     "--ignore-obsolete",
     "-io",
@@ -46,22 +51,29 @@
     "--ignore-missing",
     "-im",
     multiple=True,
-    help="""Modules that should never be marked as having missing dependencies, even if the matching package for the import statement cannot be found.
+    help="""Modules that should never be marked as missing dependencies, even if the matching package for the import statement cannot be found.
     Can be used multiple times. For example; `deptry . -io foo -io bar`.""",
 )
 @click.option(
     "--ignore-transitive",
     "-it",
     multiple=True,
-    help="""Modules that should never be marked as 'missing due to transitive' even though deptry determines them to be transitive.
+    help="""Dependencies that should never be marked as an issue due to it being a transitive dependency, even though deptry determines them to be transitive.
     Can be used multiple times. For example; `deptry . -it foo -io bar`.""",
 )
 @click.option(
-    "--ignore-directories",
+    "--ignore-misplaced-dev",
     "-id",
     multiple=True,
-    help="""Directories in which .py files should not be scanned for imports to determine if dependencies are obsolete, missing or transitive.
-    Defaults to ['venv','tests']. Specify multiple directories by using this flag twice, e.g. `-id .venv -id tests -id other_dir`.""",
+    help="""Modules that should never be marked as a misplaced development dependency, even though it seems to not be used solely for development purposes.
+    Can be used multiple times. For example; `deptry . -id foo -id bar`.""",
+)
+@click.option(
+    "--exclude",
+    multiple=True,
+    help="""Directories or files in which .py files should not be scanned for imports to determine if there are dependency issues.
+    Defaults to ['venv','tests']. Specify multiple directories by using this flag multiple times, e.g. `--exclude .venv --exclude tests
+    --exclude other_dir`.""",
 )
 @click.option(
     "--ignore-notebooks",
@@ -80,10 +92,12 @@ def deptry(
     ignore_obsolete: List[str],
     ignore_missing: List[str],
     ignore_transitive: List[str],
+    ignore_misplaced_dev: List[str],
     skip_obsolete: bool,
     skip_missing: bool,
     skip_transitive: bool,
-    ignore_directories: List[str],
+    skip_misplaced_dev: bool,
+    exclude: List[str],
     ignore_notebooks: bool,
     version: bool,
 ) -> None:
@@ -108,22 +122,26 @@ def deptry(
             ignore_obsolete=ignore_obsolete if ignore_obsolete else None,
             ignore_missing=ignore_missing if ignore_missing else None,
             ignore_transitive=ignore_transitive if ignore_transitive else None,
-            ignore_directories=ignore_directories if ignore_directories else None,
+            ignore_misplaced_dev=ignore_misplaced_dev if ignore_misplaced_dev else None,
+            exclude=exclude if exclude else None,
             ignore_notebooks=ignore_notebooks if ignore_notebooks else None,
             skip_obsolete=skip_obsolete if skip_obsolete else None,
             skip_missing=skip_missing if skip_missing else None,
             skip_transitive=skip_transitive if skip_transitive else None,
+            skip_misplaced_dev=skip_misplaced_dev if skip_misplaced_dev else None,
         )
 
         result = Core(
             ignore_obsolete=config.ignore_obsolete,
             ignore_missing=config.ignore_missing,
             ignore_transitive=config.ignore_transitive,
-            ignore_directories=config.ignore_directories,
+            ignore_misplaced_dev=config.ignore_misplaced_dev,
+            exclude=config.exclude,
             ignore_notebooks=config.ignore_notebooks,
             skip_obsolete=config.skip_obsolete,
             skip_missing=config.skip_missing,
             skip_transitive=config.skip_transitive,
+            skip_misplaced_dev=config.skip_misplaced_dev,
         ).run()
         issue_found = False
         if not skip_obsolete and "obsolete" in result and result["obsolete"]:
@@ -135,6 +153,9 @@ def deptry(
         if not skip_transitive and "transitive" in result and result["transitive"]:
             log_transitive_dependencies(result["transitive"])
             issue_found = True
+        if not skip_misplaced_dev and "misplaced_dev" in result and result["misplaced_dev"]:
+            log_misplaced_develop_dependencies(result["misplaced_dev"])
+            issue_found = True
 
         if issue_found:
             log_additional_info()
@@ -165,17 +186,24 @@ def log_transitive_dependencies(dependencies: List[str], sep="\n\t") -> None:
     logging.info(
         f"There are transitive dependencies that should be explicitly defined as dependencies in pyproject.toml:\n{sep}{sep.join(dependencies)}\n"
     )
+    logging.info("""They are currently imported but not specified directly as your project's dependencies.""")
+
+
+def log_misplaced_develop_dependencies(dependencies: List[str], sep="\n\t") -> None:
+    logging.info("\n-----------------------------------------------------\n")
+    logging.info(f"There are imported modules from development dependencies detected:\n{sep}{sep.join(dependencies)}\n")
     logging.info(
-        """They are currently imported but not specified directly as your project's dependencies. This issue also be caused
-by a development dependency that is found to be used within the scanned Python files."""
+        """Consider moving them to `[tool.poetry.dependencies]` in pyproject.toml. If this is not correct and the
+dependencies listed above are indeed development dependencies, it's likely that files were scanned that are only used
+for development purposes. Run `deptry -v .` to see a list of scanned files."""
     )
 
 
 def log_additional_info():
     logging.info("\n-----------------------------------------------------\n")
     logging.info(
         """Dependencies and directories can be ignored by passing additional command-line arguments. See `deptry --help` for more details.
-Alternatively, deptry can be configured through `pyproject.toml`:
+Alternatively, deptry can be configured through `pyproject.toml`. An example:
 
 ```
 [tool.deptry]
@@ -188,7 +216,7 @@ def log_additional_info():
 ignore_transitive = [
   'your-dependency'
 ]
-ignore_directories = [
+exclude = [
   '.venv', 'tests', 'docs'
 ]
 ```

deptry/config.py

@@ -8,11 +8,13 @@
     "ignore_obsolete": [],
     "ignore_missing": [],
     "ignore_transitive": [],
-    "ignore_directories": [".venv", "tests"],
+    "ignore_misplaced_dev": [],
+    "exclude": [".venv", "tests"],
     "ignore_notebooks": False,
     "skip_obsolete": False,
     "skip_missing": False,
     "skip_transitive": False,
+    "skip_misplaced_dev": False,
 }
 
 
@@ -28,45 +30,53 @@ def __init__(
         ignore_obsolete: Optional[List[str]],
         ignore_missing: Optional[List[str]],
         ignore_transitive: Optional[List[str]],
+        ignore_misplaced_dev: Optional[List[str]],
         skip_obsolete: Optional[bool],
         skip_missing: Optional[bool],
         skip_transitive: Optional[bool],
-        ignore_directories: Optional[List[str]],
+        skip_misplaced_dev: Optional[bool],
+        exclude: Optional[List[str]],
         ignore_notebooks: Optional[bool],
     ) -> None:
         self._set_defaults()
         self._override_config_with_pyproject_toml()
         self._override_config_with_cli_arguments(
-            ignore_obsolete,
-            ignore_missing,
-            ignore_transitive,
-            ignore_directories,
-            ignore_notebooks,
-            skip_obsolete,
-            skip_missing,
-            skip_transitive,
+            ignore_obsolete=ignore_obsolete,
+            ignore_missing=ignore_missing,
+            ignore_transitive=ignore_transitive,
+            ignore_misplaced_dev=ignore_misplaced_dev,
+            exclude=exclude,
+            ignore_notebooks=ignore_notebooks,
+            skip_obsolete=skip_obsolete,
+            skip_missing=skip_missing,
+            skip_transitive=skip_transitive,
+            skip_misplaced_dev=skip_misplaced_dev,
         )
 
     def _set_defaults(self) -> None:
         self.ignore_obsolete = DEFAULTS["ignore_obsolete"]
         self.ignore_missing = DEFAULTS["ignore_missing"]
         self.ignore_transitive = DEFAULTS["ignore_transitive"]
-        self.ignore_directories = DEFAULTS["ignore_directories"]
+        self.ignore_misplaced_dev = DEFAULTS["ignore_misplaced_dev"]
+        self.exclude = DEFAULTS["exclude"]
         self.ignore_notebooks = DEFAULTS["ignore_notebooks"]
         self.skip_obsolete = DEFAULTS["skip_obsolete"]
         self.skip_missing = DEFAULTS["skip_missing"]
         self.skip_transitive = DEFAULTS["skip_transitive"]
+        self.skip_misplaced_dev = DEFAULTS["skip_misplaced_dev"]
 
     def _override_config_with_pyproject_toml(self) -> None:
         pyproject_toml_config = self._read_configuration_from_pyproject_toml()
         if pyproject_toml_config:
             self._override_with_toml_argument("ignore_obsolete", List[str], pyproject_toml_config)
             self._override_with_toml_argument("ignore_missing", List[str], pyproject_toml_config)
             self._override_with_toml_argument("ignore_transitive", List[str], pyproject_toml_config)
+            self._override_with_toml_argument("ignore_misplaced_dev", List[str], pyproject_toml_config)
             self._override_with_toml_argument("skip_missing", List[str], pyproject_toml_config)
             self._override_with_toml_argument("skip_obsolete", List[str], pyproject_toml_config)
             self._override_with_toml_argument("skip_transitive", List[str], pyproject_toml_config)
-            self._override_with_toml_argument("ignore_directories", List[str], pyproject_toml_config)
+            self._override_with_toml_argument("skip_misplaced_dev", List[str], pyproject_toml_config)
+            self._override_with_toml_argument("exclude", List[str], pyproject_toml_config)
             self._override_with_toml_argument("ignore_notebooks", List[str], pyproject_toml_config)
 
     def _read_configuration_from_pyproject_toml(self) -> Optional[Dict]:
@@ -87,16 +97,18 @@ def _override_with_toml_argument(self, argument: str, expected_type: Any, pyproj
             setattr(self, argument, value)
             self._log_changed_by_pyproject_toml(argument, value)
 
-    def _override_config_with_cli_arguments(
+    def _override_config_with_cli_arguments(  # noqa
         self,
         ignore_obsolete: Optional[List[str]],
         ignore_missing: Optional[List[str]],
         ignore_transitive: Optional[List[str]],
-        ignore_directories: Optional[List[str]],
+        ignore_misplaced_dev: Optional[List[str]],
+        exclude: Optional[List[str]],
         ignore_notebooks: Optional[bool],
         skip_obsolete: Optional[bool],
         skip_missing: Optional[bool],
         skip_transitive: Optional[bool],
+        skip_misplaced_dev: Optional[bool],
     ) -> None:
 
         if ignore_obsolete:
@@ -111,6 +123,10 @@ def _override_config_with_cli_arguments(
             self.ignore_transitive = ignore_transitive
             self._log_changed_by_command_line_argument("ignore_transitive", ignore_transitive)
 
+        if ignore_misplaced_dev:
+            self.ignore_misplaced_dev = ignore_misplaced_dev
+            self._log_changed_by_command_line_argument("ignore_misplaced_dev", ignore_misplaced_dev)
+
         if skip_obsolete:
             self.skip_obsolete = skip_obsolete
             self._log_changed_by_command_line_argument("skip_obsolete", skip_obsolete)
@@ -123,9 +139,13 @@ def _override_config_with_cli_arguments(
             self.skip_transitive = skip_transitive
             self._log_changed_by_command_line_argument("skip_transitive", skip_transitive)
 
-        if ignore_directories:
-            self.ignore_directories = ignore_directories
-            self._log_changed_by_command_line_argument("ignore_directories", ignore_directories)
+        if skip_misplaced_dev:
+            self.skip_misplaced_dev = skip_misplaced_dev
+            self._log_changed_by_command_line_argument("skip_misplaced_dev", skip_misplaced_dev)
+
+        if exclude:
+            self.exclude = exclude
+            self._log_changed_by_command_line_argument("exclude", exclude)
 
         if ignore_notebooks:
             self.ignore_notebooks = ignore_notebooks

deptry/core.py

@@ -4,10 +4,11 @@
 
 from deptry.dependency_getter import DependencyGetter
 from deptry.import_parser import ImportParser
+from deptry.issue_finders.misplaced_dev import MisplacedDevDependenciesFinder
 from deptry.issue_finders.missing import MissingDependenciesFinder
 from deptry.issue_finders.obsolete import ObsoleteDependenciesFinder
 from deptry.issue_finders.transitive import TransitiveDependenciesFinder
-from deptry.module import Module
+from deptry.module import ModuleBuilder
 from deptry.python_file_finder import PythonFileFinder
 
 
@@ -17,53 +18,67 @@ def __init__(
         ignore_obsolete: List[str],
         ignore_missing: List[str],
         ignore_transitive: List[str],
+        ignore_misplaced_dev: List[str],
         skip_obsolete: bool,
         skip_missing: bool,
         skip_transitive: bool,
-        ignore_directories: List[str],
+        skip_misplaced_dev: bool,
+        exclude: List[str],
         ignore_notebooks: bool,
     ) -> None:
         self.ignore_obsolete = ignore_obsolete
         self.ignore_missing = ignore_missing
         self.ignore_transitive = ignore_transitive
-        self.ignore_directories = ignore_directories
+        self.ignore_misplaced_dev = ignore_misplaced_dev
+        self.exclude = exclude
         self.ignore_notebooks = ignore_notebooks
         self.skip_obsolete = skip_obsolete
         self.skip_missing = skip_missing
         self.skip_transitive = skip_transitive
+        self.skip_misplaced_dev = skip_misplaced_dev
         logging.debug("Running with the following configuration:")
         logging.debug(f"ignore_obsolete: {ignore_obsolete}")
         logging.debug(f"ignore_missing: {ignore_missing}")
         logging.debug(f"ignore_transitive: {ignore_transitive}")
+        logging.debug(f"ignore_misplaced_dev: {ignore_misplaced_dev}")
         logging.debug(f"skip_obsolete: {skip_obsolete}")
         logging.debug(f"skip_missing: {skip_missing}")
         logging.debug(f"skip_transitive: {skip_transitive}")
-        logging.debug(f"ignore_directories: {ignore_directories}")
+        logging.debug(f"skip_misplaced_dev {skip_misplaced_dev}")
+        logging.debug(f"exclude: {exclude}")
         logging.debug(f"ignore_notebooks: {ignore_notebooks}\n")
 
     def run(self) -> Dict:
 
         dependencies = DependencyGetter().get()
+        dev_dependencies = DependencyGetter(dev=True).get()
+
         all_python_files = PythonFileFinder(
-            ignore_directories=self.ignore_directories, ignore_notebooks=self.ignore_notebooks
+            exclude=self.exclude, ignore_notebooks=self.ignore_notebooks
         ).get_all_python_files_in(Path("."))
 
         imported_modules = ImportParser().get_imported_modules_for_list_of_files(all_python_files)
-        imported_modules = [Module(mod, dependencies) for mod in imported_modules]
-        imported_modules = [mod for mod in imported_modules if not mod.is_standard_library()]
+        imported_modules = [ModuleBuilder(mod, dependencies, dev_dependencies).build() for mod in imported_modules]
+        imported_modules = [mod for mod in imported_modules if not mod.standard_library]
 
         result = {}
         if not self.skip_obsolete:
             result["obsolete"] = ObsoleteDependenciesFinder(
-                imported_modules=imported_modules, dependencies=dependencies, list_to_ignore=self.ignore_obsolete
+                imported_modules=imported_modules, dependencies=dependencies, ignore_obsolete=self.ignore_obsolete
             ).find()
         if not self.skip_missing:
             result["missing"] = MissingDependenciesFinder(
-                imported_modules=imported_modules, dependencies=dependencies, list_to_ignore=self.ignore_missing
+                imported_modules=imported_modules, dependencies=dependencies, ignore_missing=self.ignore_missing
             ).find()
         if not self.skip_transitive:
             result["transitive"] = TransitiveDependenciesFinder(
-                imported_modules=imported_modules, dependencies=dependencies, list_to_ignore=self.ignore_transitive
+                imported_modules=imported_modules, dependencies=dependencies, ignore_transitive=self.ignore_transitive
+            ).find()
+        if not self.skip_misplaced_dev:
+            result["misplaced_dev"] = MisplacedDevDependenciesFinder(
+                imported_modules=imported_modules,
+                dependencies=dependencies,
+                ignore_misplaced_dev=self.ignore_misplaced_dev,
             ).find()
 
         return result