Update project files (from AAanalysis)

docs/project_cheat_sheet.md

@@ -106,8 +106,18 @@ Testing frameworks validate that your code works as intended.
 1. `pytest`: Run all tests.
 2. `pytest <file>`: Run specific tests.
 3. `pytest -k <test_name>`: Run specific test.
-4. `pytest --cov=<module>`: Generate coverage.
-5. `pytest --fixtures`: Show fixtures.
+4. `pytest --durations=0`: Show all tests sorted by duration.
+5. `pytest -x`: Stop after the first failure.
+6. `pytest --maxfail=<num>`: Stop after the specified number of failures.
+7. `pytest --lf`: Run only the tests that failed at the last run.
+8. `pytest --ff`: Run the last failures first, then all tests.
+9. `pytest --cov`: Generate coverage report for the entire codebase.
+10. `pytest --cov=<module>`: Generate coverage.
+11. `pytest --cov-report=<type>`: Specify the type of coverage report (e.g., html, xml, term).
+12. `pytest -v`: Increase verbosity of the output.
+13. `pytest -q`: Decrease verbosity of the output.
+14. `pytest --tb=short`: Display a shorter traceback format.
+15. `pytest --disable-warnings`: Disable warnings display during test run
 
 ### unittest
 

docs/source/conf.py

@@ -12,8 +12,10 @@
 
 # Create notebooks rst and table rst first
 from create_tables_doc import generate_table_rst
-from create_notebooks_docs import export_notebooks_to_rst
-processed_notebooks = export_notebooks_to_rst()
+from create_notebooks_docs import export_example_notebooks_to_rst, export_tutorial_notebooks_to_rst
+
+export_tutorial_notebooks_to_rst()
+export_example_notebooks_to_rst()
 generate_table_rst()
 
 # -- Path and Platform setup --------------------------------------------------

docs/source/create_notebooks_docs.py

@@ -8,7 +8,7 @@
 Procedure:
 1. Ensure the Jupyter tutorials you want to include in the documentation are fully executed and saved with their outputs.
 2. Run this script to convert these tutorials into RST format. (automatically in conf.py)
-3. Include the generated RST files in the Sphinx documentation's toctree.
+3. Include the generated RST files in the Sphinx documentation's toctree (Notebooks from examples are excluded).
 
 Before running this script, ensure the project is in 'editable' mode to maintain consistency across documentation:
 - If using `poetry`:
@@ -31,21 +31,30 @@
 SEP = os.sep
 FOLDER_PROJECT = str(Path(__file__).parent.parent.parent) + SEP
 FOLDER_SOURCE = os.path.dirname(os.path.abspath(__file__)) + SEP
-FOLDER_NOTEBOOKS = FOLDER_PROJECT + "tutorials" + SEP
+FOLDER_TUTORIALS = FOLDER_PROJECT + "tutorials" + SEP
 FOLDER_GENERATED_RST = FOLDER_SOURCE + "generated" + SEP  # Saving .rst directly in 'generated'
+FOLDER_EXAMPLES = FOLDER_PROJECT + "examples" + SEP
+FOLDER_EXAMPLES_RST = FOLDER_GENERATED_RST + "examples" + SEP
+FOLDER_IMAGE = FOLDER_GENERATED_RST + "images" + SEP
+
 LIST_EXCLUDE = []
 
 
 # Helper functions
 class CustomPreprocessor(Preprocessor):
+    """Class for notebook preprocessing"""
+    def __init__(self, notebook_name='default', in_examples=False, **kwargs):
+        super().__init__(**kwargs)
+        self.notebook_name = notebook_name
+        self.in_examples=in_examples
+
     def preprocess(self, nb, resources):
-        """
-        Rename image resources and update the notebook cells accordingly.
-        """
+        """Rename image resources and update the notebook cells accordingly."""
+        # Extract the base name of the notebook file
         output_items = list(resources['outputs'].items())
         for idx, (output_name, output_content) in enumerate(output_items):
-            # Formulate the new name
-            new_name = f"NOTEBOOK_{idx + 1}_{output_name}"
+            # Formulate the new name using the notebook name
+            new_name = f"{self.notebook_name}_{idx + 1}_{output_name}"
             # Modify resources['outputs']
             resources['outputs'][new_name] = resources['outputs'].pop(output_name)
             # Update the notebook cell outputs
@@ -60,26 +69,50 @@ def preprocess(self, nb, resources):
                                     # Prevent overriding
                                     old_name = filename["image/png"]
                                     if old_name.split("output")[1] == new_name.split("output")[1]:
-                                        filename['image/png'] = str(new_name)
+                                        new_file_name = "examples" + SEP + str(new_name) if self.in_examples else str(new_name)
+                                        filename['image/png'] = new_file_name
                                         break
         return nb, resources
 
 
 # Main function
-def export_notebooks_to_rst():
+def export_tutorial_notebooks_to_rst():
     """Export Jupyter tutorials to RST without execution."""
-    for filename in os.listdir(FOLDER_NOTEBOOKS):
+    for filename in os.listdir(FOLDER_TUTORIALS):
         if filename.endswith('.ipynb') and filename not in LIST_EXCLUDE:
-            full_path = os.path.join(FOLDER_NOTEBOOKS, filename)
+            notebook_name = filename.replace('.ipynb', '')
+            full_path = os.path.join(FOLDER_TUTORIALS, filename)
             # Load the notebook
             with open(full_path, 'r') as f:
                 notebook = nbformat.read(f, as_version=4)
             # Convert to notebook to RST
-            custom_preprocessor = CustomPreprocessor()
+            custom_preprocessor = CustomPreprocessor(notebook_name=notebook_name)
             rst_exporter = nbconvert.RSTExporter(preprocessors=[custom_preprocessor])
             output, resources = rst_exporter.from_notebook_node(notebook)
             # Write the RST and any accompanying files (like images)
             writer = FilesWriter(build_directory=FOLDER_GENERATED_RST)
             writer.write(output, resources, notebook_name=filename.replace('.ipynb', ''))
 
 
+def export_example_notebooks_to_rst():
+    """Export Jupyter examples to RST without execution."""
+    for root, dirs, files in os.walk(FOLDER_EXAMPLES):
+        for filename in files:
+            if filename.endswith('.ipynb') and filename not in LIST_EXCLUDE:
+                notebook_name = filename.replace('.ipynb', '')
+                full_path = os.path.join(root, filename)
+                #print(f"Processing file: {full_path}")  # Debug print
+                try:
+                    # Load the notebook
+                    with open(full_path, 'r') as f:
+                        notebook = nbformat.read(f, as_version=4)
+                    # Convert to notebook to RST
+                    custom_preprocessor = CustomPreprocessor(notebook_name=notebook_name, in_examples=True)
+                    rst_exporter = nbconvert.RSTExporter(preprocessors=[custom_preprocessor])
+                    output, resources = rst_exporter.from_notebook_node(notebook)
+                    # Write the RST and any accompanying files (like images)
+                    writer = FilesWriter(build_directory=FOLDER_EXAMPLES_RST)
+                    writer.write(output, resources, notebook_name=notebook_name)
+                except FileNotFoundError as e:
+                    print(f"Error processing file: {full_path}. Error: {e}")
+

docs/source/create_tables_doc.py

@@ -1,3 +1,5 @@
+"""This is a script to convert csv/excel files into RST files """
+
 import os
 import pandas as pd
 import re
@@ -23,16 +25,11 @@
 STR_REMOVE = "_XXX" # Check with tables_template.rst for consistency
 STR_ADD_TABLE = "ADD-TABLE"
 
-EXCLUDE_FROM_REF_CHECK = ["t1_omics_fields",
-                          "t2_quantification_methods",
-                          "t4_omics_analysis_tools",
-                          "t5_omics_post_analysis_tools",
-                          "t6_enrichment_tools"]
-
+EXCLUDE_FROM_REF_CHECK = ["t3a_aaontology_categories",
+                          "t3b_aaontology_subcategories"]
 
 # Helper Functions
 def _f_xlsx(on=True, file=None, ending=".xlsx"):
-    """"""
     if on:
         if ending not in file:
             return file.split(".")[0] + ending
@@ -52,7 +49,6 @@ def _check_references(table_name=None, table_refs=None, list_refs=None):
 
 
 def _check_tables(list_tables):
-    """"""
     f = lambda x: _f_xlsx(on=False, file=x)
     if list_tables != LIST_TABLES:
         list_missing_map = [f(x) for x in list_tables if x not in list_tables]
@@ -78,11 +74,13 @@ def _convert_excel_to_rst(df):
             else:
                 new_row.append(str(val))
         rst_output += "   * - " + "\n     - ".join(new_row) + "\n"
+    rst_output += "\n"  # Add a blank line after each table for better spacing.
     return rst_output
 
 
 # Main Functionality
 def generate_table_rst():
+    """Generate rst tables from csv and excel"""
     # Read the existing references
     with open(FILE_REF, 'r') as f:
         list_refs = f.read()

docs/source/index/badges.rst

@@ -1,66 +1,63 @@
-.. Developer Notes:
-    Comments explain how to set up each badge and its purpose. Please update badges in README.rst and vice versa
-
-.. Group 1: Build and Test Status
-.. --------------------------------
-.. Build Status Badge
-.. To set up this badge, go to your GitHub Actions tab and find the relevant workflow.
-.. Click on "Create status badge" and copy the Markdown or RST code.
-.. image:: https://github.com/breimanntools/xomics/workflows/Build/badge.svg
-   :target: https://github.com/breimanntools/xomics/actions
-   :alt: Build Status
-
-.. Python-check Badge
-.. This badge reflects the status of your Python code checks if set up in GitHub Actions.
-.. image:: https://github.com/breimanntools/xomics/workflows/Python-check/badge.svg
-   :target: https://github.com/breimanntools/xomics/actions
-   :alt: Python-check
-
-.. Group 2: Package and Version Information
-.. ----------------------------------------
-.. PyPI - Status Badge
-.. This badge automatically reflects the package status on PyPI.
-.. image:: https://img.shields.io/pypi/status/xomics.svg
+..
+    Developer Notes:
+    Please make sure that badges in badges.rst (Read The Docs)
+    and README.rst (GitHub) are the same.
+
+
+.. Group 1: Package badges
+.. |PyPI Status| image:: https://img.shields.io/pypi/status/xomicss.svg
    :target: https://pypi.org/project/xomics/
    :alt: PyPI - Status
 
-.. Supported Python Versions Badge
-.. This badge shows the Python versions your package supports.
-.. Update the badge URL to reflect the Python versions specified in your `pyproject.toml`.
-.. image:: https://img.shields.io/pypi/pyversions/xomics.svg
+.. |PyPI Version| image:: https://img.shields.io/pypi/v/xomicss.svg
    :target: https://pypi.python.org/pypi/xomics
-   :alt: Supported Python Versions
+   :alt: PyPI - Package Version
 
-.. PyPI - Package Version Badge
-.. This badge automatically shows the latest package version available on PyPI.
-.. image:: https://img.shields.io/pypi/v/xomics.svg
+.. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/xomicss.svg
    :target: https://pypi.python.org/pypi/xomics
-   :alt: PyPI - Package Version
+   :alt: Supported Python Versions
 
-.. Conda - Package Version Badge
-.. This badge shows the latest package version available on conda-forge.
-.. image:: https://anaconda.org/conda-forge/xomics/badges/version.svg
-   :target: https://anaconda.org/conda-forge/xomics
-   :alt: Conda - Package Version
+.. |Downloads| image:: https://pepy.tech/badge/xomicss
+   :target: https://pepy.tech/project/xomicss
+   :alt: Downloads
+
+.. |License| image:: https://img.shields.io/github/license/breimanntools/xomicss.svg
+   :target: https://github.com/breimanntools/xomicss/blob/master/LICENSE
+   :alt: License
 
-.. Group 3: Documentation and Code Quality
-.. ---------------------------------------
-.. Documentation Status Badge
-.. This badge reflects the build status of your latest documentation on ReadTheDocs.
-.. image:: https://readthedocs.org/projects/xomics/badge/?version=latest
-   :target: https://xomics.readthedocs.io/en/latest/?badge=latest
+.. Group 2: Testing badges
+.. |Unit Tests| image:: https://github.com/breimanntools/xomicss/actions/workflows/main.yml/badge.svg
+   :target: https://github.com/breimanntools/xomicss/actions/workflows/main.yml
+   :alt: CI/CD Pipeline
+
+.. |CodeQL| image:: https://github.com/breimanntools/xomicss/actions/workflows/codeql_analysis.yml/badge.svg
+   :target: https://github.com/breimanntools/xomicss/actions/workflows/codeql_analysis.yml
+   :alt: CodeQL
+
+.. |Codecov| image:: https://codecov.io/gh/breimanntools/xomicss/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/breimanntools/xomicss
+   :alt: Codecov
+
+.. |Documentation Status| image:: https://readthedocs.org/projects/xomicss/badge/?version=latest
+   :target: https://xomicss.readthedocs.io/en/latest/?badge=latest
    :alt: Documentation Status
 
-.. Group 4: License and Downloads
-.. ------------------------------
-.. License Badge
-.. This badge shows the license type of your project.
-.. image:: https://img.shields.io/github/license/breimanntools/xomics.svg
-   :target: https://github.com/breimanntools/xomics/blob/master/LICENSE
-   :alt: License
 
-.. Downloads Badge
-.. This badge shows the number of downloads from PyPI.
-.. image:: https://pepy.tech/badge/xomics
-   :target: https://pepy.tech/project/xomics
-   :alt: Downloads
+.. Group 3: Potential badges for future
+.. |Conda Version| image:: https://anaconda.org/conda-forge/xomicss/badges/version.svg
+   :target: https://anaconda.org/conda-forge/xomicss
+   :alt: Conda - Package Version
+
+
+..
+    Missing badges
+    |Conda Version|
+
+.. list-table::
+   :widths: 20 80
+   :header-rows: 1
+
+   * - **Package**
+     - |PyPI Status| |PyPI Version| |Supported Python Versions| |Downloads| |License|
+   * - **Testing**
+     - |Unit Tests| |CodeQL| |Codecov| |Documentation Status|

pyproject.toml

@@ -1,23 +1,23 @@
 [build-system]
-requires = ["poetry-core"]
+requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "xomics"
 version = "0.2.0"
 description = "Python framework for eXplainable Omics analysis"
-authors = [
-    "Stephan Breimann <stephanbreimann@gmail.de>",
-    "Clarissa Rienaecker <clarissa.rienaecker@web.de>"]
-license = "MIT"
+authors = ["Stephan Breimann <stephanbreimann@gmail.de>"]
+license = "BSD-3-Clause"
 readme = "README.rst"
+homepage = "https://xomics.readthedocs.io"
 
 # Add classifiers to provide more details about the package (used by PyPI)
 classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Natural Language :: English",
     "Intended Audience :: Science/Research",
     "Intended Audience :: Developers",
@@ -26,6 +26,9 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Bio-Informatics",
     "License :: OSI Approved :: MIT License",
     "Development Status :: 3 - Alpha",
+    # 1 - Planning (The project is still in the planning phase)
+    # 2 - Pre-Alpha (Initial development, not yet functional)
+    # 3 - Alpha (Early development stage, not feature-complete, may be unstable, intended for early adopters and testers)
     # 4 - Beta (feature complete, may need minor improvements
     # 5 - Prodcution/Stable (softaware is stable & deployed)
     # 6 - Mature (Stable & in production for long time)
@@ -93,8 +96,8 @@ adjustText = "0.8"
 "Repository" = "https://github.com/breimanntools/xomics"
 "Documentation" = "https://xomics.readthedocs.io"
 
-# If you use a tool for linting or formatting, you can add its configurations here.
-# For example, if you use `black` for formatting:
+
+# Configuration for black (code formatter)
 [tool.black]
 line-length = 88
 exclude = '''
@@ -104,8 +107,24 @@ exclude = '''
   | \.eggs
   | \.mypy_cache
   | \.pytest_cache
-  | \__pycache__
+  | __pycache__
   | build
   | dist
 )/
 '''
+
+# Configuration for isort (import sorting)
+[tool.isort]
+profile = "black"
+line_length = 88
+
+# Configuration for flake8 (linting)
+[tool.flake8]
+max-line-length = 88
+exclude = '''
+    .git,
+    __pycache__,
+    build,
+    dist,
+    venv
+'''