From 7b54512230a0e08a59e955b4892f47033afa3150 Mon Sep 17 00:00:00 2001 From: stephanbreimann Date: Tue, 23 Jul 2024 19:32:38 +0200 Subject: [PATCH] Update project files (from AAanalysis) --- docs/project_cheat_sheet.md | 14 +++- docs/source/conf.py | 6 +- docs/source/create_notebooks_docs.py | 57 +++++++++++--- docs/source/create_tables_doc.py | 14 ++-- docs/source/index/badges.rst | 107 +++++++++++++-------------- pyproject.toml | 35 +++++++-- 6 files changed, 146 insertions(+), 87 deletions(-) diff --git a/docs/project_cheat_sheet.md b/docs/project_cheat_sheet.md index f1acc71..233a130 100644 --- a/docs/project_cheat_sheet.md +++ b/docs/project_cheat_sheet.md @@ -106,8 +106,18 @@ Testing frameworks validate that your code works as intended. 1. `pytest`: Run all tests. 2. `pytest `: Run specific tests. 3. `pytest -k `: Run specific test. -4. `pytest --cov=`: Generate coverage. -5. `pytest --fixtures`: Show fixtures. +4. `pytest --durations=0`: Show all tests sorted by duration. +5. `pytest -x`: Stop after the first failure. +6. `pytest --maxfail=`: Stop after the specified number of failures. +7. `pytest --lf`: Run only the tests that failed at the last run. +8. `pytest --ff`: Run the last failures first, then all tests. +9. `pytest --cov`: Generate coverage report for the entire codebase. +10. `pytest --cov=`: Generate coverage. +11. `pytest --cov-report=`: Specify the type of coverage report (e.g., html, xml, term). +12. `pytest -v`: Increase verbosity of the output. +13. `pytest -q`: Decrease verbosity of the output. +14. `pytest --tb=short`: Display a shorter traceback format. +15. `pytest --disable-warnings`: Disable warnings display during test run ### unittest diff --git a/docs/source/conf.py b/docs/source/conf.py index bdbfc9d..7beb3c3 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,8 +12,10 @@ # Create notebooks rst and table rst first from create_tables_doc import generate_table_rst -from create_notebooks_docs import export_notebooks_to_rst -processed_notebooks = export_notebooks_to_rst() +from create_notebooks_docs import export_example_notebooks_to_rst, export_tutorial_notebooks_to_rst + +export_tutorial_notebooks_to_rst() +export_example_notebooks_to_rst() generate_table_rst() # -- Path and Platform setup -------------------------------------------------- diff --git a/docs/source/create_notebooks_docs.py b/docs/source/create_notebooks_docs.py index 2941e1d..62c5c11 100644 --- a/docs/source/create_notebooks_docs.py +++ b/docs/source/create_notebooks_docs.py @@ -8,7 +8,7 @@ Procedure: 1. Ensure the Jupyter tutorials you want to include in the documentation are fully executed and saved with their outputs. 2. Run this script to convert these tutorials into RST format. (automatically in conf.py) -3. Include the generated RST files in the Sphinx documentation's toctree. +3. Include the generated RST files in the Sphinx documentation's toctree (Notebooks from examples are excluded). Before running this script, ensure the project is in 'editable' mode to maintain consistency across documentation: - If using `poetry`: @@ -31,21 +31,30 @@ SEP = os.sep FOLDER_PROJECT = str(Path(__file__).parent.parent.parent) + SEP FOLDER_SOURCE = os.path.dirname(os.path.abspath(__file__)) + SEP -FOLDER_NOTEBOOKS = FOLDER_PROJECT + "tutorials" + SEP +FOLDER_TUTORIALS = FOLDER_PROJECT + "tutorials" + SEP FOLDER_GENERATED_RST = FOLDER_SOURCE + "generated" + SEP # Saving .rst directly in 'generated' +FOLDER_EXAMPLES = FOLDER_PROJECT + "examples" + SEP +FOLDER_EXAMPLES_RST = FOLDER_GENERATED_RST + "examples" + SEP +FOLDER_IMAGE = FOLDER_GENERATED_RST + "images" + SEP + LIST_EXCLUDE = [] # Helper functions class CustomPreprocessor(Preprocessor): + """Class for notebook preprocessing""" + def __init__(self, notebook_name='default', in_examples=False, **kwargs): + super().__init__(**kwargs) + self.notebook_name = notebook_name + self.in_examples=in_examples + def preprocess(self, nb, resources): - """ - Rename image resources and update the notebook cells accordingly. - """ + """Rename image resources and update the notebook cells accordingly.""" + # Extract the base name of the notebook file output_items = list(resources['outputs'].items()) for idx, (output_name, output_content) in enumerate(output_items): - # Formulate the new name - new_name = f"NOTEBOOK_{idx + 1}_{output_name}" + # Formulate the new name using the notebook name + new_name = f"{self.notebook_name}_{idx + 1}_{output_name}" # Modify resources['outputs'] resources['outputs'][new_name] = resources['outputs'].pop(output_name) # Update the notebook cell outputs @@ -60,22 +69,24 @@ def preprocess(self, nb, resources): # Prevent overriding old_name = filename["image/png"] if old_name.split("output")[1] == new_name.split("output")[1]: - filename['image/png'] = str(new_name) + new_file_name = "examples" + SEP + str(new_name) if self.in_examples else str(new_name) + filename['image/png'] = new_file_name break return nb, resources # Main function -def export_notebooks_to_rst(): +def export_tutorial_notebooks_to_rst(): """Export Jupyter tutorials to RST without execution.""" - for filename in os.listdir(FOLDER_NOTEBOOKS): + for filename in os.listdir(FOLDER_TUTORIALS): if filename.endswith('.ipynb') and filename not in LIST_EXCLUDE: - full_path = os.path.join(FOLDER_NOTEBOOKS, filename) + notebook_name = filename.replace('.ipynb', '') + full_path = os.path.join(FOLDER_TUTORIALS, filename) # Load the notebook with open(full_path, 'r') as f: notebook = nbformat.read(f, as_version=4) # Convert to notebook to RST - custom_preprocessor = CustomPreprocessor() + custom_preprocessor = CustomPreprocessor(notebook_name=notebook_name) rst_exporter = nbconvert.RSTExporter(preprocessors=[custom_preprocessor]) output, resources = rst_exporter.from_notebook_node(notebook) # Write the RST and any accompanying files (like images) @@ -83,3 +94,25 @@ def export_notebooks_to_rst(): writer.write(output, resources, notebook_name=filename.replace('.ipynb', '')) +def export_example_notebooks_to_rst(): + """Export Jupyter examples to RST without execution.""" + for root, dirs, files in os.walk(FOLDER_EXAMPLES): + for filename in files: + if filename.endswith('.ipynb') and filename not in LIST_EXCLUDE: + notebook_name = filename.replace('.ipynb', '') + full_path = os.path.join(root, filename) + #print(f"Processing file: {full_path}") # Debug print + try: + # Load the notebook + with open(full_path, 'r') as f: + notebook = nbformat.read(f, as_version=4) + # Convert to notebook to RST + custom_preprocessor = CustomPreprocessor(notebook_name=notebook_name, in_examples=True) + rst_exporter = nbconvert.RSTExporter(preprocessors=[custom_preprocessor]) + output, resources = rst_exporter.from_notebook_node(notebook) + # Write the RST and any accompanying files (like images) + writer = FilesWriter(build_directory=FOLDER_EXAMPLES_RST) + writer.write(output, resources, notebook_name=notebook_name) + except FileNotFoundError as e: + print(f"Error processing file: {full_path}. Error: {e}") + diff --git a/docs/source/create_tables_doc.py b/docs/source/create_tables_doc.py index 2c79554..f299b51 100644 --- a/docs/source/create_tables_doc.py +++ b/docs/source/create_tables_doc.py @@ -1,3 +1,5 @@ +"""This is a script to convert csv/excel files into RST files """ + import os import pandas as pd import re @@ -23,16 +25,11 @@ STR_REMOVE = "_XXX" # Check with tables_template.rst for consistency STR_ADD_TABLE = "ADD-TABLE" -EXCLUDE_FROM_REF_CHECK = ["t1_omics_fields", - "t2_quantification_methods", - "t4_omics_analysis_tools", - "t5_omics_post_analysis_tools", - "t6_enrichment_tools"] - +EXCLUDE_FROM_REF_CHECK = ["t3a_aaontology_categories", + "t3b_aaontology_subcategories"] # Helper Functions def _f_xlsx(on=True, file=None, ending=".xlsx"): - """""" if on: if ending not in file: return file.split(".")[0] + ending @@ -52,7 +49,6 @@ def _check_references(table_name=None, table_refs=None, list_refs=None): def _check_tables(list_tables): - """""" f = lambda x: _f_xlsx(on=False, file=x) if list_tables != LIST_TABLES: list_missing_map = [f(x) for x in list_tables if x not in list_tables] @@ -78,11 +74,13 @@ def _convert_excel_to_rst(df): else: new_row.append(str(val)) rst_output += " * - " + "\n - ".join(new_row) + "\n" + rst_output += "\n" # Add a blank line after each table for better spacing. return rst_output # Main Functionality def generate_table_rst(): + """Generate rst tables from csv and excel""" # Read the existing references with open(FILE_REF, 'r') as f: list_refs = f.read() diff --git a/docs/source/index/badges.rst b/docs/source/index/badges.rst index e51f249..05d8377 100755 --- a/docs/source/index/badges.rst +++ b/docs/source/index/badges.rst @@ -1,66 +1,63 @@ -.. Developer Notes: - Comments explain how to set up each badge and its purpose. Please update badges in README.rst and vice versa - -.. Group 1: Build and Test Status -.. -------------------------------- -.. Build Status Badge -.. To set up this badge, go to your GitHub Actions tab and find the relevant workflow. -.. Click on "Create status badge" and copy the Markdown or RST code. -.. image:: https://github.com/breimanntools/xomics/workflows/Build/badge.svg - :target: https://github.com/breimanntools/xomics/actions - :alt: Build Status - -.. Python-check Badge -.. This badge reflects the status of your Python code checks if set up in GitHub Actions. -.. image:: https://github.com/breimanntools/xomics/workflows/Python-check/badge.svg - :target: https://github.com/breimanntools/xomics/actions - :alt: Python-check - -.. Group 2: Package and Version Information -.. ---------------------------------------- -.. PyPI - Status Badge -.. This badge automatically reflects the package status on PyPI. -.. image:: https://img.shields.io/pypi/status/xomics.svg +.. + Developer Notes: + Please make sure that badges in badges.rst (Read The Docs) + and README.rst (GitHub) are the same. + + +.. Group 1: Package badges +.. |PyPI Status| image:: https://img.shields.io/pypi/status/xomicss.svg :target: https://pypi.org/project/xomics/ :alt: PyPI - Status -.. Supported Python Versions Badge -.. This badge shows the Python versions your package supports. -.. Update the badge URL to reflect the Python versions specified in your `pyproject.toml`. -.. image:: https://img.shields.io/pypi/pyversions/xomics.svg +.. |PyPI Version| image:: https://img.shields.io/pypi/v/xomicss.svg :target: https://pypi.python.org/pypi/xomics - :alt: Supported Python Versions + :alt: PyPI - Package Version -.. PyPI - Package Version Badge -.. This badge automatically shows the latest package version available on PyPI. -.. image:: https://img.shields.io/pypi/v/xomics.svg +.. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/xomicss.svg :target: https://pypi.python.org/pypi/xomics - :alt: PyPI - Package Version + :alt: Supported Python Versions -.. Conda - Package Version Badge -.. This badge shows the latest package version available on conda-forge. -.. image:: https://anaconda.org/conda-forge/xomics/badges/version.svg - :target: https://anaconda.org/conda-forge/xomics - :alt: Conda - Package Version +.. |Downloads| image:: https://pepy.tech/badge/xomicss + :target: https://pepy.tech/project/xomicss + :alt: Downloads + +.. |License| image:: https://img.shields.io/github/license/breimanntools/xomicss.svg + :target: https://github.com/breimanntools/xomicss/blob/master/LICENSE + :alt: License -.. Group 3: Documentation and Code Quality -.. --------------------------------------- -.. Documentation Status Badge -.. This badge reflects the build status of your latest documentation on ReadTheDocs. -.. image:: https://readthedocs.org/projects/xomics/badge/?version=latest - :target: https://xomics.readthedocs.io/en/latest/?badge=latest +.. Group 2: Testing badges +.. |Unit Tests| image:: https://github.com/breimanntools/xomicss/actions/workflows/main.yml/badge.svg + :target: https://github.com/breimanntools/xomicss/actions/workflows/main.yml + :alt: CI/CD Pipeline + +.. |CodeQL| image:: https://github.com/breimanntools/xomicss/actions/workflows/codeql_analysis.yml/badge.svg + :target: https://github.com/breimanntools/xomicss/actions/workflows/codeql_analysis.yml + :alt: CodeQL + +.. |Codecov| image:: https://codecov.io/gh/breimanntools/xomicss/branch/master/graph/badge.svg + :target: https://codecov.io/gh/breimanntools/xomicss + :alt: Codecov + +.. |Documentation Status| image:: https://readthedocs.org/projects/xomicss/badge/?version=latest + :target: https://xomicss.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -.. Group 4: License and Downloads -.. ------------------------------ -.. License Badge -.. This badge shows the license type of your project. -.. image:: https://img.shields.io/github/license/breimanntools/xomics.svg - :target: https://github.com/breimanntools/xomics/blob/master/LICENSE - :alt: License -.. Downloads Badge -.. This badge shows the number of downloads from PyPI. -.. image:: https://pepy.tech/badge/xomics - :target: https://pepy.tech/project/xomics - :alt: Downloads +.. Group 3: Potential badges for future +.. |Conda Version| image:: https://anaconda.org/conda-forge/xomicss/badges/version.svg + :target: https://anaconda.org/conda-forge/xomicss + :alt: Conda - Package Version + + +.. + Missing badges + |Conda Version| + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - **Package** + - |PyPI Status| |PyPI Version| |Supported Python Versions| |Downloads| |License| + * - **Testing** + - |Unit Tests| |CodeQL| |Codecov| |Documentation Status| diff --git a/pyproject.toml b/pyproject.toml index 3085c6f..4e04ccc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,15 @@ [build-system] -requires = ["poetry-core"] +requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] name = "xomics" version = "0.2.0" description = "Python framework for eXplainable Omics analysis" -authors = [ - "Stephan Breimann ", - "Clarissa Rienaecker "] -license = "MIT" +authors = ["Stephan Breimann "] +license = "BSD-3-Clause" readme = "README.rst" +homepage = "https://xomics.readthedocs.io" # Add classifiers to provide more details about the package (used by PyPI) classifiers = [ @@ -18,6 +17,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Natural Language :: English", "Intended Audience :: Science/Research", "Intended Audience :: Developers", @@ -26,6 +26,9 @@ classifiers = [ "Topic :: Scientific/Engineering :: Bio-Informatics", "License :: OSI Approved :: MIT License", "Development Status :: 3 - Alpha", + # 1 - Planning (The project is still in the planning phase) + # 2 - Pre-Alpha (Initial development, not yet functional) + # 3 - Alpha (Early development stage, not feature-complete, may be unstable, intended for early adopters and testers) # 4 - Beta (feature complete, may need minor improvements # 5 - Prodcution/Stable (softaware is stable & deployed) # 6 - Mature (Stable & in production for long time) @@ -93,8 +96,8 @@ adjustText = "0.8" "Repository" = "https://github.com/breimanntools/xomics" "Documentation" = "https://xomics.readthedocs.io" -# If you use a tool for linting or formatting, you can add its configurations here. -# For example, if you use `black` for formatting: + +# Configuration for black (code formatter) [tool.black] line-length = 88 exclude = ''' @@ -104,8 +107,24 @@ exclude = ''' | \.eggs | \.mypy_cache | \.pytest_cache - | \__pycache__ + | __pycache__ | build | dist )/ ''' + +# Configuration for isort (import sorting) +[tool.isort] +profile = "black" +line_length = 88 + +# Configuration for flake8 (linting) +[tool.flake8] +max-line-length = 88 +exclude = ''' + .git, + __pycache__, + build, + dist, + venv +'''