sciknoworg · HamedBabaei · Dec 17, 2025 · May 28, 2025 · May 28, 2025 · May 28, 2025
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
@@ -0,0 +1,39 @@
+name: Publish Python Package
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build-and-publish:
+    name: Build and Publish
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "export PATH=\"$HOME/.local/bin:$PATH\"" >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --no-ansi
+
+      - name: Build the package
+        run: poetry build
+
+      - name: Configure Poetry for PyPI
+        run: |
+          poetry config pypi-token.pypi ${{ secrets.TWINE_API_TOKEN }}
+
+      - name: Publish to PyPI
+        run: |
+          poetry publish --no-interaction --no-ansi
diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml
@@ -0,0 +1,38 @@
+name: Test YESciEval Package
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [3.10.x, 3.11.x, 3.12.x, 3.13.x]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Configure Poetry and install dependencies
+        run: |
+          poetry config virtualenvs.create false
+          poetry install --no-interaction --with dev
+
+      - name: Run tests
+        run: |
+          poetry run pytest
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,8 @@ __pycache__/
 *.py[cod]
 *$py.class
 .idea/
+assets/models
+trash/
 # C extensions
 *.so
 trashdir/

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,11 @@
+## Changelog
+
+### v0.2.0 (May 30, 2025)
+- Add custom judge module.
+- Add documentation.
+- Bug fixing.
+- Add readme.
+- Add test CI/CD
+
+### v0.1.0 (May 30, 2025)
+- First version of YESciEval
diff --git a/LICENSE b/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2025 XXX
+Copyright (c) 2025 SciKnowOrg
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -1,28 +1,116 @@
 <div align="center">
-  <img src="images/logo.png" width="50%" height="30%"/>
+     <img src="https://raw.githubusercontent.com/sciknoworg/YESciEval/main/images/logo.png" alt="OntoLearner Logo" width="500"/>
 </div>
 
 <div align="center">
+ <a href="https://badge.fury.io/py/YESciEval"><img src="https://badge.fury.io/py/YESciEval.svg" alt="PyPI version"></a>
+ <a href="https://huggingface.co/collections/SciKnowOrg/yescieval-judges-6839d27aac00da416d25ee66"><img src="https://img.shields.io/badge/huggingface-YESciEval-blue?logo=huggingface" alt="YESciEval HF"></a>
+ <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"></a>
+ <a href="https://yescieval.readthedocs.io/"><img src="https://app.readthedocs.org/projects/yescieval/badge/" alt="Documentation Status"></a>
 
 
-[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
-[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+</div>
 
+Large Language Models (LLMs) have become pivotal in powering scientific question-answering across modern search engines, yet their evaluation robustness remains largely underexplored. To address this gap, we introduce **YESciEval** — an open-source framework that leverages fine-grained rubric-based assessments combined with reinforcement learning to reduce optimism bias in LLM evaluators.
 
-</div>
+YESciEval provides a comprehensive library for evaluating the quality of synthesized scientific answers using predefined rubrics and sophisticated LLM-based judgment models. This framework enables you to assess answers on key criteria by utilizing pretrained judges and parsing LLM outputs into structured JSON formats for detailed analysis.
+
+
+## 🧪 Installation
+You can install ``YESciEval`` from PyPI using pip:
+
+```bash
+pip install yescieval
+```
+Next, verify the installation:
+```python
+import yescieval
+
+print(yescieval.__version__)
+```
+
+## 🔗  Essential Resources
+
+Specialized Judges within YESciEval are:
+
+| Judge      | Domain                             | Dataset Used                                                                                    | 🤗 Hugging Face                                                                                  |
+|----------------|------------------------------------|-------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------|
+| **Ask Judge**  | Multidisciplinary (33 disciplines) | [ORKGSyn (Open Research Knowledge Graph)](https://data.uni-hannover.de/dataset/yescieval-corpus) | [SciKnowOrg/YESciEval-ASK-Llama-3.1-8B](https://huggingface.co/SciKnowOrg/YESciEval-ASK-Llama-3.1-8B)       |
+| **BioASQ Judge**| Biomedical                         | [BioASQ](https://data.uni-hannover.de/dataset/yescieval-corpus)                                                                             | [SciKnowOrg/YESciEval-BioASQ-Llama-3.1-8B](https://huggingface.co/SciKnowOrg/YESciEval-BioASQ-Llama-3.1-8B) |
+
+
+For further information dive into YESciEval's extensive documentation to explore its models and usage at **[📚 YESciEval Documentation](https://yescieval.readthedocs.io/)**.    
+## 🚀 Quick Tour
+
+Get started with YESciEval in just a few lines of code. This guide demonstrates how to initialize inputs, load judge, and initiate rubric for evaluation of the answer.
+
+
+```python
+from yescieval import Readability, AutoJudge
+
+# Sample papers
+papers = {
+    "A Study on AI": "This paper discusses recent advances in artificial intelligence, including deep learning.",
+    "Machine Learning Basics": "An overview of supervised learning methods such as decision trees and SVMs.",
+    "Neural Networks Explained": "Explains backpropagation and gradient descent for training networks.",
+    "Ethics in AI": "Explores ethical concerns in automated decision-making systems.",
+    "Applications of AI in Healthcare": "Details how AI improves diagnostics and personalized medicine."
+}
+
+# Question and synthesized answer
+question = "How is AI used in modern healthcare systems?"
+answer = (
+    "AI is being used in healthcare for diagnosing diseases, predicting patient outcomes, "
+    "and assisting in treatment planning. It also supports personalized medicine and medical imaging."
+)
+
+# Step 1: Create a rubric
+rubric = Readability(papers=papers, question=question, answer=answer)
+
+# Step 2: Load a judge model (Ask Judge by default)
+judge = AutoJudge()
+judge.from_pretrained(
+    model_id="SciKnowOrg/YESciEval-ASK-Llama-3.1-8B",
+    token="your_huggingface_token",
+)
+
+# Step 3: Evaluate the answer
+result = judge.evaluate(rubric=rubric)
+print("Raw Evaluation Output:")
+print(result)
+```
+
+Judges within YESciEval are defined as follows:
 
+| Class Name       | Description                                                                                  |
+| ---------------- |----------------------------------------------------------------------------------------------|
+| `AutoJudge`      | Base class for loading and running evaluation models with PEFT adapters.                     |
+| `AskAutoJudge`   | Multidisciplinary judge tuned on the ORKGSyn dataset from the Open Research Knowledge Graph. |
+| `BioASQAutoJudge` | Biomedical domain judge tuned on the BioASQ dataset from the BioASQ challenge.               |
+| `CustomAutoJudge`| Custom LLM that can be used as a judge within YESciEval rubrics                              |
 
-## 📋 What is the YESciEval?
+A total of nine evaluation rubrics were defined as part of the YESciEval test framework and can be used via ``yescieval``. Following simple example shows how to import rubrics in your code:
 
+```python
+from yescieval import Informativeness, Correctness, Completeness, 
+                      Coherence, Relevancy, Integration, 
+                      Cohesion, Readability, Conciseness
+```
 
-Large Language Models (LLMs) drive scientific question-answering on modern search engines, yet their evaluation robustness remains underexplored. We introduce **YESciEval**, an open-source framework that combines fine-grained rubric-based assessment with reinforcement learning to mitigate optimism bias in LLM evaluators. The framework is presented as f ollows:
+A complete list of rubrics are available at YESciEval [📚 Rubrics](https://yescieval.readthedocs.io/rubrics.html) page.
 
+## 💡 Acknowledgements
 
-We release multidisciplinary scienceQ&A datasets, including adversarial variants, with evaluation scores from multiple LLMs. Independent of proprietary models and human feedback, our approach enables scalable, cost-free evaluation. By advancing reliable LLM-as-a-judge models, this work supports AI alignment and fosters robust, transparent evaluation essential for scientific inquiry and artificial general intelligence.
+If you use YESciEval in your research, please cite:
 
-## 📃 License
+```bibtex
+@article{d2025yescieval,
+      title={YESciEval: Robust LLM-as-a-Judge for Scientific Question Answering},
+      author={D'Souza, Jennifer and Giglou, Hamed Babaei and M{\"u}nch, Quentin},
+      journal={arXiv preprint arXiv:2505.14279},
+      year={2025}
+   }
+```
 
 This work is licensed under a [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT).
 

diff --git a/docs/Makefile b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/make.bat b/docs/make.bat
@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -0,0 +1,13 @@
+sphinx
+sphinx-rtd-theme
+sphinx_autodoc_typehints
+myst-parser
+sphinx_markdown_tables
+sphinx-copybutton
+sphinxcontrib-mermaid
+sphinx-panels
+sphinx-design
+sphinx-tabs
+sphinx-inline-tabs
+snowballstemmer
+sphinx_toolbox
diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css
diff --git a/docs/source/_static/custom.js b/docs/source/_static/custom.js
diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html
@@ -0,0 +1,9 @@
+{% extends "!layout.html" %}
+{% block extrahead %}
+    <link href="{{ pathto("_static/custom.css", True) }}" rel="stylesheet" type="text/css">
+{% endblock %}
+
+{# Override breadcrumbs with our custom template #}
+{% block breadcrumbs %}
+    {% include "breadcrumbs.html" %}
+{% endblock %}