diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..8dd399a --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d319354 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,93 @@ +name: Continuous Integration +on: # yamllint disable-line rule:truthy + push: + branches: + - main + pull_request: + branches: ["**"] + +jobs: + python-ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Python 3 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install ruff + run: pip install ruff + - name: Run ruff + run: ruff --check . + python-mypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Python 3 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: mypy + uses: pre-commit/action@v3.0.0 + with: + extra_args: mypy --all-files + lint-requirements-txt: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: lint + uses: pre-commit/action@v3.0.0 + with: + extra_args: requirements-txt-fixer --all-files + lint-markdown: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Node + uses: actions/setup-node@v3 + with: + node-version: 18.x + - name: lint + uses: pre-commit/action@v3.0.0 + with: + extra_args: markdownlint-cli2 --all-files + lint-yaml: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: lint + uses: pre-commit/action@v3.0.0 + with: + extra_args: yamllint --all-files + lint-json: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: lint + uses: pre-commit/action@v3.0.0 + with: + extra_args: check-json --all-files + lint-shell-script: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: lint + uses: pre-commit/action@v3.0.0 + with: + extra_args: shellcheck --all-files + lint-trailing-whitespace: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: lint + uses: pre-commit/action@v3.0.0 + with: + extra_args: trailing-whitespace --all-files + lint-eof-newline: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: lint + uses: pre-commit/action@v3.0.0 + with: + extra_args: end-of-file-fixer --all-files diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..168d102 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,26 @@ +name: Release workflow + +on: + push: + tags: + - "v[0123456789].*" + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: build + run: | + python -m pip install --upgrade build hatch + python -m hatch version "${GITHUB_REF_NAME}" + python -m build + - name: publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d939240 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__ +build +llments.egg-info +.vscode \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..053b197 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.5 + hooks: + - id: ruff + - id: ruff-format + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.6.1' + hooks: + - id: mypy + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.10.0 + hooks: + - id: markdownlint-cli2 + - repo: https://github.com/adrienverge/yamllint + rev: v1.33.0 + hooks: + - id: yamllint + - repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.9.0 + hooks: + - id: shellcheck + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + - id: check-json + - id: requirements-txt-fixer + - id: trailing-whitespace diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..040e1ad --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) Carnegie Mellon University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b31b564 --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# llments + +## Development Information + +```bash +pip install . +pre-commit install +``` diff --git a/llments/__init__.py b/llments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/llments/eval/__init__.py b/llments/eval/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/llments/eval/eval.py b/llments/eval/eval.py new file mode 100644 index 0000000..aa3e2a1 --- /dev/null +++ b/llments/eval/eval.py @@ -0,0 +1,38 @@ + +import abc +import dataclasses + + +@abc.abstract +class PairwiseEvaluator: + """A class that defines an evaluation function, assessing a hypothesized string.""" + + @abc.abstractmethod + def evaluate(self, hyp: str, ref: str) -> float: + """Returns an evaluation score between 0 and 1 for two strings. + + Args: + hyp: The hypothesized string (e.g. a system output). + ref: The reference string (e.g. a gold-standard output). + """ + ... + + +@dataclasses.dataclass +class EvaluatorMetadata: + ... + + +@abc.abstract +class GeneralEvaluator: + """A class that defines an evaluation function, assessing a hypothesized string.""" + + @abc.abstractmethod + def evaluate(self, hyp: str, ref: EvaluatorMetadata) -> float: + """Returns an evaluation score between 0 and 1 for two strings. + + Args: + hyp: The hypothesized string (e.g. a system output). + ref: The reference string (e.g. a gold-standard output). + """ + ... diff --git a/llments/lm/__init__.py b/llments/lm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/llments/lm/lm.py b/llments/lm/lm.py new file mode 100644 index 0000000..ad4f186 --- /dev/null +++ b/llments/lm/lm.py @@ -0,0 +1,4 @@ + +class LanguageModel: + + \ No newline at end of file diff --git a/llments/lm/loader.py b/llments/lm/loader.py new file mode 100644 index 0000000..ef588c6 --- /dev/null +++ b/llments/lm/loader.py @@ -0,0 +1,15 @@ + + +from llments.lm.lm import LanguageModel + + +def load_from_spec_file(spec_file: str) -> LanguageModel: + """Load a language model from a specification file. + + Args: + spec_file: The path to the specification file. + + Returns: + A language model. + """ + raise NotImplementedError diff --git a/llments/lm/loader_test.py b/llments/lm/loader_test.py new file mode 100644 index 0000000..dd8fbf4 --- /dev/null +++ b/llments/lm/loader_test.py @@ -0,0 +1,4 @@ + +def test_load_from_spec_file(): + """Test that load_from_specification_file() loads a language model.""" + raise NotImplementedError diff --git a/llments/version.py b/llments/version.py new file mode 100644 index 0000000..974f94b --- /dev/null +++ b/llments/version.py @@ -0,0 +1 @@ +VERSION = "0.0.0a1" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..db5d2cd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "llments" +authors = [ + {name = "Graham Neubig", email = "neubig@gmail.com"}, +] +description = "llments" +readme = "README.md" +requires-python = ">=3.10" +license = {text = "MIT"} +classifiers = [ + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [] +dynamic = ["version"] + +[project.optional-dependencies] +dev = [ + "pre-commit" +] + +[tool.setuptools.dynamic] +version = {attr = "llments.version.VERSION"}