Initial commit

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203

.github/workflows/ci.yml

@@ -0,0 +1,93 @@
+name: Continuous Integration
+on: # yamllint disable-line rule:truthy
+  push:
+    branches:
+      - main
+  pull_request:
+    branches: ["**"]
+
+jobs:
+  python-ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Python 3
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install ruff
+        run: pip install ruff
+      - name: Run ruff
+        run: ruff --check .
+  python-mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Python 3
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: mypy
+        uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: mypy --all-files
+  lint-requirements-txt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: lint
+        uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: requirements-txt-fixer --all-files
+  lint-markdown:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+      - name: lint
+        uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: markdownlint-cli2 --all-files
+  lint-yaml:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: lint
+        uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: yamllint --all-files
+  lint-json:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: lint
+        uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: check-json --all-files
+  lint-shell-script:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: lint
+        uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: shellcheck --all-files
+  lint-trailing-whitespace:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: lint
+        uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: trailing-whitespace --all-files
+  lint-eof-newline:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: lint
+        uses: pre-commit/action@v3.0.0
+        with:
+          extra_args: end-of-file-fixer --all-files

.github/workflows/release.yml

@@ -0,0 +1,26 @@
+name: Release workflow
+
+on:
+  push:
+    tags:
+      - "v[0123456789].*"
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - name: build
+        run: |
+          python -m pip install --upgrade build hatch
+          python -m hatch version "${GITHUB_REF_NAME}"
+          python -m build
+      - name: publish
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -0,0 +1,4 @@
+__pycache__
+build
+llments.egg-info
+.vscode

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.5
+    hooks:
+      - id: ruff
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v1.6.1'
+    hooks:
+    -   id: mypy
+  - repo: https://github.com/DavidAnson/markdownlint-cli2
+    rev: v0.10.0
+    hooks:
+      - id: markdownlint-cli2
+  - repo: https://github.com/adrienverge/yamllint
+    rev: v1.33.0
+    hooks:
+      - id: yamllint
+  - repo: https://github.com/koalaman/shellcheck-precommit
+    rev: v0.9.0
+    hooks:
+      - id: shellcheck
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: check-json
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace

LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) Carnegie Mellon University
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,8 @@
+# llments
+
+## Development Information
+
+```bash
+pip install .
+pre-commit install
+```

llments/eval/eval.py

@@ -0,0 +1,38 @@
+
+import abc
+import dataclasses
+
+
+@abc.abstract
+class PairwiseEvaluator:
+    """A class that defines an evaluation function, assessing a hypothesized string."""
+
+    @abc.abstractmethod
+    def evaluate(self, hyp: str, ref: str) -> float:
+        """Returns an evaluation score between 0 and 1 for two strings.
+
+        Args:
+            hyp: The hypothesized string (e.g. a system output).
+            ref: The reference string (e.g. a gold-standard output).
+        """
+        ...
+
+
+@dataclasses.dataclass
+class EvaluatorMetadata:
+    ...
+
+
+@abc.abstract
+class GeneralEvaluator:
+    """A class that defines an evaluation function, assessing a hypothesized string."""
+
+    @abc.abstractmethod
+    def evaluate(self, hyp: str, ref: EvaluatorMetadata) -> float:
+        """Returns an evaluation score between 0 and 1 for two strings.
+
+        Args:
+            hyp: The hypothesized string (e.g. a system output).
+            ref: The reference string (e.g. a gold-standard output).
+        """
+        ...

llments/lm/lm.py

@@ -0,0 +1,4 @@
+
+class LanguageModel:
+
+

llments/lm/loader.py

@@ -0,0 +1,15 @@
+
+
+from llments.lm.lm import LanguageModel
+
+
+def load_from_spec_file(spec_file: str) -> LanguageModel:
+    """Load a language model from a specification file.
+
+    Args:
+        spec_file: The path to the specification file.
+
+    Returns:
+        A language model.
+    """
+    raise NotImplementedError

llments/lm/loader_test.py

@@ -0,0 +1,4 @@
+
+def test_load_from_spec_file():
+    """Test that load_from_specification_file() loads a language model."""
+    raise NotImplementedError

llments/version.py

@@ -0,0 +1 @@
+VERSION = "0.0.0a1"

pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "llments"
+authors = [
+    {name = "Graham Neubig", email = "neubig@gmail.com"},
+]
+description = "llments"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+classifiers = [
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dependencies = []
+dynamic = ["version"]
+
+[project.optional-dependencies]
+dev = [
+    "pre-commit"
+]
+
+[tool.setuptools.dynamic]
+version = {attr = "llments.version.VERSION"}