From 2ca60d755182c7599a517e1b0c781b7139a6d050 Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Tue, 30 Jul 2024 15:56:18 -0400 Subject: [PATCH] kestrel v2 beta release --- .github/workflows/publish-to-pypi.yml | 41 +++++++++++++++ CHANGELOG.rst | 51 +++++++++++++++++++ packages/kestrel_core/pyproject.toml | 2 +- .../pyproject.toml | 4 +- .../pyproject.toml | 4 +- packages/kestrel_jupyter/pyproject.toml | 4 +- packages/kestrel_tool/pyproject.toml | 4 +- 7 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/publish-to-pypi.yml diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 00000000..dd01c63a --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,41 @@ +name: Publish to PyPI + +on: + # Disable automatic publishing until Kestrel v2 finishes beta and replaces Kestrel v1 + #release: + # types: [published] + workflow_dispatch: + +jobs: + publish: + strategy: + matrix: + package: + - kestrel_core + - kestrel_interface_opensearch + - kestrel_interface_sqlalchemy + - kestrel_jupyter + - kestrel_tool + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./packages/${{ matrix.package }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install building environment + run: | + pip install --upgrade pip setuptools wheel + pip install --upgrade build twine + - name: Build and publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + # `build` is installed as a Python module, not a standalone commandline + python -m build --sdist --wheel --outdir dist/ . + twine check dist/* + twine upload --verbose --skip-existing dist/* diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1a5dcb64..523d5d17 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,4 +11,55 @@ Unreleased This is the Changelog for Kestrel 2. Look for Changelog for Kestrel 1 in the ``develop_v1`` branch. +2.0.0b (2024-07-30) +================== + +Added +----- + +- Commands supported + - NEW + - GET + - FIND + - DISP + - INFO + - APPLY + - EXPLAIN + - expression + +- Supported Entities + - `event` is a first-class citizen in Kestrel v2 + - Check `kestrel.mapping.types.*` for details + +- Supported Relations + - Relation between entity and entity + - Relation between event and entity + - Check `kestrel.config.relations.*` for details + +- Kestrel Intermediate Representation Graph (IRGraph) + - GIT compilation with IRGraph + - Kestrel segments IRGraph to execute on multiple interfaces/datastores/exec_env + - Kestrel cache glues executions together for a session + +- OCSF/ECS/STIX syntax supported in frontend + - Type inferencing supported + - Comparison field translation supported + - Project field translation supported + +- Datasource Interfaces + - Sqlalchemy fully working + - Multi-store support + - Query column translation supported + - Value translation supported + - Opensearch halfy done + +- Analytics Interfaces + - Python analytics interface works for `DataFrame` but not `Display` objects + +- Kestrel Tool + - `mkdb` to ingest NLJSON logs into SQL databases + +- Example Mappings + - Four examples mappings created for BlackHat 2024 (SecurityDatasets GoldenSAML case) + .. _Keep a Changelog: https://keepachangelog.com/en/1.0.0/ diff --git a/packages/kestrel_core/pyproject.toml b/packages/kestrel_core/pyproject.toml index 16769686..10dbe43c 100644 --- a/packages/kestrel_core/pyproject.toml +++ b/packages/kestrel_core/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "kestrel_core" -version = "2.0.0" +version = "2.0.0b" description = "Kestrel Threat Hunting Language" readme = "README.rst" requires-python = ">=3.8" diff --git a/packages/kestrel_interface_opensearch/pyproject.toml b/packages/kestrel_interface_opensearch/pyproject.toml index 2c0fe636..7fd9d729 100644 --- a/packages/kestrel_interface_opensearch/pyproject.toml +++ b/packages/kestrel_interface_opensearch/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "kestrel_interface_opensearch" -version = "2.0.0" +version = "2.0.0b" description = "Kestrel OpenSearch Datasource Interface" readme = "README.rst" requires-python = ">=3.8" @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ - "kestrel_core>=2.0.0", + "kestrel_core>=2.0.0b", "opensearch-py>=2.6.0", ] diff --git a/packages/kestrel_interface_sqlalchemy/pyproject.toml b/packages/kestrel_interface_sqlalchemy/pyproject.toml index c4309e70..e9161dfa 100644 --- a/packages/kestrel_interface_sqlalchemy/pyproject.toml +++ b/packages/kestrel_interface_sqlalchemy/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "kestrel_interface_sqlalchemy" -version = "2.0.0" +version = "2.0.0b" description = "Kestrel SQLAlchemy Datasource Interface" readme = "README.rst" requires-python = ">=3.8" @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ - "kestrel_core>=2.0.0", + "kestrel_core>=2.0.0b", ] [project.urls] diff --git a/packages/kestrel_jupyter/pyproject.toml b/packages/kestrel_jupyter/pyproject.toml index d97c115b..0f7c886b 100644 --- a/packages/kestrel_jupyter/pyproject.toml +++ b/packages/kestrel_jupyter/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "kestrel_jupyter" -version = "2.0.0" +version = "2.0.0b" description = "Kestrel Jupyter Kernel" readme = "README.rst" requires-python = ">=3.8" @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ - "kestrel_core==2.0.0", + "kestrel_core==2.0.0b", "jupyterlab-server", "jupyterlab", "jupyter_client", diff --git a/packages/kestrel_tool/pyproject.toml b/packages/kestrel_tool/pyproject.toml index d6287163..4f578ca8 100644 --- a/packages/kestrel_tool/pyproject.toml +++ b/packages/kestrel_tool/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "kestrel_tool" -version = "2.0.0" +version = "2.0.0b" description = "Kestrel Threat Hunting Language CLI Multi-tool" readme = "README.rst" requires-python = ">=3.8" @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ - "kestrel_core>=2.0.0", + "kestrel_core>=2.0.0b", "typer>=0.12.3", ]