Merge pull request #574 from opencybersecurityalliance/k2-beta-release

kestrel v2 beta release

.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,41 @@
+name: Publish to PyPI
+
+on:
+  # Disable automatic publishing until Kestrel v2 finishes beta and replaces Kestrel v1
+  #release:
+  #  types: [published]
+  workflow_dispatch:
+
+jobs:
+  publish:
+    strategy:
+      matrix:
+        package:
+          - kestrel_core
+          - kestrel_interface_opensearch
+          - kestrel_interface_sqlalchemy
+          - kestrel_jupyter
+          - kestrel_tool
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./packages/${{ matrix.package }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install building environment
+        run: |
+          pip install --upgrade pip setuptools wheel
+          pip install --upgrade build twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          # `build` is installed as a Python module, not a standalone commandline
+          python -m build --sdist --wheel --outdir dist/ .
+          twine check dist/*
+          twine upload --verbose --skip-existing dist/*

CHANGELOG.rst

@@ -11,4 +11,55 @@ Unreleased
 
 This is the Changelog for Kestrel 2. Look for Changelog for Kestrel 1 in the ``develop_v1`` branch.
 
+2.0.0b (2024-07-30)
+==================
+
+Added
+-----
+
+- Commands supported
+    - NEW
+    - GET
+    - FIND
+    - DISP
+    - INFO
+    - APPLY
+    - EXPLAIN
+    - expression
+
+- Supported Entities
+    - `event` is a first-class citizen in Kestrel v2
+    - Check `kestrel.mapping.types.*` for details
+
+- Supported Relations
+    - Relation between entity and entity
+    - Relation between event and entity
+    - Check `kestrel.config.relations.*` for details
+
+- Kestrel Intermediate Representation Graph (IRGraph)
+    - GIT compilation with IRGraph
+    - Kestrel segments IRGraph to execute on multiple interfaces/datastores/exec_env
+    - Kestrel cache glues executions together for a session
+
+- OCSF/ECS/STIX syntax supported in frontend
+    - Type inferencing supported
+    - Comparison field translation supported
+    - Project field translation supported
+
+- Datasource Interfaces
+    - Sqlalchemy fully working
+        - Multi-store support
+        - Query column translation supported
+        - Value translation supported
+    - Opensearch halfy done
+
+- Analytics Interfaces
+    - Python analytics interface works for `DataFrame` but not `Display` objects
+
+- Kestrel Tool
+    - `mkdb` to ingest NLJSON logs into SQL databases
+
+- Example Mappings
+    - Four examples mappings created for BlackHat 2024 (SecurityDatasets GoldenSAML case)
+
 .. _Keep a Changelog: https://keepachangelog.com/en/1.0.0/

packages/kestrel_core/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kestrel_core"
-version = "2.0.0"
+version = "2.0.0b"
 description = "Kestrel Threat Hunting Language"
 readme = "README.rst"
 requires-python = ">=3.8"

packages/kestrel_interface_opensearch/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kestrel_interface_opensearch"
-version = "2.0.0"
+version = "2.0.0b"
 description = "Kestrel OpenSearch Datasource Interface"
 readme = "README.rst"
 requires-python = ">=3.8"
@@ -26,7 +26,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "kestrel_core>=2.0.0",
+    "kestrel_core>=2.0.0b",
     "opensearch-py>=2.6.0",
 ]
 

packages/kestrel_interface_sqlalchemy/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kestrel_interface_sqlalchemy"
-version = "2.0.0"
+version = "2.0.0b"
 description = "Kestrel SQLAlchemy Datasource Interface"
 readme = "README.rst"
 requires-python = ">=3.8"
@@ -26,7 +26,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "kestrel_core>=2.0.0",
+    "kestrel_core>=2.0.0b",
 ]
 
 [project.urls]

packages/kestrel_jupyter/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kestrel_jupyter"
-version = "2.0.0"
+version = "2.0.0b"
 description = "Kestrel Jupyter Kernel"
 readme = "README.rst"
 requires-python = ">=3.8"
@@ -26,7 +26,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "kestrel_core==2.0.0",
+    "kestrel_core==2.0.0b",
     "jupyterlab-server",
     "jupyterlab",
     "jupyter_client",

packages/kestrel_tool/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kestrel_tool"
-version = "2.0.0"
+version = "2.0.0b"
 description = "Kestrel Threat Hunting Language CLI Multi-tool"
 readme = "README.rst"
 requires-python = ">=3.8"
@@ -26,7 +26,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "kestrel_core>=2.0.0",
+    "kestrel_core>=2.0.0b",
     "typer>=0.12.3",
 ]