Release v0.10.0 (#418)

# Changelog

## Breaking Changes

- Dropped support for python 3.8; added support for python 3.12 (#400);
- Reworked DB architecture to support partials turn reads/writes (#93).
  Old Context storages are incompatible with the new ones.
  See tutorial Context Storages: 8 for more info;
- `Context.labels`, `Context.requests`, `Context.responses` are now only
lazily loaded (#93).
  Items from older turns can be loaded on demand.
  Their `__getitem__` and `get` methods are now async.


## Features 

- Added `LLMResponse` and `LLMCondition` classes that allow using LLMs
(#376).
See the new LLM Integration tutorials and LLM user guide for more info;
- Added option to extract group slots partially (#394).
  See tutorial Slots: 2 for more information;
- `Message.original_message` is replaced with `Message.origin` which
stores both
the original message and the interface from which the message originated
(#398);
- Added `Context.current_turn_id` field which stores the number of the
current turn (#93);
- Added `Context.created_at`, `Context.updated_at` timestamp fields
(#93);
- Added `Context.turns` property which allows iterating over
requests/labels/responses by their turn ids (#93);
- `Context.labels`, `Context.requests`, `Context.responses` now support
slicing (#93).
`__getitem__`, `__setitem__` and `__delitem__` methods can now accept
slices of turn ids in addition to single turn id.
`get` method can now accepts iterable of turn ids in addition to single
turn id.


## Documentation

- Documentation is now versioned (#346, #409).
You can select preferred version via the drop-down menu in the top-right
corner.


## Developer changes

- Context now has field `origin_interface` to store name of the
interface that created it (#398);
- Added script `docs_no_docker` for building documentation without
docker (ef11ff9);
- Added in-RAM context storage to be the default one instead of a plain
dict (#93);
- Removed methods `Context.add_request`, `Context.add_label` and
`Context.add_response` (#93).
  Use setters with `Context.current_turn_id` instead.

.env_file

@@ -1,4 +1,6 @@
 TG_BOT_TOKEN=token
+OPENAI_API_KEY=api_key
+ANTHROPIC_API_KEY=api_key
 MYSQL_USERNAME=root
 MYSQL_PASSWORD=pass
 MYSQL_ROOT_PASSWORD=pass

.github/workflows/build_and_publish_docs.yml

@@ -6,6 +6,8 @@ on:
     - dev
     - master
     - test/**
+    tags:
+    - v[0-9]+.[0-9]+.[0-9]+
   pull_request:
     branches:
     - dev
@@ -21,6 +23,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: set up python 3.9
         uses: actions/setup-python@v5
@@ -33,40 +37,29 @@ jobs:
 
       - name: setup poetry and install dependencies
         run: |
-          python -m pip install --upgrade pip poetry
+          python -m pip install --upgrade pip poetry==1.8.5
           python -m poetry install --with tutorials,docs --all-extras --no-ansi --no-interaction
 
+      - name: save docs version into a variable
+        run: |
+          echo "DOC_VERSION=${{ github.head_ref || github.ref_name }}" >> $GITHUB_ENV
+
       - name: build documentation
         env:
           TG_BOT_TOKEN: ${{ secrets.TG_BOT_TOKEN }}
           TG_BOT_USERNAME: ${{ secrets.TG_BOT_USERNAME }}
+          DOC_VERSION: ${{ env.DOC_VERSION }}
         run: |
           python -m poetry run poe docs
 
-      - name: remove jekyll theming
-        run: touch docs/build/.nojekyll
-
-      - name: save branch name without slashes
-        if: ${{ github.ref != 'refs/heads/master' }}
-        env:
-          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
-        run: |
-          BRANCH_NAME=${{ env.BRANCH_NAME }}
-          BRANCH_NAME=${BRANCH_NAME////_}
-          echo BRANCH_NAME=${BRANCH_NAME} >> $GITHUB_ENV
-
-      - name: save artifact
-        if: ${{ github.ref != 'refs/heads/master' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ format('github-pages-for-branch-{0}', env.BRANCH_NAME) }}
-          path: docs/build/
-          retention-days: 3
-
       - name: deploy website
-        if: ${{ github.ref == 'refs/heads/master' }}
-        uses: JamesIves/github-pages-deploy-action@v4.6.4
+        uses: JamesIves/github-pages-deploy-action@v4
         with:
           branch: gh-pages
           folder: docs/build/
-          single-commit: True
+          target-folder: ${{ env.DOC_VERSION }}
+          clean: True
+
+      - name: print link to the documentation
+        run: |
+          echo "https://deeppavlov.github.io/chatsky/${{ github.head_ref || github.ref_name }}"

.github/workflows/build_and_upload_release.yml

@@ -22,7 +22,7 @@ jobs:
 
       - name: setup poetry
         run: |
-          python -m pip install --upgrade pip poetry
+          python -m pip install --upgrade pip poetry==1.8.5
 
       - name: build wheels and test uploading to pypi
         if: startsWith(github.ref, 'refs/tags/v') != true

.github/workflows/codestyle.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: setup poetry and install dependencies
         run: |
-          python -m pip install --upgrade pip poetry
+          python -m pip install --upgrade pip poetry==1.8.5
           python -m poetry install --with lint --no-ansi --no-interaction
 
       - name: run codestyle

.github/workflows/generate_version_switcher.yml

@@ -0,0 +1,44 @@
+name: generate_version_switcher
+
+on:
+  workflow_run:
+    workflows: ["build_and_publish_docs"]
+    types:
+      - completed
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    name: generate and update version switcher's json file
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: install GitPython
+        run: python -m pip install GitPython
+
+      - name: generate version switcher
+        env:
+          VERSION_SWITCHER_STARTING_TAG: ${{ vars.VERSION_SWITCHER_STARTING_TAG }}
+          VERSION_SWITCHER_TAG_BLACKLIST: ${{ vars.VERSION_SWITCHER_TAG_BLACKLIST }}
+          VERSION_SWITCHER_TAG_WHITELIST: ${{ vars.VERSION_SWITCHER_TAG_WHITELIST }}
+        run: |
+          python ./scripts/switcher_gen.py
+
+      - name: copy version switcher for updating it
+        run: |
+          mkdir docs/source/switcher/
+          cp docs/source/_static/switcher.json docs/source/switcher/switcher.json
+
+      - name: update version switcher
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          branch: gh-pages
+          folder: docs/source/switcher/
+          clean: False

.github/workflows/test_coverage.yml

@@ -32,7 +32,7 @@ jobs:
 
       - name: setup poetry and install dependencies
         run: |
-          python -m pip install --upgrade pip poetry
+          python -m pip install --upgrade pip poetry==1.8.5
           python -m poetry install --with test,tutorials --all-extras --no-ansi --no-interaction
 
       - name: run tests

.github/workflows/test_full.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         os: [macOS-latest, windows-latest, ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
@@ -34,7 +34,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          python -m pip install --upgrade pip poetry
+          python -m pip install --upgrade pip poetry==1.8.5
           python -m poetry install --with test,tutorials --all-extras --no-ansi --no-interaction
 
       - name: run pytest
@@ -49,14 +49,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: set up python 3.8
+      - name: set up python 3.9
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: install dependencies
         run: |
-          python -m pip install --upgrade pip poetry
+          python -m pip install --upgrade pip poetry==1.8.5
           python -m poetry install --with test --no-ansi --no-interaction
 
       - name: run pytest

.gitignore

@@ -3,6 +3,7 @@
 dist/
 venv/
 build/
+dbs/
 docs/source/apiref
 docs/source/_misc
 docs/source/release_notes.rst

CONTRIBUTING.md

@@ -73,6 +73,14 @@ Note that you'll need `pandoc` installed on your system in order to build docs.
 
 After that `docs/build` dir will be created and you can open index file `docs/build/index.html` in your browser of choice.
 
+You can also build docs faster without docker by running
+
+```bash
+poetry run poe docs_no_docker
+```
+
+Without docker some tutorials (that require docker services) will show errors and drawio diagrams will not be built.
+
 #### Documentation links
 
 In your tutorials, you can use special expanding directives in markdown cells.
@@ -121,6 +129,9 @@ poetry run poe quick_test
 
 _There's also quick_test_coverage for quick htmlcov generation, though it is very likely to be incomplete due to deselection of some tests._
 
+Both `quick_test_coverage` and `test_all` generate html coverage report.
+You can view it by opening the `htmlcov/index.html` file.
+
 To make sure that the code satisfies only the style requirements, run
 ```bash
 poetry run poe lint

README.md

@@ -4,7 +4,7 @@
 [![Codestyle](https://github.com/deeppavlov/chatsky/workflows/codestyle/badge.svg?branch=dev)](https://github.com/deeppavlov/chatsky/actions/workflows/codestyle.yml)
 [![Tests](https://github.com/deeppavlov/chatsky/workflows/test_coverage/badge.svg?branch=dev)](https://github.com/deeppavlov/chatsky/actions/workflows/test_coverage.yml)
 [![License Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/deeppavlov/chatsky/blob/master/LICENSE)
-![Python 3.8, 3.9, 3.10, 3.11](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11-green.svg)
+![Python 3.9, 3.10, 3.11, 3.12](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-green.svg)
 [![PyPI](https://img.shields.io/pypi/v/chatsky)](https://pypi.org/project/chatsky/)
 [![Downloads](https://static.pepy.tech/badge/chatsky)](https://pepy.tech/project/chatsky)
 
@@ -37,7 +37,7 @@ in the evolving landscape of Python applications and IoT connectivity.
 ## System Requirements
 
 - Supported operating systems include Ubuntu 18.04+, Windows 10+ (partial support), and MacOS Big Sur (partial support);
-- Python version 3.8 or higher is necessary for proper functionality;
+- Python version 3.9 or higher is necessary for proper functionality;
 - A minimum of 1 GB of RAM is required for optimal performance;
 - If analytics collection or database integration is intended, Docker version 20 or higher may be necessary.
 

chatsky/__rebuild_pydantic_models__.py

@@ -5,9 +5,14 @@
 from chatsky.core.script import Node
 from chatsky.core.pipeline import Pipeline
 from chatsky.slots.slots import SlotManager
-from chatsky.core.context import FrameworkData, ServiceState
+from chatsky.context_storages import DBContextStorage
+from chatsky.core.ctx_dict import ContextDict
+from chatsky.core.ctx_utils import ServiceState, FrameworkData, ContextMainInfo
 from chatsky.core.service import PipelineComponent
+from chatsky.llm import LLM_API
 
+ContextMainInfo.model_rebuild()
+ContextDict.model_rebuild()
 PipelineComponent.model_rebuild()
 Pipeline.model_rebuild()
 Script.model_rebuild()

chatsky/conditions/__init__.py

@@ -11,3 +11,4 @@
 )
 from chatsky.conditions.slots import SlotsExtracted
 from chatsky.conditions.service import ServiceFinished
+from chatsky.conditions.llm import LLMCondition

chatsky/conditions/llm.py

@@ -0,0 +1,77 @@
+"""
+LLM Conditions
+--------------
+This module provides LLM-based conditions.
+"""
+
+from pydantic import Field
+from typing import Optional
+
+from chatsky.core import BaseCondition, Context
+from chatsky.core.script_function import AnyResponse
+from chatsky.llm.methods import BaseMethod
+from chatsky.llm.langchain_context import get_langchain_context
+from chatsky.llm.filters import BaseHistoryFilter, DefaultFilter
+from chatsky.llm.prompt import PositionConfig, Prompt
+
+
+class LLMCondition(BaseCondition):
+    """
+    LLM-based condition.
+    Uses prompt to produce result from model and evaluates the result using given method.
+    """
+
+    llm_model_name: str
+    """
+    Key of the model in the :py:attr:`~chatsky.core.pipeline.Pipeline.models` dictionary.
+    """
+    prompt: AnyResponse = Field(default="", validate_default=True)
+    """
+    Condition prompt.
+    """
+    history: int = 1
+    """
+    Number of dialogue turns aside from the current one to keep in history. `-1` for full history.
+    """
+    filter_func: BaseHistoryFilter = Field(default_factory=DefaultFilter)
+    """
+    Filter function to filter messages in history.
+    """
+    prompt_misc_filter: str = Field(default=r"prompt")
+    """
+    Regular expression to find prompts by key names in MISC dictionary.
+    """
+    position_config: Optional[PositionConfig] = None
+    """
+    Config for positions of prompts and messages in history.
+    """
+    max_size: int = 5000
+    """
+    Maximum size of any message in chat in symbols.
+    If a message exceeds the limit it will not be sent to the LLM and a warning
+    will be produced.
+    """
+    method: BaseMethod
+    """
+    Method that takes model's output and returns boolean.
+    """
+
+    async def call(self, ctx: Context) -> bool:
+        model = ctx.pipeline.models[self.llm_model_name]
+
+        history_messages = []
+        history_messages.extend(
+            await get_langchain_context(
+                system_prompt=await model.system_prompt(ctx),
+                ctx=ctx,
+                call_prompt=Prompt(message=self.prompt),
+                prompt_misc_filter=self.prompt_misc_filter,
+                position_config=self.position_config or model.position_config,
+                length=self.history,
+                filter_func=self.filter_func,
+                llm_model_name=self.llm_model_name,
+                max_size=self.max_size,
+            )
+        )
+
+        return await model.condition(history_messages, self.method)

chatsky/conditions/standard.py

@@ -202,7 +202,7 @@ def __init__(
         super().__init__(flow_labels=flow_labels, labels=labels, last_n_indices=last_n_indices)
 
     async def call(self, ctx: Context) -> bool:
-        labels = list(ctx.labels.values())[-self.last_n_indices :]  # noqa: E203
+        labels = await ctx.labels.get(ctx.labels.keys()[-self.last_n_indices :])  # noqa: E203
         for label in labels:
             if label.flow_name in self.flow_labels or label in self.labels:
                 return True

chatsky/context_storages/__init__.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
 
-from .database import DBContextStorage, threadsafe_method, context_storage_factory
-from .json import JSONContextStorage, json_available
-from .pickle import PickleContextStorage, pickle_available
+from .database import DBContextStorage, context_storage_factory
+from .file import JSONContextStorage, PickleContextStorage, ShelveContextStorage, json_available, pickle_available
 from .sql import SQLContextStorage, postgres_available, mysql_available, sqlite_available, sqlalchemy_available
 from .ydb import YDBContextStorage, ydb_available
 from .redis import RedisContextStorage, redis_available
+from .memory import MemoryContextStorage
 from .mongo import MongoContextStorage, mongo_available
-from .shelve import ShelveContextStorage
 from .protocol import PROTOCOLS, get_protocol_install_suggestion