From 5967dee9e0b2be97fd472b40416601474ee82e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Noco=C5=84?= Date: Mon, 9 Dec 2024 10:13:30 +0100 Subject: [PATCH] Fixed after dev review --- .../ai_actions/assets/js/addAudioModule.js | 2 +- .../ai_actions/assets/js/transcribe.audio.js | 6 ++- code_samples/ai_actions/config/services.yaml | 5 ++- .../Handler/LLaVaTextToTextActionHandler.php | 4 +- .../WhisperAudioToTextActionHandler.php | 4 +- .../AI/REST/Input/Parser/TranscribeAudio.php | 15 ++++++- .../AI/REST/Value/TranscribeAudioAction.php | 26 ++++++++++- .../src/Form/Type/TextToTextOptionsType.php | 2 +- .../Form/Type/TranscribeAudioOptionsType.php | 2 +- .../edit/form_fields_binary_ai.html.twig | 14 +++--- docs/ai_actions/extend_ai_actions.md | 43 ++++++++++++------- 11 files changed, 87 insertions(+), 36 deletions(-) diff --git a/code_samples/ai_actions/assets/js/addAudioModule.js b/code_samples/ai_actions/assets/js/addAudioModule.js index 13b5f99cf0a..8143164ab8c 100644 --- a/code_samples/ai_actions/assets/js/addAudioModule.js +++ b/code_samples/ai_actions/assets/js/addAudioModule.js @@ -1,4 +1,4 @@ import { addModule } from '../../vendor/ibexa/connector-ai/src/bundle/Resources/public/js/core/create.ai.module'; -import { default as TranscribeAudio } from './transcribe.audio'; +import { TranscribeAudio } from './transcribe.audio'; addModule(TranscribeAudio); diff --git a/code_samples/ai_actions/assets/js/transcribe.audio.js b/code_samples/ai_actions/assets/js/transcribe.audio.js index 74e94352492..f98abd700da 100644 --- a/code_samples/ai_actions/assets/js/transcribe.audio.js +++ b/code_samples/ai_actions/assets/js/transcribe.audio.js @@ -1,6 +1,5 @@ import BaseAIComponent from '../../vendor/ibexa/connector-ai/src/bundle/Resources/public/js/core/base.ai.component'; - -export default class TranscribeAudio extends BaseAIComponent { +export class TranscribeAudio extends BaseAIComponent { constructor(mainElement, config) { super(mainElement, config); @@ -19,6 +18,9 @@ export default class TranscribeAudio extends BaseAIComponent { if (request.status === 200) { return this.convertToBase64(request.responseText); } + else { + processError(request.responseText); + } } getRequestBody() { diff --git a/code_samples/ai_actions/config/services.yaml b/code_samples/ai_actions/config/services.yaml index b3b9b3af27f..786132d0199 100644 --- a/code_samples/ai_actions/config/services.yaml +++ b/code_samples/ai_actions/config/services.yaml @@ -62,9 +62,12 @@ services: - { name: ibexa.ai.action.handler, priority: 0 } - { name: app.connector_ai.action.handler.audio_to_text, priority: 0 } + Ibexa\Contracts\ConnectorAi\ActionConfiguration\OptionsFormatterInterface: + alias: Ibexa\ConnectorAi\ActionConfiguration\JsonOptionsFormatter + #REST services App\AI\REST\Input\Parser\TranscribeAudio: - parent: Ibexa\ConnectorAi\REST\Input\Parser\Action + parent: Ibexa\Rest\Server\Common\Parser tags: - { name: ibexa.rest.input.parser, mediaType: application/vnd.ibexa.api.ai.TranscribeAudio } diff --git a/code_samples/ai_actions/src/AI/Handler/LLaVaTextToTextActionHandler.php b/code_samples/ai_actions/src/AI/Handler/LLaVaTextToTextActionHandler.php index 8cc9146f3b7..f04423a8100 100644 --- a/code_samples/ai_actions/src/AI/Handler/LLaVaTextToTextActionHandler.php +++ b/code_samples/ai_actions/src/AI/Handler/LLaVaTextToTextActionHandler.php @@ -4,8 +4,8 @@ use Ibexa\Contracts\ConnectorAi\Action\ActionHandlerInterface; use Ibexa\Contracts\ConnectorAi\Action\DataType\Text; +use Ibexa\Contracts\ConnectorAi\Action\Response\TextResponse; use Ibexa\Contracts\ConnectorAi\Action\TextToText\Action as TextToTextAction; -use Ibexa\Contracts\ConnectorAi\Action\TextToText\ActionResponse as TextToTextActionResponse; use Ibexa\Contracts\ConnectorAi\ActionInterface; use Ibexa\Contracts\ConnectorAi\ActionResponseInterface; use Symfony\Contracts\HttpClient\HttpClientInterface; @@ -63,7 +63,7 @@ public function handle(ActionInterface $action, array $context = []): ActionResp $output = strip_tags(json_decode($response->getContent(), true)['choices'][0]['message']['content']); - return new TextToTextActionResponse(new Text([$output])); + return new TextResponse(new Text([$output])); } public static function getIdentifier(): string diff --git a/code_samples/ai_actions/src/AI/Handler/WhisperAudioToTextActionHandler.php b/code_samples/ai_actions/src/AI/Handler/WhisperAudioToTextActionHandler.php index 21c62ab25b0..df20dd0ccfb 100644 --- a/code_samples/ai_actions/src/AI/Handler/WhisperAudioToTextActionHandler.php +++ b/code_samples/ai_actions/src/AI/Handler/WhisperAudioToTextActionHandler.php @@ -5,7 +5,7 @@ use App\AI\ActionType\TranscribeAudioActionType; use Ibexa\Contracts\ConnectorAi\Action\ActionHandlerInterface; use Ibexa\Contracts\ConnectorAi\Action\DataType\Text; -use Ibexa\Contracts\ConnectorAi\Action\TextToText\ActionResponse; +use Ibexa\Contracts\ConnectorAi\Action\Response\TextResponse; use Ibexa\Contracts\ConnectorAi\ActionInterface; use Ibexa\Contracts\ConnectorAi\ActionResponseInterface; use Symfony\Component\Process\Exception\ProcessFailedException; @@ -55,7 +55,7 @@ public function handle(ActionInterface $action, array $context = []): ActionResp unlink($path); - return new ActionResponse(new Text([$output])); + return new TextResponse(new Text([$output])); } public static function getIdentifier(): string diff --git a/code_samples/ai_actions/src/AI/REST/Input/Parser/TranscribeAudio.php b/code_samples/ai_actions/src/AI/REST/Input/Parser/TranscribeAudio.php index e7164401d2e..0d0387d7713 100644 --- a/code_samples/ai_actions/src/AI/REST/Input/Parser/TranscribeAudio.php +++ b/code_samples/ai_actions/src/AI/REST/Input/Parser/TranscribeAudio.php @@ -5,13 +5,16 @@ use App\AI\DataType\Audio as AudioDataType; use App\AI\REST\Value\TranscribeAudioAction; use Ibexa\ConnectorAi\REST\Input\Parser\Action; +use Ibexa\Contracts\ConnectorAi\Action\RuntimeContext; use Ibexa\Contracts\Rest\Input\ParsingDispatcher; +use Ibexa\Rest\Input\BaseParser; -final class TranscribeAudio extends Action +final class TranscribeAudio extends BaseParser { public const AUDIO_KEY = 'Audio'; public const BASE64_KEY = 'base64'; + /** @param array $data */ public function parse(array $data, ParsingDispatcher $parsingDispatcher): TranscribeAudioAction { $this->assertInputIsValid($data); @@ -34,4 +37,14 @@ private function assertInputIsValid(array $data): void throw new \InvalidArgumentException('Missing base64 key'); } } + + /** + * @param array $data + */ + private function getRuntimeContext(array $data): RuntimeContext + { + return new RuntimeContext( + $data[Action::RUNTIME_CONTEXT_KEY] ?? [] + ); + } } diff --git a/code_samples/ai_actions/src/AI/REST/Value/TranscribeAudioAction.php b/code_samples/ai_actions/src/AI/REST/Value/TranscribeAudioAction.php index 3b8b1dc0947..9ab41eb4665 100644 --- a/code_samples/ai_actions/src/AI/REST/Value/TranscribeAudioAction.php +++ b/code_samples/ai_actions/src/AI/REST/Value/TranscribeAudioAction.php @@ -2,8 +2,30 @@ namespace App\AI\REST\Value; -use Ibexa\ConnectorAi\REST\Value\RestAction; +use App\AI\DataType\Audio; +use Ibexa\Contracts\ConnectorAi\Action\RuntimeContext; -final class TranscribeAudioAction extends RestAction +final class TranscribeAudioAction { + private Audio $input; + + private RuntimeContext $runtimeContext; + + public function __construct( + Audio $input, + RuntimeContext $runtimeContext + ) { + $this->input = $input; + $this->runtimeContext = $runtimeContext; + } + + public function getInput(): Audio + { + return $this->input; + } + + public function getRuntimeContext(): RuntimeContext + { + return $this->runtimeContext; + } } diff --git a/code_samples/ai_actions/src/Form/Type/TextToTextOptionsType.php b/code_samples/ai_actions/src/Form/Type/TextToTextOptionsType.php index 27cfffd9c71..c56e32bdbaa 100644 --- a/code_samples/ai_actions/src/Form/Type/TextToTextOptionsType.php +++ b/code_samples/ai_actions/src/Form/Type/TextToTextOptionsType.php @@ -21,7 +21,7 @@ public function buildForm(FormBuilderInterface $builder, array $options): void public function configureOptions(OptionsResolver $resolver): void { $resolver->setDefaults([ - 'translation_domain' => 'ibexa_connector_ai', + 'translation_domain' => 'app_ai', 'translation_mode' => false, ]); diff --git a/code_samples/ai_actions/src/Form/Type/TranscribeAudioOptionsType.php b/code_samples/ai_actions/src/Form/Type/TranscribeAudioOptionsType.php index 42396c0db56..8c65319a6c1 100644 --- a/code_samples/ai_actions/src/Form/Type/TranscribeAudioOptionsType.php +++ b/code_samples/ai_actions/src/Form/Type/TranscribeAudioOptionsType.php @@ -21,7 +21,7 @@ public function buildForm(FormBuilderInterface $builder, array $options): void public function configureOptions(OptionsResolver $resolver): void { $resolver->setDefaults([ - 'translation_domain' => 'ibexa_connector_ai', + 'translation_domain' => 'app_ai', 'translation_mode' => false, ]); diff --git a/code_samples/ai_actions/templates/themes/admin/admin/ui/fieldtype/edit/form_fields_binary_ai.html.twig b/code_samples/ai_actions/templates/themes/admin/admin/ui/fieldtype/edit/form_fields_binary_ai.html.twig index d52f9b0898d..0833afdd065 100644 --- a/code_samples/ai_actions/templates/themes/admin/admin/ui/fieldtype/edit/form_fields_binary_ai.html.twig +++ b/code_samples/ai_actions/templates/themes/admin/admin/ui/fieldtype/edit/form_fields_binary_ai.html.twig @@ -1,18 +1,18 @@ {% extends '@ibexadesign/ui/field_type/edit/ezbinaryfile.html.twig' %} {% block ezbinaryfile_preview %} - {{ parent() }} + {{ parent() }} {% set transcriptFieldIdentifier = 'transcript' %} {% set fieldTypeIdentifiers = form.parent.parent.vars.value|keys %} {% if transcriptFieldIdentifier in fieldTypeIdentifiers %} - {% set module_id = 'TranscribeAudio' %} - {% set ai_config_id = 'transcribe_audio' %} - {% set container_selector = '.ibexa-edit-content' %} - {% set input_selector = '.ibexa-field-edit-preview__action--preview' %} - {% set output_selector = '#ezplatform_content_forms_content_edit_fieldsData_transcript_value' %} - {% set cancel_wrapper_selector = '.ibexa-field-edit-preview__media-wrapper' %} + {% set module_id = 'TranscribeAudio' %} + {% set ai_config_id = 'transcribe_audio' %} + {% set container_selector = '.ibexa-edit-content' %} + {% set input_selector = '.ibexa-field-edit-preview__action--preview' %} + {% set output_selector = '#ezplatform_content_forms_content_edit_fieldsData_transcript_value' %} + {% set cancel_wrapper_selector = '.ibexa-field-edit-preview__media-wrapper' %} {% embed '@ibexadesign/connector_ai/ui/ai_module/ai_component.html.twig' with { ai_config_id, diff --git a/docs/ai_actions/extend_ai_actions.md b/docs/ai_actions/extend_ai_actions.md index eeb6f6788cb..1ff4a40b87e 100644 --- a/docs/ai_actions/extend_ai_actions.md +++ b/docs/ai_actions/extend_ai_actions.md @@ -27,9 +27,12 @@ This action is parameterized using the [RuntimeContext](../api/php_api/php_api_r | Action Context | Action Handler options |Set additional parameters for the Action Handler | Information about the model, temperature, prompt, and max tokens allowed. | | Action Context | System options | Set additional information, not matching the other option collections | Information about the fallback locale | -Both `ActionContext` and `RuntimeContext` are passed to the Action Handler (an object implementing the [ActionHandlerInterface](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-Action-ActionHandlerInterface.html)) to execute the action. The Action Handler is responsible for combining all the options passed, sending them to the AI service and returning an [ActionResponse](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-ActionResponseInterface.html). +Both `ActionContext` and `RuntimeContext` are passed to the Action Handler (an object implementing the [ActionHandlerInterface](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-Action-ActionHandlerInterface.html)) to execute the action. The Action Handler is responsible for combining all the options together, sending them to the AI service and returning an [ActionResponse](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-ActionResponseInterface.html). -You can pass the Action Handler directly to the `ActionServiceInterface::execute()` method. If omitted, an Action Handler able to handle the executed Action is selected automatically. You can influence this choice by creating your own class implementing the [ActionHandlerResolverInterface](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-Action-ActionHandlerResolverInterface.html) or by listening to the [ResolveActionHandlerEvent](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-Events-ResolveActionHandlerEvent.html) Event the default implementation emits. +You can pass the Action Handler directly to the `ActionServiceInterface::execute()` method, overriding all the other ways of selecting the Action Handler. +You can also specify the Action Handler by making it part of the passed [Action Configuration](#action-configurations). +In other cases, the Action Handler is selected automatically. +You can influence this choice by creating your own class implementing the [ActionHandlerResolverInterface](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-Action-ActionHandlerResolverInterface.html) or by listening to the [ResolveActionHandlerEvent](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-Events-ResolveActionHandlerEvent.html) Event that the default implementation emits. You can influence the execution of an Action with two events: @@ -76,7 +79,7 @@ Actions Configurations are tied to a specific Action Type and are translatable. ### Execute Actions with Action Configurations -Reuse existing Action Configurations to simplify the execution of AI Actions. You can pass one directly to the `actionServiceInterface::execute()` method: +Reuse existing Action Configurations to simplify the execution of AI Actions. You can pass one directly to the `ActionServiceInterface::execute()` method: ``` php hl_lines="7-8" [[= include_file('code_samples/ai_actions/src/Command/ActionConfigurationCreateCommand.php', 75, 83) =]] @@ -99,18 +102,18 @@ The following example adds a new Action Handler connecting to a local AI run usi Create a class implementing the [ActionHandlerInterface](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-Action-ActionHandlerInterface.html) and register it as a service: -- The `ActionHandlerInterface::supports()` method decided whether the Action Handler is able to execute run given Action. +- The `ActionHandlerInterface::supports()` method decided whether the Action Handler is able to execute given Action. - The `ActionHandlerInterface::handle()` method is responsible for combining all the Action options together, sending them to the AI service and forming an Action Response. - The `ActionHandlerInterface::getIdentifier()` method returns the identifier of the Action Handler which you can use to refer to it in other places in the code. See the code sample below, together with a matching service definition: -``` php hl_lines="27-30 32-67 69-72" +``` php hl_lines="19 27-30 32-67 69-72" [[= include_file('code_samples/ai_actions/src/AI/Handler/LLaVaTextToTextActionHandler.php') =]] ``` ``` yaml -[[= include_file('code_samples/ai_actions/config/services.yaml', 33, 37) =]] +[[= include_file('code_samples/ai_actions/config/services.yaml', 28, 33) =]] ``` The `ibexa.ai.action.handler` tag is used by the `ActionHandlerResolverInterface` to find all the Action Handlers in the system. @@ -130,11 +133,19 @@ The example handler uses the `system_prompt` option, which becomes part of the A ``` ``` yaml -[[= include_file('code_samples/ai_actions/config/services.yaml', 38, 45) =]] +[[= include_file('code_samples/ai_actions/config/services.yaml', 34, 41) =]] ``` The created Form Type adds the `system_prompt` field to the Form. Use the `Ibexa\Bundle\ConnectorAi\Form\FormMapper\ActionConfiguration\ActionHandlerOptionsFormMapper` class together with the `ibexa.connector_ai.action_configuration.form_mapper.options` service tag to make it part of the Action Handler options form. Pass the Action Handler identifier (`LLaVATextToText`) as the type when tagging the service. +The Action Handler and Action Type options are rendered in the back office using the built-in Twig option formatter. You can create your own formatting by creating a class implementing the [OptionsFormatterInterface](../api/php_api/php_api_reference/classes/Ibexa-Contracts-ConnectorAi-ActionConfiguration-OptionsFormatterInterface.html) interface and aliasing it to `Ibexa\Contracts\ConnectorAi\ActionConfiguration\OptionsFormatterInterface`. + +The following service definition switches the options rendering to the other built-in option formatter, displaying the options as JSON. + +``` yaml +[[= include_file('code_samples/ai_actions/config/services.yaml', 64, 66) =]] +``` + ## Custom Action Type use case With custom Action Types you can create your own tasks for the AI services to perform. They can be integrated with the rest of the AI framework provided by [[= product_name_base =]] and incorporated into the back office. @@ -199,7 +210,7 @@ The built-in `Ibexa\Bundle\ConnectorAi\Form\FormMapper\ActionConfiguration\Actio An example Action Handler combines the input data and the Action Type options and passes them to the Whisper executable to form an Action Response. The language of the transcribed data is extracted from the Runtime Context for better results. The Action Type options provided in the Action Context dictate whether the timestamps will be removed before returning the result. -``` php hl_lines="30-33 48" +``` php hl_lines="32-35 50" [[= include_file('code_samples/ai_actions/src/AI/Handler/WhisperAudioToTextActionHandler.php') =]] ``` @@ -219,7 +230,7 @@ Start by creating an Input Parser able to handle the `application/vnd.ibexa.api. ``` ``` yaml -[[= include_file('code_samples/ai_actions/config/services.yaml', 65, 69) =]] +[[= include_file('code_samples/ai_actions/config/services.yaml', 68, 72) =]] ``` The `TranscribeAudioAction` is a value object holding the parsed request data. @@ -244,7 +255,7 @@ To transform the `TranscribeAudioAction` into a REST response you need to create ``` ``` yaml -[[= include_file('code_samples/ai_actions/config/services.yaml', 70, 73) =]] +[[= include_file('code_samples/ai_actions/config/services.yaml', 73, 76) =]] ``` - A visitor converting the response value object into a serialized REST response: @@ -255,7 +266,7 @@ To transform the `TranscribeAudioAction` into a REST response you need to create ``` ``` yaml -[[= include_file('code_samples/ai_actions/config/services.yaml', 74, 78) =]] +[[= include_file('code_samples/ai_actions/config/services.yaml', 77, 81) =]] ``` You can now execute a specific Action Configuration for the new custom Action Type through REST API by sending the following request: @@ -298,7 +309,7 @@ And add it to the SiteAccess configuration for the `admin_group`: The configuration of the AI component takes the following parameters: - `module_id` - name of the JavaScript module to handle the invoked action. `ImgToText` is a built-in one handling alternative text use case, `TranscribeAudio` is a custom one. -- `ai_config_id` - identifier of the Action Type to load. The [ibexa_ai_config Twig function](ai_actions_twig_functions.md#ibexa_ai_config) is used under the hood. +- `ai_config_id` - identifier of the Action Type to load Action Configurations for. The [ibexa_ai_config Twig function](ai_actions_twig_functions.md#ibexa_ai_config) is used under the hood. - `container_selector` - CSS selector to narrow down the HTML area which is affected by the AI component. - `input_selector` - CSS selector indicating the input field (must be the below the `container_selector` in the HTML structure). - `output_selector` - CSS selector indicating the output field (must be the below the `container_selector` in the HTML structure). @@ -307,7 +318,7 @@ The configuration of the AI component takes the following parameters: Now create the JavaScript module mentioned in the template that is responsible for: - gathering the input data (downloading the attached binary file and converting it into base64) -- executing the Action Configuration chosen by the editor using the REST API +- executing the Action Configuration chosen by the editor through the REST API - attaching the response to the output field You can find the code of the module below. Place it in a file called `assets/js/transcribe.audio.js` @@ -316,15 +327,15 @@ You can find the code of the module below. Place it in a file called `assets/js/ [[= include_file('code_samples/ai_actions/assets/js/transcribe.audio.js') =]] ``` -The last step is adding the module to the list of AI modules in the system. +The last step is adding the module to the list of AI modules in the system, by using the provided `addModule` function. -Use the provided `addModule` function to do so: +Create a file called `assets/js/addAudioModule.js`: ``` js [[= include_file('code_samples/ai_actions/assets/js/addAudioModule.js') =]] ``` -And include it into the back office using Webpack Encore. See [configuring assets from main project files](importing_assets_from_bundle/#configuration-from-main-project-files) to learn more about this mechanism. +And include it into the back office using Webpack Encore. See [configuring assets from main project files](importing_assets_from_bundle.md#configuration-from-main-project-files) to learn more about this mechanism. ``` js [[= include_file('code_samples/ai_actions/webpack.config.js', 40, 47) =]]