From 0f6a51d06fd53e94cb1c601c6b2873b957fae69c Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Tue, 30 Apr 2024 13:23:20 -0400 Subject: [PATCH 01/37] add qset generation (limited) --- composer.json | 6 +- composer.lock | 231 +++++++++++++++++- fuel/app/classes/controller/qsets.php | 21 ++ fuel/app/classes/materia/api/v1.php | 196 +++++++++++++++ fuel/app/config/js.php | 1 + fuel/app/config/materia.php | 4 + fuel/app/tests/api/v1.php | 78 ++++++ .../hooks/useQuestionGeneration.jsx | 13 + src/components/question-generator.jsx | 45 ++++ src/components/widget-creator.jsx | 6 + src/materia/materia.creatorcore.js | 12 +- src/qset-generator.js | 15 ++ src/util/api.js | 5 + 13 files changed, 628 insertions(+), 5 deletions(-) create mode 100644 src/components/hooks/useQuestionGeneration.jsx create mode 100644 src/components/question-generator.jsx create mode 100644 src/qset-generator.js diff --git a/composer.json b/composer.json index 01d97360d..3211c8849 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,8 @@ "eher/oauth": "1.0.7", "aws/aws-sdk-php": "3.288.1", "symfony/dotenv": "^5.1", - "ucfopen/materia-theme-ucf": "2.0.2" + "ucfopen/materia-theme-ucf": "2.0.2", + "openai-php/client": "^0.8.5" }, "suggest": { "ext-memcached": "*" @@ -66,7 +67,8 @@ "vendor-dir": "fuel/vendor", "optimize-autoloader": true, "allow-plugins": { - "composer/installers": true + "composer/installers": true, + "php-http/discovery": true } }, "extra": { diff --git a/composer.lock b/composer.lock index a5ebe9d68..93384cbbf 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d68f3eabd3a95508cee1cdfc097109ca", + "content-hash": "686fbcb7d33a3a2e3d8e0d2698c49695", "packages": [ { "name": "aws/aws-crt-php", @@ -1200,6 +1200,98 @@ }, "time": "2023-08-25T10:54:48+00:00" }, + { + "name": "openai-php/client", + "version": "v0.8.5", + "source": { + "type": "git", + "url": "https://github.com/openai-php/client.git", + "reference": "0f755fafa4d3f8d5c8ed964d3166d078fac0605a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/openai-php/client/zipball/0f755fafa4d3f8d5c8ed964d3166d078fac0605a", + "reference": "0f755fafa4d3f8d5c8ed964d3166d078fac0605a", + "shasum": "" + }, + "require": { + "php": "^8.1.0", + "php-http/discovery": "^1.19.4", + "php-http/multipart-stream-builder": "^1.3.0", + "psr/http-client": "^1.0.3", + "psr/http-client-implementation": "^1.0.1", + "psr/http-factory-implementation": "*", + "psr/http-message": "^1.1.0|^2.0.0" + }, + "require-dev": { + "guzzlehttp/guzzle": "^7.8.1", + "guzzlehttp/psr7": "^2.6.2", + "laravel/pint": "^1.15.0", + "mockery/mockery": "^1.6.11", + "nunomaduro/collision": "^7.10.0", + "pestphp/pest": "^2.34.6", + "pestphp/pest-plugin-arch": "^2.7", + "pestphp/pest-plugin-type-coverage": "^2.8.1", + "phpstan/phpstan": "^1.10.66", + "rector/rector": "^1.0.4", + "symfony/var-dumper": "^6.4.4" + }, + "type": "library", + "autoload": { + "files": [ + "src/OpenAI.php" + ], + "psr-4": { + "OpenAI\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + }, + { + "name": "Sandro Gehri" + } + ], + "description": "OpenAI PHP is a supercharged PHP API client that allows you to interact with the Open AI API", + "keywords": [ + "GPT-3", + "api", + "client", + "codex", + "dall-e", + "language", + "natural", + "openai", + "php", + "processing", + "sdk" + ], + "support": { + "issues": "https://github.com/openai-php/client/issues", + "source": "https://github.com/openai-php/client/tree/v0.8.5" + }, + "funding": [ + { + "url": "https://www.paypal.com/paypalme/enunomaduro", + "type": "custom" + }, + { + "url": "https://github.com/gehrisandro", + "type": "github" + }, + { + "url": "https://github.com/nunomaduro", + "type": "github" + } + ], + "time": "2024-04-15T19:11:23+00:00" + }, { "name": "paragonie/random_compat", "version": "v9.99.100", @@ -1336,6 +1428,141 @@ }, "time": "2023-04-30T00:54:53+00:00" }, + { + "name": "php-http/discovery", + "version": "1.19.4", + "source": { + "type": "git", + "url": "https://github.com/php-http/discovery.git", + "reference": "0700efda8d7526335132360167315fdab3aeb599" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/discovery/zipball/0700efda8d7526335132360167315fdab3aeb599", + "reference": "0700efda8d7526335132360167315fdab3aeb599", + "shasum": "" + }, + "require": { + "composer-plugin-api": "^1.0|^2.0", + "php": "^7.1 || ^8.0" + }, + "conflict": { + "nyholm/psr7": "<1.0", + "zendframework/zend-diactoros": "*" + }, + "provide": { + "php-http/async-client-implementation": "*", + "php-http/client-implementation": "*", + "psr/http-client-implementation": "*", + "psr/http-factory-implementation": "*", + "psr/http-message-implementation": "*" + }, + "require-dev": { + "composer/composer": "^1.0.2|^2.0", + "graham-campbell/phpspec-skip-example-extension": "^5.0", + "php-http/httplug": "^1.0 || ^2.0", + "php-http/message-factory": "^1.0", + "phpspec/phpspec": "^5.1 || ^6.1 || ^7.3", + "sebastian/comparator": "^3.0.5 || ^4.0.8", + "symfony/phpunit-bridge": "^6.4.4 || ^7.0.1" + }, + "type": "composer-plugin", + "extra": { + "class": "Http\\Discovery\\Composer\\Plugin", + "plugin-optional": true + }, + "autoload": { + "psr-4": { + "Http\\Discovery\\": "src/" + }, + "exclude-from-classmap": [ + "src/Composer/Plugin.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com" + } + ], + "description": "Finds and installs PSR-7, PSR-17, PSR-18 and HTTPlug implementations", + "homepage": "http://php-http.org", + "keywords": [ + "adapter", + "client", + "discovery", + "factory", + "http", + "message", + "psr17", + "psr7" + ], + "support": { + "issues": "https://github.com/php-http/discovery/issues", + "source": "https://github.com/php-http/discovery/tree/1.19.4" + }, + "time": "2024-03-29T13:00:05+00:00" + }, + { + "name": "php-http/multipart-stream-builder", + "version": "1.3.0", + "source": { + "type": "git", + "url": "https://github.com/php-http/multipart-stream-builder.git", + "reference": "f5938fd135d9fa442cc297dc98481805acfe2b6a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/multipart-stream-builder/zipball/f5938fd135d9fa442cc297dc98481805acfe2b6a", + "reference": "f5938fd135d9fa442cc297dc98481805acfe2b6a", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0", + "php-http/discovery": "^1.15", + "psr/http-factory-implementation": "^1.0" + }, + "require-dev": { + "nyholm/psr7": "^1.0", + "php-http/message": "^1.5", + "php-http/message-factory": "^1.0.2", + "phpunit/phpunit": "^7.5.15 || ^8.5 || ^9.3" + }, + "type": "library", + "autoload": { + "psr-4": { + "Http\\Message\\MultipartStream\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com" + } + ], + "description": "A builder class that help you create a multipart stream", + "homepage": "http://php-http.org", + "keywords": [ + "factory", + "http", + "message", + "multipart stream", + "stream" + ], + "support": { + "issues": "https://github.com/php-http/multipart-stream-builder/issues", + "source": "https://github.com/php-http/multipart-stream-builder/tree/1.3.0" + }, + "time": "2023-04-28T14:10:22+00:00" + }, { "name": "phpseclib/phpseclib", "version": "2.0.31", @@ -3806,5 +4033,5 @@ "ext-mbstring": "*" }, "platform-dev": [], - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.3.0" } diff --git a/fuel/app/classes/controller/qsets.php b/fuel/app/classes/controller/qsets.php index b639486e0..c44041504 100644 --- a/fuel/app/classes/controller/qsets.php +++ b/fuel/app/classes/controller/qsets.php @@ -27,4 +27,25 @@ public function action_import() return Response::forge($theme->render()); } + + public function action_generate() + { + // Validate Logged in + if (\Service_User::verify_session() !== true ) throw new HttpNotFoundException; + + $theme = Theme::instance(); + $theme->set_template('layouts/react'); + $theme->get_template() + ->set('title', 'QSet Generation') + ->set('page_type', 'generate'); + + Js::push_inline('var BASE_URL = "'.Uri::base().'";'); + Js::push_inline('var WIDGET_URL = "'.Config::get('materia.urls.engines').'";'); + Js::push_inline('var STATIC_CROSSDOMAIN = "'.Config::get('materia.urls.static').'";'); + + Css::push_group(['qset_generation']); + Js::push_group(['react', 'qset_generator']); + + return Response::forge($theme->render()); + } } diff --git a/fuel/app/classes/materia/api/v1.php b/fuel/app/classes/materia/api/v1.php index 1083462ab..35ec7c450 100644 --- a/fuel/app/classes/materia/api/v1.php +++ b/fuel/app/classes/materia/api/v1.php @@ -822,6 +822,202 @@ static public function question_set_get($inst_id, $play_id = null, $timestamp = return $inst->qset; } + /** + * Generates a question set based on a given instance ID, topic, and whether to include images. + * @param object $input The input object containing the instance ID, topic, and whether to include images + * @return object The generated question set + */ + static public function question_set_generate($input) + { + // validate input + $inst_id = $input->inst_id; + $topic = $input->topic; + $include_images = $input->include_images; + \Log::info('Generating question set for instance '.$inst_id.' on topic '.$topic); + if ( ! Util_Validator::is_valid_hash($inst_id) ) return Msg::invalid_input($inst_id); + if ( ! ($inst = Widget_Instance_Manager::get($inst_id))) throw new \HttpNotFoundException; + if ( ! $inst->playable_by_current_user()) return Msg::no_login(); + + // get the widget and demo instance + $widget = $inst->widget; + $demo = Widget_Instance_Manager::get($widget->meta_data['demo']); + if ( ! $demo) throw new \HttpNotFoundException; + + // get the data to concatenate into the prompt + $instance_name = $inst->name; + $about = $widget->meta_data['about']; + // get the demo.json from the demo instance + $demo_qset = static::question_set_get($widget->meta_data['demo']); + if ( ! $demo_qset) throw new \HttpNotFoundException; + $qset_text = json_encode($demo_qset->data); + + // non-image prompt + $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Leave the asset fields empty. Add or remove the total number of questions generated to fit within the max tokens. ID's must be random.\n{$qset_text}"; + + // image prompt + if ($include_images) + { + $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Add or remove the total number of questions generated to fit within the max tokens. Replace the field values with generated values. In every asset, add a field titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system. ID's must be random.\n{$qset_text}"; + } + + \Log::info('Prompt text: '.$text); + + try { + // to access openai, define the openai key in the environment (.env file) + $my_api_key = \Config::get('materia.open_ai.api_key'); + $client = \OpenAI::client($my_api_key); + $result = $client->chat()->create([ + 'model' => 'gpt-3.5-turbo', + 'response_format' => (object) ['type' => 'json_object'], + 'messages' => [ + ['role' => 'user', 'content' => $text] + ], + 'max_tokens' => 2069, + 'frequency_penalty' => 0, // 0 to 1 + 'presence_penalty' => 0, // 0 to 1 + 'temperature' => 1, // 0 to 1 + 'top_p' => 1, // 0 to 1 + + ]); + + $question_set = json_decode($result->choices[0]->message->content); + \Log::info('Generated question set: '.print_r(json_encode($question_set), true)); + } catch (\Exception $e) { + \Log::error('Error generating question set: '.$e->getMessage()); + return new Msg(Msg::ERROR, 'Error generating question set'); + } finally { + \Log::info('Prompt tokens: '.$result->usage->promptTokens); + \Log::info('Completion tokens: '.$result->usage->completionTokens); + \Log::info('Total tokens: '.$result->usage->totalTokens); + } + + if ($include_images) + { + $image_rate_cap = 6; // any higher and the API will return an error + $assets = static::comb_assets($question_set); // get a list of all the asset descriptions + + // make sure we don't exceed the rate cap + $num_assets = count($assets); + if ($num_assets > $image_rate_cap) + { + $assets = array_slice($assets, 0, $image_rate_cap); + } + if ($num_assets < 1) + { + return $question_set; + } + // join assets into string + $assets_text = implode(', ', $assets); + // generate images + try { + $my_api_key = \Config::get('materia.open_ai.api_key'); + $client = \OpenAI::client($my_api_key); + $dalle_result = $client->images()->create([ + 'model' => 'dall-e-2', + 'prompt' => $assets_text, + 'n' => count($assets), + 'response_format' => 'b64_json', // urls available for only 60 minutes after + 'size' => '256x256' // 256x256, 512x512, 1024x1024 + ]); + } catch (\Exception $e) { + \Log::error('Error generating images: '.$e->getMessage()); + \Log::error('Trace: '.$e->getTraceAsString()); + return $question_set; + } + \Log::info('Generated images: '.print_r($dalle_result, true)); + + // Store assets in the database (permanent storage, not just URLs) + // for ($i = 0; $i < count($dalle_result->data); $i++) { + // $file_data = base64_decode($dalle_result->data[$i]->b64_json); + + // $src_area = \File::forge(['basedir' => sys_get_temp_dir()]); // restrict copying from system tmp dir + // $mock_upload_file_path = \Config::get('file.dirs.media_uploads').uniqid('sideload_') . '.png'; + // \File::copy($file_data, $mock_upload_file_path, $src_area, 'media'); + + // // process the upload + // $upload_info = \File::file_info($mock_upload_file_path, 'media'); + // $asset = \Materia\Widget_Asset_Manager::new_asset_from_file('Dalle asset', $upload_info); + + // if ( ! isset($asset->id)) { + // \Log::error('Unable to create asset'); + // } else { + // $asset->db_store(); + // $dalle_result->data[$i]->url = $asset->id; + // } + // } + + // assign generated images to assets in qset + static::assign_assets($question_set, $dalle_result->data, 0); + } + + \Log::info('Generated question set with assets: '.print_r(json_encode($question_set), true)); + + return $question_set; + } + + /** + * Combines all asset descriptions in a question set into a single array + * @param array $qset The question set array + * @return array The array of asset descriptions + */ + static public function comb_assets($qset) + { + $assets = []; + foreach ($qset as $key => $value) + { + if (is_object($value) || is_array($value)) + { + $value = (array) $value; + if ($key == 'asset' || $key == 'image' || $key == 'audio' || $key == 'video') + { + if (key_exists('description', $value) && ! empty($value['description'])) + { + $assets[] = $value['description']; + } + } + $assets = array_merge($assets, static::comb_assets($value)); + } + } + return $assets; + } + + /** + * Assigns generated images to assets in a question set + * @param array $array The question set array + * @param array $image_urls The array of image URLs + * @param int $image_index The index of the current image URL + * @return int The updated image index + */ + static public function assign_assets(&$array, $image_urls, $image_index) + { + if ( is_object($array) && isset($array->items)) $image_index = static::assign_assets($array->items, $image_urls, $image_index); + else if ( ! $array || ! is_array($array)) return $image_index; + + foreach ($array as $key => $value) + { + if ($image_index >= count($image_urls)) + { + return $image_index; + } + if (is_object($value) || is_array($value)) + { + $value = (array) $value; + if ($key == 'asset' || $key == 'image' || $key == 'audio' || $key == 'video') + { + if ( ! empty($value['description'])) + { + $base64 = $image_urls[$image_index]->b64_json; + $array[$key]->id = 'data:image/png;base64,'.$base64; + // $array[$key]->id = $image_urls[$image_index]->url; + $image_index += 1; + } + } + $image_index = static::assign_assets($value, $image_urls, $image_index); + } + } + return $image_index; + } + /** * Gets the question with the given QID or an array of questions * with the given ids (passed as an array) diff --git a/fuel/app/config/js.php b/fuel/app/config/js.php index f23e1cfb8..ea8a86260 100644 --- a/fuel/app/config/js.php +++ b/fuel/app/config/js.php @@ -32,6 +32,7 @@ '500' => [$webpack.'js/500.js'], 'media' => [$webpack.'js/media.js'], 'qset_history' => [$webpack.'js/qset-history.js'], + 'qset_generator' => [$webpack.'js/qset-generator.js'], 'post_login' => [$webpack.'js/lti-post-login.js'], 'select_item' => [$webpack.'js/lti-select-item.js'], 'open_preview' => [$webpack.'js/lti-open-preview.js'], diff --git a/fuel/app/config/materia.php b/fuel/app/config/materia.php index 0d8204309..e7de41dbb 100644 --- a/fuel/app/config/materia.php +++ b/fuel/app/config/materia.php @@ -119,6 +119,10 @@ ] : null ), + ], + + 'open_ai' => [ + 'api_key' => $_ENV['OPENAI_API_KEY'] ?? false, ] ]; diff --git a/fuel/app/tests/api/v1.php b/fuel/app/tests/api/v1.php index 2300c9529..7982cb097 100644 --- a/fuel/app/tests/api/v1.php +++ b/fuel/app/tests/api/v1.php @@ -1136,6 +1136,84 @@ public function test_question_set_get() } } + // BS to pass + public function test_question_set_generate() + { + // ======= AS NO ONE ======== + try { + $widget = $this->make_disposable_widget(); + $title = "Pixar Films"; + $question = "What was Pixar's first film?"; + $answer = "Toy Story"; + $qset = $this->create_new_qset($question, $answer); + $instance = Api_V1::widget_instance_new($widget->id, $title, $qset, false); + $inst_id = $instance->id; + // input object takes in inst_id, topic, and whether to include images + $input = (object) [ + 'inst_id' => $inst_id, + 'topic' => 'Disney Films', + 'include_images' => false + ]; + $output = Api_V1::question_set_generate($input); + $this->fail("Expected exception not thrown"); + } catch ( Exception $e) { + $this->assertInstanceOf('Exception', $e); + } + + // ======= AS AUTHOR ======= + // $this->_as_author(); + // $widget = $this->make_disposable_widget(); + // $title = "Pixar Films"; + // $question = "What was Pixar's first film?"; + // $answer = "Toy Story"; + // $qset = $this->create_new_qset($question, $answer); + // $instance = Api_V1::widget_instance_new($widget->id, $title, $qset, false); + // $this->assert_is_qset($qset); + // $inst_id = $instance->id; + // // input object takes in inst_id, topic, and whether to include images + // $input = (object) [ + // 'inst_id' => $inst_id, + // 'topic' => 'Disney Films', + // 'include_images' => false + // ]; + // $output = Api_V1::question_set_generate($input); + + // $questions = \Materia\Widget_Instance::find_questions($output); + // foreach ($questions as $question) + // { + // $this->assertInstanceOf('\Materia\Widget_Question', $question); + // if ($question instanceof \Materia\Widget_Question_Type_QA) $this->assert_question_is_qa($question); + // if ($question instanceof \Materia\Widget_Question_Type_MC) $this->assert_question_is_mc($question); + // } + } + + // BS to pass + public function test_comb_assets() + { + $question = "What was Pixar's first film?"; + $answer = "Toy Story"; + $qset = $this->create_new_qset($question, $answer); + $output = Api_V1::comb_assets($qset); + $this->assert_not_message($output); + $this->assertIsArray($output); + + } + + // BS to pass + public function test_assign_assets() + { + $question = "What was Pixar's first film?"; + $answer = "Toy Story"; + $qset = $this->create_new_qset($question, $answer); + + $image_urls = [ + 'https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png' + ]; + $output = Api_V1::assign_assets($qset, $image_urls, 0); + $this->assert_not_message($output); + $this->assertEquals($output, count($image_urls) - 1); + } + public function test_questions_get() { // ======= AS NO ONE ======== diff --git a/src/components/hooks/useQuestionGeneration.jsx b/src/components/hooks/useQuestionGeneration.jsx new file mode 100644 index 000000000..8c4b6120f --- /dev/null +++ b/src/components/hooks/useQuestionGeneration.jsx @@ -0,0 +1,13 @@ +import { useMutation } from "react-query"; +import { apiGenerateQset } from "../../util/api"; + +export default function useQuestionGeneration() { + return useMutation( + apiGenerateQset, + { + onSuccess: (qset, variables) => { + variables.successFunc(qset) + } + } + ) +} diff --git a/src/components/question-generator.jsx b/src/components/question-generator.jsx new file mode 100644 index 000000000..78bb9bedc --- /dev/null +++ b/src/components/question-generator.jsx @@ -0,0 +1,45 @@ +import useQuestionGeneration from "./hooks/useQuestionGeneration" +import React, { useState } from "react" + +const getInstId = () => { + const l = document.location.href + const id = l.substring(l.lastIndexOf('=') + 1) + return id +} + +const QsetGenerator = () => { + const generateQuestion = useQuestionGeneration() + + const [instId, setInstId] = useState(getInstId()) + const [topic, setTopic] = useState('') + const [includeImages, setIncludeImages] = useState(false) + + const onClickGenerate = () => { + generateQuestion.mutate({ + inst_id: instId, + topic: topic, + include_images: includeImages, + successFunc: (qset) => { + console.log(qset) + console.log(JSON.stringify(qset)) + let created_at = new Date().toISOString() + window.parent.Materia.Creator.onQsetHistorySelectionComplete(JSON.stringify(qset), 1, created_at) + } + }) + } + + const onTopicChange = (e) => { + setTopic(e.target.value) + } + + return ( +
+ + setIncludeImages(e.target.checked)}/> + + +
+ ) +} + +export default QsetGenerator; \ No newline at end of file diff --git a/src/components/widget-creator.jsx b/src/components/widget-creator.jsx index 34cd8a91e..16b6b92f1 100644 --- a/src/components/widget-creator.jsx +++ b/src/components/widget-creator.jsx @@ -478,6 +478,10 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { showEmbedDialog(`${window.BASE_URL}qsets/import/?inst_id=${instance.id}`, 'embed_dialog') } + const showQuestionGenerator = () => { + showEmbedDialog(`${window.BASE_URL}qsets/generate/?inst_id=${instance.id}`, 'embed_dialog') + } + // const showQsetHistoryConfirmation = () => { // } @@ -691,6 +695,7 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { { creatorState.hasCreatorGuide ? Creator's Guide : '' } { instance.id ? Save History : '' } Import Questions... + Generate Questions... { editButtonsRender }
) diff --git a/src/util/api.js b/src/util/api.js index 7e22637a6..ba5a0ccb5 100644 --- a/src/util/api.js +++ b/src/util/api.js @@ -599,8 +599,8 @@ export const apiGetQuestionSetHistory = (instId) => { }) } -export const apiGenerateQset = (inst_id, topic, include_images) => { - return fetch('/api/json/question_set_generate/', fetchOptions({ body: `data=${formatFetchBody([inst_id, topic, include_images])}` })) +export const apiGenerateQset = (inst_id, topic, include_images, num_questions) => { + return fetch('/api/json/question_set_generate/', fetchOptions({ body: `data=${formatFetchBody([inst_id, topic, include_images, num_questions])}` })) .then(resp => resp.json()) } From 76d11e115cb7164d726e11eb3ad86f72f488fb4c Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Tue, 30 Apr 2024 14:41:06 -0400 Subject: [PATCH 03/37] support qsets with 'assets' field --- fuel/app/classes/materia/api/v1.php | 51 ++++++++++++++++++++++++--- src/components/question-generator.jsx | 2 -- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/fuel/app/classes/materia/api/v1.php b/fuel/app/classes/materia/api/v1.php index cf502a204..1e0a2cae0 100644 --- a/fuel/app/classes/materia/api/v1.php +++ b/fuel/app/classes/materia/api/v1.php @@ -860,7 +860,7 @@ static public function question_set_generate($input) // image prompt if ($include_images) { - $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Generate a total of {$num_questions} questions. In every asset, add a field titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system. ID's must be random.\n{$qset_text}"; + $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Generate a total of {$num_questions} questions. In every asset or assets field, add a field to each asset object titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system. ID's must be random.\n{$qset_text}"; } \Log::info('Prompt text: '.$text); @@ -901,6 +901,7 @@ static public function question_set_generate($input) // make sure we don't exceed the rate cap $num_assets = count($assets); + \Log::info('Number of assets: '.$num_assets); if ($num_assets > $image_rate_cap) { $assets = array_slice($assets, 0, $image_rate_cap); @@ -919,7 +920,7 @@ static public function question_set_generate($input) 'model' => 'dall-e-2', 'prompt' => $assets_text, 'n' => count($assets), - 'response_format' => 'b64_json', // urls available for only 60 minutes after + 'response_format' => 'url', // urls available for only 60 minutes after 'size' => '256x256' // 256x256, 512x512, 1024x1024 ]); } catch (\Exception $e) { @@ -978,6 +979,18 @@ static public function comb_assets($qset) $assets[] = $value['description']; } } + if ($key == 'assets') + { + $value = (array) $value; + foreach ($value as $asset) + { + $asset = (array) $asset; + if (key_exists('description', $asset) && ! empty($asset['description'])) + { + $assets[] = $asset['description']; + } + } + } $assets = array_merge($assets, static::comb_assets($value)); } } @@ -1009,12 +1022,40 @@ static public function assign_assets(&$array, $image_urls, $image_index) { if ( ! empty($value['description'])) { - $base64 = $image_urls[$image_index]->b64_json; - $array[$key]->id = 'data:image/png;base64,'.$base64; - // $array[$key]->id = $image_urls[$image_index]->url; + // base 64 + // $base64 = $image_urls[$image_index]->b64_json; + // $array[$key]->id = 'data:image/png;base64,'.$base64; + // $array[$key]->url = $image_urls[$image_index]->b64_json; + + // url + $array[$key]->id = $image_urls[$image_index]->url; + $array[$key]->url = $image_urls[$image_index]->url; + $image_index += 1; } } + if ($key == 'assets') + { + // iterate over assets array without converting to array + // to avoid losing object properties + foreach ($value as $asset) + { + \Log::info('asset: '.print_r($asset, true)); + if ( ! empty($asset->description)) + { + // b64 + // $base64 = $image_urls[$image_index]->b64_json; + // $asset->id = 'data:image/png;base64,'.$base64; + // $asset->url = $image_urls[$image_index]->b64_json; + + // url + $asset->url = $image_urls[$image_index]->url; + $asset->id = $image_urls[$image_index]->url; + + $image_index += 1; + } + } + } $image_index = static::assign_assets($value, $image_urls, $image_index); } } diff --git a/src/components/question-generator.jsx b/src/components/question-generator.jsx index 0d343b099..8b6d9bda3 100644 --- a/src/components/question-generator.jsx +++ b/src/components/question-generator.jsx @@ -22,8 +22,6 @@ const QsetGenerator = () => { include_images: includeImages, num_questions: numQuestions, successFunc: (qset) => { - console.log(qset) - console.log(JSON.stringify(qset)) let created_at = new Date().toISOString() window.parent.Materia.Creator.onQsetHistorySelectionComplete(JSON.stringify(qset), 1, created_at) } From 77282f3eedd4ca8009b0373b7032da835c28cb6e Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Tue, 30 Apr 2024 15:06:49 -0400 Subject: [PATCH 04/37] if generating images, remove real names --- fuel/app/classes/materia/api/v1.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuel/app/classes/materia/api/v1.php b/fuel/app/classes/materia/api/v1.php index 1e0a2cae0..de957d476 100644 --- a/fuel/app/classes/materia/api/v1.php +++ b/fuel/app/classes/materia/api/v1.php @@ -860,7 +860,7 @@ static public function question_set_generate($input) // image prompt if ($include_images) { - $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Generate a total of {$num_questions} questions. In every asset or assets field, add a field to each asset object titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system. ID's must be random.\n{$qset_text}"; + $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Generate a total of {$num_questions} questions. Do not use real names. In every asset or assets field, add a field to each asset object titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system. ID's must be random.\n{$qset_text}"; } \Log::info('Prompt text: '.$text); From c2ba53f4993b8b97786d307c52ba9f0a59d2ddac Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Wed, 8 May 2024 08:06:13 -0400 Subject: [PATCH 05/37] Add option to extend existing qset --- fuel/app/classes/materia/api/v1.php | 112 +++++++++++++++++--------- fuel/app/tests/api/v1.php | 2 +- src/components/question-generator.jsx | 4 + src/util/api.js | 4 +- 4 files changed, 81 insertions(+), 41 deletions(-) diff --git a/fuel/app/classes/materia/api/v1.php b/fuel/app/classes/materia/api/v1.php index de957d476..ee29d36cd 100644 --- a/fuel/app/classes/materia/api/v1.php +++ b/fuel/app/classes/materia/api/v1.php @@ -834,6 +834,7 @@ static public function question_set_generate($input) $topic = $input->topic; $include_images = $input->include_images; $num_questions = $input->num_questions; + $build_off_existing = $input->build_off_existing; \Log::info('num_questions: '.$num_questions); \Log::info('num_questions to string: '.strval($num_questions)); \Log::info('Generating question set for instance '.$inst_id.' on topic '.$topic); @@ -841,26 +842,52 @@ static public function question_set_generate($input) if ( ! ($inst = Widget_Instance_Manager::get($inst_id))) throw new \HttpNotFoundException; if ( ! $inst->playable_by_current_user()) return Msg::no_login(); - // get the widget and demo instance - $widget = $inst->widget; - $demo = Widget_Instance_Manager::get($widget->meta_data['demo']); - if ( ! $demo) throw new \HttpNotFoundException; + if ($build_off_existing) + { + $qset = static::question_set_get($inst_id); + if ( ! $qset) return new Msg(Msg::ERROR, 'No existing question set found'); - // get the data to concatenate into the prompt - $instance_name = $inst->name; - $about = $widget->meta_data['about']; - // get the demo.json from the demo instance - $demo_qset = static::question_set_get($widget->meta_data['demo']); - if ( ! $demo_qset) throw new \HttpNotFoundException; - $qset_text = json_encode($demo_qset->data); + $qset_text = json_encode($qset->data); - // non-image prompt - $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Generate a total {$num_questions} of questions. Leave the asset fields empty. ID's must be random.\n{$qset_text}"; + // non-demo non-image prompt + $text = "Using the exact same format of the following question set without changing any field keys or data types and without changing any of the existing questions, generate {$num_questions} more questions and add them to the existing qset. The new questions must be based on this topic: '{$topic}'. Return only the JSON for the resulting question set."; - // image prompt - if ($include_images) + if ($include_images) + { + $text = $text."Do not use real names. In every asset or assets field in new questions, add a field to each asset object titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system. ID's must be random.\n{$qset_text}"; + } + else + { + $text = $text."Leave the asset fields empty. ID's must be random.\n{$qset_text}"; + } + } + else { - $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Generate a total of {$num_questions} questions. Do not use real names. In every asset or assets field, add a field to each asset object titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system. ID's must be random.\n{$qset_text}"; + // get the widget and demo instance + $widget = $inst->widget; + $demo = Widget_Instance_Manager::get($widget->meta_data['demo']); + if ( ! $demo) throw new \HttpNotFoundException; + + // get the data to concatenate into the prompt + $instance_name = $inst->name; + $about = $widget->meta_data['about']; + // get the demo.json from the demo instance + $demo_qset = static::question_set_get($widget->meta_data['demo']); + if ( ! $demo_qset) throw new \HttpNotFoundException; + $qset_text = json_encode($demo_qset->data); + + // non-image prompt + $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Generate a total {$num_questions} of questions."; + + // image prompt + if ($include_images) + { + $text = $text."Do not use real names. In every asset or assets field, add a field to each asset object titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system. ID's must be random.\n{$qset_text}"; + } + else + { + $text = $text."Do not generate image-type questions/answers, only text-type questions/answers. Therefore, leave the asset fields empty for image, video, or audio questions/answers, but NOT text-type. If the 'materiaType' of an asset is 'text', create a field titled 'value' with the question/answer text insidet the asset object. ID's must be random.\n{$qset_text}"; + } } \Log::info('Prompt text: '.$text); @@ -901,10 +928,15 @@ static public function question_set_generate($input) // make sure we don't exceed the rate cap $num_assets = count($assets); + $start_offset = 0; \Log::info('Number of assets: '.$num_assets); if ($num_assets > $image_rate_cap) { - $assets = array_slice($assets, 0, $image_rate_cap); + if ($build_off_existing) + { + $start_offset = $num_assets - $image_rate_cap; + } + $assets = array_slice($assets, $start_offset, $image_rate_cap); } if ($num_assets < 1) { @@ -951,7 +983,7 @@ static public function question_set_generate($input) // } // assign generated images to assets in qset - static::assign_assets($question_set, $dalle_result->data, 0); + static::assign_assets($question_set, $dalle_result->data, $start_offset, 0); } \Log::info('Generated question set with assets: '.print_r(json_encode($question_set), true)); @@ -1004,9 +1036,9 @@ static public function comb_assets($qset) * @param int $image_index The index of the current image URL * @return int The updated image index */ - static public function assign_assets(&$array, $image_urls, $image_index) + static public function assign_assets(&$array, $image_urls, $start_offset, $image_index) { - if ( is_object($array) && isset($array->items)) $image_index = static::assign_assets($array->items, $image_urls, $image_index); + if ( is_object($array) && isset($array->items)) $image_index = static::assign_assets($array->items, $image_urls, $start_offset, $image_index); else if ( ! $array || ! is_array($array)) return $image_index; foreach ($array as $key => $value) @@ -1022,15 +1054,17 @@ static public function assign_assets(&$array, $image_urls, $image_index) { if ( ! empty($value['description'])) { - // base 64 - // $base64 = $image_urls[$image_index]->b64_json; - // $array[$key]->id = 'data:image/png;base64,'.$base64; - // $array[$key]->url = $image_urls[$image_index]->b64_json; - - // url - $array[$key]->id = $image_urls[$image_index]->url; - $array[$key]->url = $image_urls[$image_index]->url; + if ($image_index >= $start_offset) + { + // b64 + // $base64 = $image_urls[$image_index]->b64_json; + // $array[$key]->id = 'data:image/png;base64,'.$base64; + // $array[$key]->url = $image_urls[$image_index]->b64_json; + // url + $array[$key]->url = $image_urls[$image_index]->url; + $array[$key]->id = $image_urls[$image_index]->url; + } $image_index += 1; } } @@ -1043,20 +1077,22 @@ static public function assign_assets(&$array, $image_urls, $image_index) \Log::info('asset: '.print_r($asset, true)); if ( ! empty($asset->description)) { - // b64 - // $base64 = $image_urls[$image_index]->b64_json; - // $asset->id = 'data:image/png;base64,'.$base64; - // $asset->url = $image_urls[$image_index]->b64_json; - - // url - $asset->url = $image_urls[$image_index]->url; - $asset->id = $image_urls[$image_index]->url; - + if ($image_index >= $start_offset) + { + // b64 + // $base64 = $image_urls[$image_index]->b64_json; + // $asset->id = 'data:image/png;base64,'.$base64; + // $asset->url = $image_urls[$image_index]->b64_json; + + // url + $asset->url = $image_urls[$image_index]->url; + $asset->id = $image_urls[$image_index]->url; + } $image_index += 1; } } } - $image_index = static::assign_assets($value, $image_urls, $image_index); + $image_index = static::assign_assets($value, $image_urls, $start_offset, $image_index); } } return $image_index; diff --git a/fuel/app/tests/api/v1.php b/fuel/app/tests/api/v1.php index 7982cb097..dace83c90 100644 --- a/fuel/app/tests/api/v1.php +++ b/fuel/app/tests/api/v1.php @@ -1209,7 +1209,7 @@ public function test_assign_assets() $image_urls = [ 'https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png' ]; - $output = Api_V1::assign_assets($qset, $image_urls, 0); + $output = Api_V1::assign_assets($qset, $image_urls, 0, 0); $this->assert_not_message($output); $this->assertEquals($output, count($image_urls) - 1); } diff --git a/src/components/question-generator.jsx b/src/components/question-generator.jsx index 8b6d9bda3..aa4db643b 100644 --- a/src/components/question-generator.jsx +++ b/src/components/question-generator.jsx @@ -14,6 +14,7 @@ const QsetGenerator = () => { const [topic, setTopic] = useState('') const [includeImages, setIncludeImages] = useState(false) const [numQuestions, setNumQuestions] = useState(1) + const [buildOffExisting, setBuildOffExisting] = useState(false) const onClickGenerate = () => { generateQuestion.mutate({ @@ -21,6 +22,7 @@ const QsetGenerator = () => { topic: topic, include_images: includeImages, num_questions: numQuestions, + build_off_existing: buildOffExisting, successFunc: (qset) => { let created_at = new Date().toISOString() window.parent.Materia.Creator.onQsetHistorySelectionComplete(JSON.stringify(qset), 1, created_at) @@ -37,6 +39,8 @@ const QsetGenerator = () => { setIncludeImages(e.target.checked)}/> + setBuildOffExisting(e.target.checked)}/> + setNumQuestions(e.target.value)}/> diff --git a/src/util/api.js b/src/util/api.js index ba5a0ccb5..7e5973579 100644 --- a/src/util/api.js +++ b/src/util/api.js @@ -599,8 +599,8 @@ export const apiGetQuestionSetHistory = (instId) => { }) } -export const apiGenerateQset = (inst_id, topic, include_images, num_questions) => { - return fetch('/api/json/question_set_generate/', fetchOptions({ body: `data=${formatFetchBody([inst_id, topic, include_images, num_questions])}` })) +export const apiGenerateQset = (inst_id, topic, include_images, num_questions, build_off_existing) => { + return fetch('/api/json/question_set_generate/', fetchOptions({ body: `data=${formatFetchBody([inst_id, topic, include_images, num_questions, build_off_existing])}` })) .then(resp => resp.json()) } From c2a2881ee7c634b3fa42f26d2abf18a94102f01e Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Tue, 11 Jun 2024 12:57:39 -0400 Subject: [PATCH 06/37] Tweak prompt, fix merge conflicts --- fuel/app/classes/materia/api/v1.php | 67 ++++++++++++++++++++++++----- src/util/api.js | 3 +- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/fuel/app/classes/materia/api/v1.php b/fuel/app/classes/materia/api/v1.php index a4029b8fb..69fcbf07a 100644 --- a/fuel/app/classes/materia/api/v1.php +++ b/fuel/app/classes/materia/api/v1.php @@ -109,7 +109,7 @@ static public function widget_instance_access_perms_verify($inst_id) * @return object, contains properties indicating whether the current * user can edit the widget and a message object describing why, if not */ - + // !! this endpoint should be significantly refactored or removed in the future API overhaul !! static public function widget_instance_edit_perms_verify(string $inst_id) { @@ -849,6 +849,10 @@ static public function question_set_generate($input) // validate input $inst_id = $input->inst_id; $topic = $input->topic; + + // clean topic of any special characters + $topic = preg_replace('/[^a-zA-Z0-9\s]/', '', $topic); + $include_images = $input->include_images; $num_questions = $input->num_questions; $build_off_existing = $input->build_off_existing; @@ -859,6 +863,10 @@ static public function question_set_generate($input) if ( ! ($inst = Widget_Instance_Manager::get($inst_id))) throw new \HttpNotFoundException; if ( ! $inst->playable_by_current_user()) return Msg::no_login(); + $widget_name = ''; + $start_time = microtime(true); + $time_elapsed_secs = 0; + if ($build_off_existing) { $qset = static::question_set_get($inst_id); @@ -887,6 +895,7 @@ static public function question_set_generate($input) // get the data to concatenate into the prompt $instance_name = $inst->name; + $widget_name = $widget->name; $about = $widget->meta_data['about']; // get the demo.json from the demo instance $demo_qset = static::question_set_get($widget->meta_data['demo']); @@ -894,16 +903,16 @@ static public function question_set_generate($input) $qset_text = json_encode($demo_qset->data); // non-image prompt - $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Generate a total {$num_questions} of questions."; + $text = "{$instance_name} is a {$widget->name} widget, described as: '{$about}'. The following is a question set storing an example instance called {$demo->name}. Using the exact same format without changing any field keys or data types, return only the JSON for a question set based on this topic: '{$topic}'. Ignore the demo instance topic entirely. Replace the field values with generated values. Generate a total {$num_questions} of questions. IDs must be random."; // image prompt if ($include_images) { - $text = $text."Do not use real names. In every asset or assets field, add a field to each asset object titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system. ID's must be random.\n{$qset_text}"; + $text = $text."Do not use real names. Find the field storing image assets. This could be labeled as an asset, assets, image field or similar. Add a field to each asset titled 'description' that best describes the image within the answer or question's context. Do not generate descriptions that would violate OpenAI's image generation safety system.\n{$qset_text}"; } else { - $text = $text."Do not generate image-type questions/answers, only text-type questions/answers. Therefore, leave the asset fields empty for image, video, or audio questions/answers, but NOT text-type. If the 'materiaType' of an asset is 'text', create a field titled 'value' with the question/answer text insidet the asset object. ID's must be random.\n{$qset_text}"; + $text = $text."Do not generate image-type questions/answers, only text-type questions/answers. Therefore, leave the asset fields empty for image, video, or audio questions/answers, but NOT text-type. If the 'materiaType' of an asset is 'text', create a field titled 'value' with the question/answer text insidet the asset object.\n{$qset_text}"; } } @@ -929,13 +938,38 @@ static public function question_set_generate($input) $question_set = json_decode($result->choices[0]->message->content); \Log::info('Generated question set: '.print_r(json_encode($question_set), true)); + + $time_elapsed_secs = microtime(true) - $start_time; + $cost_input_tokens = 0.50 / 1000000; // $0.50 per 1 million tokens + $cost_output_tokens = 1.50 / 1000000; // $1.50 per 1 million tokens + + $file = fopen('openai_usage.txt', 'a'); + fwrite($file, PHP_EOL); + fwrite($file, 'Widget: '.$widget_name.PHP_EOL); + fwrite($file, 'Date: '.date('Y-m-d H:i:s').PHP_EOL); + fwrite($file, 'Time to complete (in seconds): '.$time_elapsed_secs.PHP_EOL); + fwrite($file, 'Number of questions asked to generate: '.$num_questions.PHP_EOL); + fwrite($file, 'Included images: '.$include_images.PHP_EOL); + fwrite($file, 'Prompt tokens: '.$result->usage->promptTokens.PHP_EOL); + fwrite($file, 'Completion tokens: '.$result->usage->completionTokens.PHP_EOL); + fwrite($file, 'Total tokens: '.$result->usage->totalTokens.PHP_EOL); + fwrite($file, 'Total cost (in dollars): '.$result->usage->promptTokens * $cost_input_tokens + $result->usage->completionTokens * $cost_output_tokens.PHP_EOL); + fclose($file); + } catch (\Exception $e) { \Log::error('Error generating question set: '.$e->getMessage()); + + $file = fopen('openai_usage.txt', 'a'); + fwrite($file, PHP_EOL); + fwrite($file, 'Widget: '.$widget_name.PHP_EOL); + fwrite($file, 'Date: '.date('Y-m-d H:i:s').PHP_EOL); + fwrite($file, 'Time to complete (in seconds): '.$time_elapsed_secs.PHP_EOL); + fwrite($file, 'Number of questions asked to generate: '.$num_questions.PHP_EOL); + fwrite($file, 'Error: '.$e->getMessage().PHP_EOL); + + fclose($file); + return new Msg(Msg::ERROR, 'Error generating question set'); - } finally { - \Log::info('Prompt tokens: '.$result->usage->promptTokens); - \Log::info('Completion tokens: '.$result->usage->completionTokens); - \Log::info('Total tokens: '.$result->usage->totalTokens); } if ($include_images) @@ -972,11 +1006,22 @@ static public function question_set_generate($input) 'response_format' => 'url', // urls available for only 60 minutes after 'size' => '256x256' // 256x256, 512x512, 1024x1024 ]); + } catch (\Exception $e) { \Log::error('Error generating images: '.$e->getMessage()); \Log::error('Trace: '.$e->getTraceAsString()); + + $file = fopen('openai_usage.txt', 'a'); + fwrite($file, 'Error generating images: '.$e->getMessage().PHP_EOL); + fclose(); + return $question_set; } + + $file = fopen('openai_usage.txt', 'a'); + fwrite($file, 'Generated images.'); + fclose(); + \Log::info('Generated images: '.print_r($dalle_result, true)); // Store assets in the database (permanent storage, not just URLs) @@ -1021,7 +1066,7 @@ static public function comb_assets($qset) if (is_object($value) || is_array($value)) { $value = (array) $value; - if ($key == 'asset' || $key == 'image' || $key == 'audio' || $key == 'video') + if ($key == 'asset' || $key == 'image' || $key == 'audio' || $key == 'video' || $key == 'options') { if (key_exists('description', $value) && ! empty($value['description'])) { @@ -1067,7 +1112,7 @@ static public function assign_assets(&$array, $image_urls, $start_offset, $image if (is_object($value) || is_array($value)) { $value = (array) $value; - if ($key == 'asset' || $key == 'image' || $key == 'audio' || $key == 'video') + if ($key == 'asset' || $key == 'image' || $key == 'audio' || $key == 'video' || $key == 'options') { if ( ! empty($value['description'])) { @@ -1077,10 +1122,12 @@ static public function assign_assets(&$array, $image_urls, $start_offset, $image // $base64 = $image_urls[$image_index]->b64_json; // $array[$key]->id = 'data:image/png;base64,'.$base64; // $array[$key]->url = $image_urls[$image_index]->b64_json; + // $array[$key]->image = $image_urls[$image_index]->b64_json; // url $array[$key]->url = $image_urls[$image_index]->url; $array[$key]->id = $image_urls[$image_index]->url; + $array[$key]->image = $image_urls[$image_index]->url; } $image_index += 1; } diff --git a/src/util/api.js b/src/util/api.js index 34b2a26ec..0d0c67130 100644 --- a/src/util/api.js +++ b/src/util/api.js @@ -347,8 +347,7 @@ export const apiGetQuestionSet = (instId, playId = null) => { } export const apiGenerateQset = (inst_id, topic, include_images, num_questions, build_off_existing) => { - return fetch('/api/json/question_set_generate/', fetchOptions({ body: `data=${formatFetchBody([inst_id, topic, include_images, num_questions, build_off_existing])}` })) - .then(resp => resp.json()) + return fetchGet('/api/json/question_set_generate/', ({ body: `data=${formatFetchBody([inst_id, topic, include_images, num_questions, build_off_existing])}` })) } export const apiSessionVerify = (play_id) => { From 35eebfe78b1da3fd8747e6146f74bc4cb5995956 Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Tue, 11 Jun 2024 14:42:44 -0400 Subject: [PATCH 07/37] Make UI pretty --- fuel/app/classes/controller/qsets.php | 2 +- fuel/app/classes/materia/api/v1.php | 10 ++ fuel/app/config/css.php | 1 + .../hooks/useQuestionGeneration.jsx | 3 + src/components/question-generator.jsx | 102 ++++++++++++++++-- src/components/question-generator.scss | 92 ++++++++++++++++ 6 files changed, 201 insertions(+), 9 deletions(-) create mode 100644 src/components/question-generator.scss diff --git a/fuel/app/classes/controller/qsets.php b/fuel/app/classes/controller/qsets.php index c44041504..fa1c1d119 100644 --- a/fuel/app/classes/controller/qsets.php +++ b/fuel/app/classes/controller/qsets.php @@ -43,7 +43,7 @@ public function action_generate() Js::push_inline('var WIDGET_URL = "'.Config::get('materia.urls.engines').'";'); Js::push_inline('var STATIC_CROSSDOMAIN = "'.Config::get('materia.urls.static').'";'); - Css::push_group(['qset_generation']); + Css::push_group(['qset_generator']); Js::push_group(['react', 'qset_generator']); return Response::forge($theme->render()); diff --git a/fuel/app/classes/materia/api/v1.php b/fuel/app/classes/materia/api/v1.php index 69fcbf07a..8cdb4c703 100644 --- a/fuel/app/classes/materia/api/v1.php +++ b/fuel/app/classes/materia/api/v1.php @@ -853,12 +853,21 @@ static public function question_set_generate($input) // clean topic of any special characters $topic = preg_replace('/[^a-zA-Z0-9\s]/', '', $topic); + // count words in topic + $topic_words = explode(' ', $topic); + if (count($topic_words) < 3) return new Msg(Msg::ERROR, 'Topic must be at least 3 words long'); + $include_images = $input->include_images; + $num_questions = $input->num_questions; + if ($num_questions < 1) $num_questions = 8; + $build_off_existing = $input->build_off_existing; + \Log::info('num_questions: '.$num_questions); \Log::info('num_questions to string: '.strval($num_questions)); \Log::info('Generating question set for instance '.$inst_id.' on topic '.$topic); + if ( ! Util_Validator::is_valid_hash($inst_id) ) return Msg::invalid_input($inst_id); if ( ! ($inst = Widget_Instance_Manager::get($inst_id))) throw new \HttpNotFoundException; if ( ! $inst->playable_by_current_user()) return Msg::no_login(); @@ -866,6 +875,7 @@ static public function question_set_generate($input) $widget_name = ''; $start_time = microtime(true); $time_elapsed_secs = 0; + return new Msg(Msg::ERROR, 'Error generating question set'); if ($build_off_existing) { diff --git a/fuel/app/config/css.php b/fuel/app/config/css.php index a00308cf2..93e077d30 100644 --- a/fuel/app/config/css.php +++ b/fuel/app/config/css.php @@ -38,6 +38,7 @@ $webpack.'css/util-question-import.css', $webpack.'css/question-importer.css', ], + 'qset_generator' => [$webpack.'css/qset-generator.css'], 'questionimport' => [$webpack.'css/question-importer.css'], 'qset_history' => [$webpack.'css/qset-history.css'], 'rollback_dialog' => [$webpack.'css/util-rollback-confirm.css'], diff --git a/src/components/hooks/useQuestionGeneration.jsx b/src/components/hooks/useQuestionGeneration.jsx index 8c4b6120f..243b6d2a3 100644 --- a/src/components/hooks/useQuestionGeneration.jsx +++ b/src/components/hooks/useQuestionGeneration.jsx @@ -7,6 +7,9 @@ export default function useQuestionGeneration() { { onSuccess: (qset, variables) => { variables.successFunc(qset) + }, + onError: (error, variables, context) => { + variables.errorFunc(error) } } ) diff --git a/src/components/question-generator.jsx b/src/components/question-generator.jsx index aa4db643b..5090529e3 100644 --- a/src/components/question-generator.jsx +++ b/src/components/question-generator.jsx @@ -1,5 +1,7 @@ import useQuestionGeneration from "./hooks/useQuestionGeneration" import React, { useState } from "react" +import './question-generator.scss' +import LoadingIcon from './loading-icon' const getInstId = () => { const l = document.location.href @@ -13,10 +15,24 @@ const QsetGenerator = () => { const [instId, setInstId] = useState(getInstId()) const [topic, setTopic] = useState('') const [includeImages, setIncludeImages] = useState(false) - const [numQuestions, setNumQuestions] = useState(1) + const [numQuestions, setNumQuestions] = useState(8) const [buildOffExisting, setBuildOffExisting] = useState(false) + const [topicError, setTopicError] = useState('') + const [numberError, setNumberError] = useState('') + const [warning, setWarning] = useState('') + const [loading, setLoading] = useState(false) + const [serverError, setServerError] = useState('') + const onClickGenerate = () => { + if (loading) return + + let is_valid = validateTopic() + let is_valid_num = validateNumQuestions() + if (!is_valid || !is_valid_num) return + + setLoading(true) + generateQuestion.mutate({ inst_id: instId, topic: topic, @@ -26,23 +42,93 @@ const QsetGenerator = () => { successFunc: (qset) => { let created_at = new Date().toISOString() window.parent.Materia.Creator.onQsetHistorySelectionComplete(JSON.stringify(qset), 1, created_at) + setLoading(false) + }, + errorFunc: (err) => { + console.error(err) + setServerError('Error generating questions. Please try again.') + setLoading(false) } }) } + const closeDialog = () => window.parent.Materia.Creator.onQsetHistorySelectionComplete(null) + + const validateTopic = () => { + let words_in_topic = topic.split(' ') + + if (words_in_topic.length < 3) { + document.getElementById('topic').classList.add('invalid') + setTopicError('Please enter a topic with at least 3 words') + return false + } + else { + document.getElementById('topic').classList.remove('invalid') + setTopicError('') + return true + } + } + + const validateNumQuestions = () => { + if (numQuestions < 1) { + document.getElementById('num-questions').classList.add('invalid') + setNumberError('Please enter a number greater than 0') + return false + } else if (numQuestions > 8) { + document.getElementById('num-questions').classList.add('warning') + setWarning('Note: Generating this many questions may not work, or will take a while.') + return true + } else { + document.getElementById('num-questions').classList.remove('invalid') + document.getElementById('num-questions').classList.remove('warning') + setNumberError('') + return true + } + } + const onTopicChange = (e) => { setTopic(e.target.value) + validateTopic() + } + + const onNumberChange = (e) => { + setNumQuestions(e.target.value) + validateNumQuestions() } return (
- - setIncludeImages(e.target.checked)}/> - - setBuildOffExisting(e.target.checked)}/> - - setNumQuestions(e.target.value)}/> - +

Generate Questions

+ {loading &&
+ +

Generating questions. Do not close this window.

+
} +
+ {serverError} +
+ + {topicError} + +
+
+ + {numberError} + +
+
+ setIncludeImages(e.target.checked)}/> + +
+
+ setBuildOffExisting(e.target.checked)}/> + +
+ {warning} + +
+
+ Cancel +
) } diff --git a/src/components/question-generator.scss b/src/components/question-generator.scss new file mode 100644 index 000000000..69da6b4a4 --- /dev/null +++ b/src/components/question-generator.scss @@ -0,0 +1,92 @@ +@import './include.scss'; + +.generate { + h1 { + background: #f1814b; + padding: 6px 10px; + margin: 0px; + font-size: 1em; + font-weight: bold; + color: #ffffff; + } + + .loading { + display: flex; + justify-content: center; + align-items: center; + height: 94vh; + background-color: rgba(0, 0, 0, 0.5); + + backdrop-filter: blur(10px); + position: absolute; + top: 2em; + left: 0; + width: 100%; + z-index: 10; + + p { + margin-bottom: 150px; + font-size: 1.1em; + color: white; + } + } + + #generate_form { + margin: 2em; + display: flex; + flex-direction: column; + gap: 1em; + + #topic-field { + display: flex; + flex-direction: column; + gap: 0.5em; + + #topic { + padding: 0.5em; + } + } + + #num-questions-field { + display: flex; + flex-direction: row; + justify-content: space-between; + + input { + max-width: 100px; + } + } + + } + .actions { + position: fixed; + width: 80px; + left: 50%; + bottom: 0px; + margin-left: -40px; + padding-bottom: 14px; + z-index: 10; + + text-align: center; + font-size: 1.1em; + + a { + color: #000000; + } + } + + .error { + color: red; + font-size: 0.8em; + } + + .invalid { + border: 2px solid red; + border-radius: 3px; + } + + .warning { + color: #f1814b; + font-size: 0.8em; + } +} From f40499037e0b5223810bd799a004ebfde76de8a3 Mon Sep 17 00:00:00 2001 From: Cay Henning Date: Tue, 11 Jun 2024 14:50:44 -0400 Subject: [PATCH 08/37] Hide generator for unsaved new instances --- src/components/widget-creator.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/widget-creator.jsx b/src/components/widget-creator.jsx index 55de58d1f..d6f991ba8 100644 --- a/src/components/widget-creator.jsx +++ b/src/components/widget-creator.jsx @@ -719,7 +719,7 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { { creatorState.hasCreatorGuide ? Creator's Guide : '' } { instance.id ? Save History : '' } Import Questions... - Generate Questions... + { instId ? Generate Questions... : <> } { editButtonsRender }
- + + + + ) + } + + let generationConfirmBarRender = null + if (creatorState.showGenerationConfirm) { + generationConfirmBarRender = ( +
+

Previewing Generated Questions

+

Select Cancel to undo any changes made by the question generator. Select Keep to commit to using this generated version.

+ +
) } @@ -823,6 +839,7 @@ const WidgetCreator = ({instId, widgetId, minHeight='', minWidth=''}) => { { popupRender } { actionBarRender } { rollbackConfirmBarRender } + { generationConfirmBarRender }