diff --git a/eslint.config.js b/eslint.config.js index dbd80cee4..9b41b1adc 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -61,7 +61,7 @@ module.exports = defineConfig([ }], }, }, - globalIgnores(["**/.eslintrc.js", "./langchain-services/**/*"]), + globalIgnores(["**/.eslintrc.js", "./langchain-services/**/*", "packages/frontend/webworkers/*"]), { files: ["**/*.entity.ts"], diff --git a/package.json b/package.json index a88ed8a89..cd73517af 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "node-fullstack-websocket", + "name": "helpme-server", "version": "0.1.0", "private": true, "scripts": { diff --git a/packages/common/chatbot-api-types.ts b/packages/common/chatbot-api-types.ts new file mode 100644 index 000000000..d9bd2a618 --- /dev/null +++ b/packages/common/chatbot-api-types.ts @@ -0,0 +1,755 @@ +import { Transform, Type } from 'class-transformer' +import { + IsArray, + IsBoolean, + IsDate, + IsEnum, + IsInstance, + IsInt, + IsNumber, + IsObject, + IsOptional, + IsString, + ValidateNested, +} from 'class-validator' + +/* KEEP UP TO DATE */ + +export enum DocumentType { + Inserted = 'inserted_document', + InsertedLMS = 'inserted_lms_document', + InsertedQuestion = 'inserted_question', + PDF = 'pdf', + PPTX = 'pptx', + DOCX = 'docx', + MD = 'md', + TXT = 'txt', + CSV = 'csv', + TSV = 'tsv', +} + +export const DocumentTypeDisplayMap = { + [DocumentType.Inserted]: 'Inserted', + [DocumentType.InsertedLMS]: 'LMS', + [DocumentType.InsertedQuestion]: 'Inserted Question', + [DocumentType.PDF]: 'PDF', + [DocumentType.PPTX]: 'PPTX', + [DocumentType.DOCX]: 'DOCX', + [DocumentType.MD]: 'MD', + [DocumentType.TXT]: 'TXT', + [DocumentType.CSV]: 'CSV', + [DocumentType.TSV]: 'TSV', +} + +export const DocumentTypeColorMap = { + [DocumentType.Inserted]: '#5E60CE', + [DocumentType.InsertedLMS]: '#4EA8DE', + [DocumentType.InsertedQuestion]: '#56CFE1', + [DocumentType.PDF]: '#E63946', + [DocumentType.PPTX]: '#F77F00', + [DocumentType.DOCX]: '#457B9D', + [DocumentType.MD]: '#CA00CA', + [DocumentType.TXT]: '#00D844', + [DocumentType.CSV]: '#FFB703', + [DocumentType.TSV]: '#8338EC', +} + +export enum ChatbotQueryTypeEnum { + DEFAULT = 'default', + ABSTRACT = 'abstract', +} + +export class ChatMessage { + @IsString() + type!: string + + @IsString() + message!: string +} + +export class ImageDescription { + @IsInt() + imageId!: number + + @IsString() + description!: string +} + +export class Citation { + @IsString() + docName!: string + + @IsEnum(DocumentType) + type!: DocumentType + + @IsString() + @IsOptional() + sourceLink?: string + + @IsObject() + @IsOptional() + confidences?: Record + + @IsArray() + @Type(() => Number) + @IsOptional() + pageNumbers?: number[] + + @IsString() + documentId!: string + + @IsString() + questionId!: string + + @IsString() + @IsOptional() + aggregateId?: string +} + +export class ChatbotAskBody { + @IsString() + question!: string + + @IsArray() + @ValidateNested({ each: true }) + @Type(() => ChatMessage) + history!: ChatMessage[] +} + +export class ChatbotAskResponse { + @IsString() + question!: string + + @IsString() + answer!: string + + @IsInt() + courseId!: number + + @IsString() + questionId!: string + + @IsArray() + @ValidateNested({ each: true }) + @Type(() => Citation) + citations!: Citation[] + + @IsBoolean() + verified!: boolean + + @IsBoolean() + isPreviousQuestion!: boolean + + @IsOptional() + @IsArray() + @ValidateNested({ each: true }) + @Type(() => ImageDescription) + imageDescriptions?: ImageDescription[] +} + +export class ChatbotQueryBody { + @IsString() + query!: string + + @IsEnum(ChatbotQueryTypeEnum) + type!: ChatbotQueryTypeEnum + + @IsObject() + @IsOptional() + params?: Record + + @IsInt() + @IsOptional() + courseId?: number +} + +export class ChatbotProviderResponse { + @IsString() + type!: string + + @IsOptional() + @IsString() + baseUrl!: string + + @IsString() + defaultModelName!: string + + @IsString() + defaultVisionModelName!: string + + @IsOptional() + @IsObject() + headers?: Record +} + +export class ChatbotModelResponse { + @IsInstance(ChatbotProviderResponse) + @ValidateNested() + @Type(() => ChatbotProviderResponse) + provider!: ChatbotProviderResponse + + @IsString() + modelName!: string +} + +export class ChatbotOrganizationSettings { + @IsInstance(ChatbotProviderResponse) + @ValidateNested() + @Type(() => ChatbotProviderResponse) + defaultProvider!: ChatbotProviderResponse +} + +export class ChatbotCourseSettingsProperties { + @IsOptional() + @IsInstance(ChatbotOrganizationSettings) + @ValidateNested() + organizationSettings?: ChatbotOrganizationSettings + + @IsOptional() + @IsInstance(ChatbotModelResponse) + @ValidateNested() + model?: ChatbotModelResponse + + @IsOptional() + @IsString() + modelName?: string + + @IsString() + prompt!: string + + @IsNumber() + similarityThresholdDocuments!: number + + @IsNumber() + similarityThresholdQuestions!: number + + @IsNumber() + temperature!: number + + @IsInt() + topK!: number +} + +export class CreateChatbotCourseSettingsBody extends ChatbotCourseSettingsProperties {} + +export class UpdateChatbotCourseSettingsBody { + @IsOptional() + @IsInstance(ChatbotOrganizationSettings) + @ValidateNested() + organizationSettings?: ChatbotOrganizationSettings + + @IsOptional() + @IsInstance(ChatbotModelResponse) + @ValidateNested() + model?: ChatbotModelResponse + + @IsOptional() + @IsString() + modelName?: string + + @IsOptional() + @IsString() + prompt?: string + + @IsOptional() + @IsNumber() + similarityThresholdDocuments?: number + + @IsOptional() + @IsNumber() + similarityThresholdQuestions?: number + + @IsOptional() + @IsNumber() + temperature?: number + + @IsOptional() + @IsInt() + topK?: number +} + +export class CreateDocumentAggregateBody { + @IsString() + title!: string + + @IsString() + source!: string + + @IsString() + documentText!: string + + @IsString() + @IsOptional() + lmsDocumentId?: string + + @IsString() + @IsOptional() + prefix?: string +} + +export class UpdateDocumentAggregateBody { + @IsString() + @IsOptional() + title?: string + + @IsString() + @IsOptional() + source?: string + + @IsString() + @IsOptional() + documentText?: string + + @IsString() + @IsOptional() + lmsDocumentId?: string + + @IsString() + @IsOptional() + prefix?: string +} + +export class UploadDocumentAggregateBody { + @IsString() + source!: string + + @Transform((params) => + params.value === 'true' + ? true + : params.value === 'false' + ? false + : undefined, + ) + @IsBoolean() + @IsOptional() + parseAsPng?: boolean + + @IsString() + @IsOptional() + lmsDocumentId?: string + + @IsString() + @IsOptional() + prefix?: string +} + +export class UploadURLDocumentAggregateBody { + @IsString() + url!: string + + @IsString() + @IsOptional() + source?: string + + @Transform((params) => + params.value === 'true' + ? true + : params.value === 'false' + ? false + : undefined, + ) + @IsBoolean() + @IsOptional() + parseAsPng?: boolean + + @IsString() + @IsOptional() + lmsDocumentId?: string + + @IsString() + @IsOptional() + prefix?: string +} + +export class CloneCourseDocumentsBody { + @IsOptional() + @IsBoolean() + includeDocuments?: boolean + + @IsOptional() + @IsBoolean() + includeInsertedQuestions?: boolean + @IsOptional() + @IsBoolean() + includeInsertedDocuments?: boolean + + @IsObject() + docIdMap!: Record +} + +export class CreateDocumentChunkBody { + @IsString() + content!: string + + @IsEnum(DocumentType) + type!: DocumentType + + @IsBoolean() + @IsOptional() + disabled?: boolean + + @IsString() + @IsOptional() + title?: string + + @IsString() + @IsOptional() + source?: string + + @IsArray({ each: true }) + @Type(() => Number) + @IsOptional() + lines?: [number, number] + + @IsInt() + @IsOptional() + pageNumber?: number + + @IsInt() + @IsOptional() + asyncQuestionId?: number + + @IsString() + @IsOptional() + aggregateId?: string + + @IsString() + @IsOptional() + questionId?: string + + @IsString() + @IsOptional() + prefix?: string +} + +export class UpdateDocumentChunkBody { + @IsString() + @IsOptional() + content?: string + + @IsEnum(DocumentType) + @IsOptional() + type?: DocumentType + + @IsBoolean() + @IsOptional() + disabled?: boolean + + @IsString() + @IsOptional() + title?: string + + @IsString() + @IsOptional() + source?: string + + @IsArray({ each: true }) + @Type(() => Number) + @IsOptional() + lines?: [number, number] + + @IsInt() + @IsOptional() + pageNumber?: number + + @IsInt() + @IsOptional() + asyncQuestionId?: number + + @IsString() + @IsOptional() + aggregateId?: string + + @IsString() + @IsOptional() + questionId?: string + + @IsString() + @IsOptional() + prefix?: string +} + +export class CreateQuestionBody { + @IsString() + question!: string + + @IsString() + answer!: string + + @IsArray() + @Type(() => String) + @IsOptional() + @ValidateNested({ each: true }) + sourceDocumentIds?: string[] + + @IsBoolean() + @IsOptional() + verified?: boolean + + @IsBoolean() + @IsOptional() + suggested?: boolean +} + +export class UpdateQuestionBody extends CreateQuestionBody { + @IsString() + @IsOptional() + question!: string + + @IsString() + @IsOptional() + answer!: string +} + +export class SuggestedQuestionResponse { + @IsString() + id!: string + + @IsInt() + courseId!: number + + @IsString() + question!: string + + @IsString() + answer!: string + + @IsDate() + askedAt!: Date + + @IsBoolean() + verified!: boolean + + @IsArray() + @ValidateNested({ each: true }) + @Type(() => Citation) + citations!: Citation[] +} + +export class ChatbotCourseSettingsResponse extends ChatbotCourseSettingsProperties { + @IsInt() + courseId!: number +} + +export class ChatbotDocumentAggregateResponse { + @IsString() + id!: string + + @IsInt() + courseId!: number + + @IsString() + title!: string + + @IsEnum(DocumentType) + type!: DocumentType + + @IsString() + source!: string + + @IsString() + @IsOptional() + lmsDocumentId?: string + + @IsArray() + @Type(() => ChatbotDocumentResponse) + @ValidateNested({ each: true }) + subDocuments!: ChatbotDocumentResponse[] + + @IsString() + @IsOptional() + size?: string +} + +export class ChatbotQuestionResponse { + @IsString() + id!: string + + @IsInt() + courseId!: number + + @IsString() + question!: string + + @IsString() + answer!: string + + @IsDate() + @Transform((params) => new Date(params.value)) + askedAt!: Date + + @IsBoolean() + inserted!: boolean + + @IsBoolean() + suggested!: boolean + + @IsBoolean() + verified!: boolean + + @IsArray() + @Type(() => ChatbotDocumentResponse) + insertedDocuments!: ChatbotDocumentResponse[] + + @IsArray() + @Type(() => ChatbotCitationResponse) + @ValidateNested({ each: true }) + citations!: ChatbotCitationResponse[] +} + +export class ChatbotDocumentResponse { + @IsString() + id!: string + + @IsInt() + courseId!: number + + @IsString() + content!: string + + @IsEnum(DocumentType) + type!: DocumentType + + @IsBoolean() + disabled!: boolean + + @IsString() + @IsOptional() + title?: string + + @IsString() + @IsOptional() + source?: string + + @IsArray() + @Type(() => Number) + @ValidateNested({ each: true }) + lines?: [number, number] + + @IsInt() + @IsOptional() + pageNumber?: number + + @IsInt() + @IsOptional() + asyncQuestionId?: number + + @IsDate() + @Transform((params) => new Date(params.value)) + firstInsertedAt!: Date + + @IsArray() + @Type(() => ChatbotDocumentQueryResponse) + @ValidateNested({ each: true }) + queries!: ChatbotDocumentQueryResponse[] + + @IsArray() + @Type(() => ChatbotCitationResponse) + @ValidateNested({ each: true }) + citations!: ChatbotCitationResponse[] + + @IsString() + @IsOptional() + aggregateId?: string + + @IsString() + @IsOptional() + questionId?: string + + @IsInstance(ChatbotQuestionResponse) + @IsOptional() + parentQuestion?: ChatbotQuestionResponse + + @IsInstance(ChatbotDocumentAggregateResponse) + @IsOptional() + aggregate?: ChatbotDocumentAggregateResponse +} + +export class ChatbotDocumentQueryResponse { + @IsString() + id!: string + + @IsString() + documentId!: string + + @IsString() + query!: string + + @IsInstance(ChatbotDocumentResponse) + document!: ChatbotDocumentResponse +} + +export class ChatbotCitationResponse { + @IsString() + documentId!: string + + @IsString() + questionId!: string + + @IsDate() + citedAt!: Date + + @IsNumber() + @IsOptional() + confidence?: number + + @IsInstance(ChatbotQuestionResponse) + question!: ChatbotQuestionResponse + + @IsInstance(ChatbotDocumentResponse) + document!: ChatbotDocumentResponse +} + +export class ChatbotDocumentListResponse { + @IsString() + @IsOptional() + aggregateId?: string + + @IsString() + type!: 'aggregate' | 'chunk' + + @IsString() + title!: string + + @IsArray() + @Type(() => ChatbotDocumentResponse) + @ValidateNested({ each: true }) + documents!: ChatbotDocumentResponse[] +} + +export class GenerateDocumentQueryBody { + @IsBoolean() + @IsOptional() + deleteOld?: boolean +} + +export class UpsertDocumentQueryBody { + @IsString() + query!: string +} + +/* WEBSOCKET EVENTS */ + +export enum ChatbotResultEvents { + GET_RESULT = 'chatbot/get_result', + POST_RESULT = 'chatbot/post_result', + RESULT_RECEIVED = 'chatbot/received_result', +} + +export enum ChatbotResultEventName { + ADD_AGGREGATE = 'add_aggregate_complete', + UPDATE_AGGREGATE = 'update_aggregate_complete', + ADD_CHUNK = 'add_chunk_complete', + UPDATE_CHUNK = 'update_chunk_complete', + DOCUMENT_QUERIES = 'query_generation_complete', +} + +export type ChatbotResultEventArgs = { + returnMessage: ChatbotResultEvents + type: ChatbotResultEventName +} + +export type ChatbotEventParams = { + type: ChatbotResultEventName + resultId: string +} diff --git a/packages/common/index.ts b/packages/common/index.ts index adf3d3e22..a9f409d3d 100644 --- a/packages/common/index.ts +++ b/packages/common/index.ts @@ -1,4 +1,4 @@ -import { Exclude, Type } from 'class-transformer' +import { Exclude, Expose, Type } from 'class-transformer' import { IsArray, IsBoolean, @@ -19,6 +19,15 @@ import { import 'reflect-metadata' import { Cache } from 'cache-manager' import { Ajv } from 'ajv' +import { + ChatbotAskBody, + ChatbotAskResponse, + ChatbotQuestionResponse, + ChatMessage, + Citation, +} from './chatbot-api-types' + +export * from './chatbot-api-types' export const PROD_URL = 'https://coursehelp.ubc.ca' @@ -250,9 +259,8 @@ export type CourseCloneAttributes = { chatbot?: { settings?: boolean documents?: boolean - manuallyCreatedChunks?: boolean insertedQuestions?: boolean - insertedLMSData?: boolean + insertedDocuments?: boolean } } } @@ -273,9 +281,8 @@ export const defaultCourseCloneAttributes: CourseCloneAttributes = { chatbot: { settings: true, documents: true, - manuallyCreatedChunks: true, insertedQuestions: true, - insertedLMSData: false, + insertedDocuments: false, }, }, } @@ -369,175 +376,119 @@ export enum AccountType { // chatbot questions and interactions -export interface UpdateDocumentChunkParams { - documentText: string - metadata: { - name: string - source: string - } -} - -// comes from helpme db -export interface ChatbotQuestionResponseHelpMeDB { - id: number - vectorStoreId: string - interactionId: number - questionText: string - responseText: string - timestamp: Date - userScore: number - suggested: boolean - isPreviousQuestion: boolean - correspondingChatbotQuestion?: ChatbotQuestionResponseChatbotDB // used by chatbot_questions page on frontend - timesAsked?: number // same as above -} - -// comes from chatbot db -export interface ChatbotQuestionResponseChatbotDB { - id: string - pageContent: string // this is the question - metadata: { - answer: string - timestamp?: string // i found a chatbot question without a timestamp 😭 - courseId: string - verified: boolean - sourceDocuments: SourceDocument[] - suggested: boolean - inserted?: boolean - } - userScoreTotal?: number // NOT returned from db, it's calculated and used by chatbot_questions page on frontend - timesAsked?: number // same as above - interactionsWithThisQuestion?: InteractionResponse[] // same as above - mostRecentlyAskedHelpMeVersion?: ChatbotQuestionResponseHelpMeDB | null // same as above -} +export class PaginatedResponse { + @IsInt() + total!: number -interface Loc { - pageNumber: number + @IsArray() + items!: T[] } -// source document return type (from chatbot db) -export interface SourceDocument { - id?: string - metadata?: { - loc?: Loc - name: string - type?: string - source?: string - courseId?: string - fromLMS?: boolean - apiDocId?: number - } - type?: string - // TODO: is it content or pageContent? since this file uses both. EDIT: It seems to be both/either. Gross. - content?: string - pageContent: string - docName: string - docId?: string // no idea if this exists in the actual data EDIT: yes it does, sometimes - pageNumbers?: number[] // same with this, but this might only be for the edit question modal - pageNumbersString?: string // used only for the edit question modal - sourceLink?: string - pageNumber?: number - key?: string // used for front-end rendering -} +export class HelpMeChatMessage extends ChatMessage { + @IsBoolean() + @IsOptional() + verified?: boolean -export interface PreDeterminedQuestion { - id: string - pageContent: string - metadata: { - answer: string - courseId: string - inserted: boolean - sourceDocuments: SourceDocument[] - suggested: boolean - verified: boolean - } -} + @IsArray() + @IsOptional() + @Type(() => Citation) + @ValidateNested({ each: true }) + citations?: Citation[] -export interface Message { - type: 'apiMessage' | 'userMessage' - message: string | void - verified?: boolean - sourceDocuments?: SourceDocument[] + @IsString() + @IsOptional() questionId?: string - thinkText?: string | null // used on frontend only -} -export interface ChatbotQueryParams { - query: string - type: 'default' | 'abstract' + @IsString() + @IsOptional() + thinkText?: string // used on frontend only } -export interface ChatbotAskParams { - question: string - history: Message[] +export class HelpMeChatbotAskBody extends ChatbotAskBody { + @IsString() + question!: string + + @IsArray() + @Type(() => HelpMeChatMessage) + @ValidateNested({ each: true }) + history!: HelpMeChatMessage[] + + @IsInt() + @IsOptional() interactionId?: number - onlySaveInChatbotDB?: boolean + + @IsBoolean() + @IsOptional() + save?: boolean } -export interface ChatbotAskSuggestedParams { - question: string - responseText: string - vectorStoreId: string +export class ChatbotAskSuggestedBody { + @IsString() + vectorStoreId!: string } -export interface AddDocumentChunkParams { - documentText: string - metadata: { - name: string - type: string - source?: string - loc?: Loc - id?: string - courseId?: number - } - prefix?: string +export class HelpMeChatbotQuestionResponse { + @IsInt() + id!: number + + @IsString() + vectorStoreId!: string + + @IsInt() + interactionId!: number + + @IsInt() + userScore!: number + + @IsBoolean() + isPreviousQuestion!: boolean + + @IsDate() + @IsOptional() + timestamp?: Date + + @IsInstance(ChatbotQuestionResponse) + @IsOptional() + chatbotQuestion?: ChatbotQuestionResponse } -export interface AddDocumentAggregateParams { - name: string - source: string - documentText: string - metadata?: any - prefix?: string +export class HelpMeChatbotQuestionTableResponse extends HelpMeChatbotQuestionResponse { + @IsArray() + @ValidateNested({ each: true }) + @IsOptional() + children?: HelpMeChatbotQuestionTableResponse[] + + @IsBoolean() + @IsOptional() + isChild?: boolean + + @IsInt() + @IsOptional() + userScoreTotal?: number + + @IsInt() + @IsOptional() + timesAsked?: number } -export interface UpdateDocumentAggregateParams { - documentText: string - metadata?: any - prefix?: string +export class HelpMeChatbotAskResponse extends ChatbotAskResponse { + internal!: Omit } -export interface UpdateChatbotQuestionParams { - id: string - inserted?: boolean - sourceDocuments?: SourceDocument[] - question?: string - answer?: string - verified?: boolean - suggested?: boolean - selectedDocuments?: { - docId: string - pageNumbersString: string - }[] -} - -// this is the response from the backend when new questions are asked -// if question is I don't know, only answer and questionId are returned -export interface ChatbotAskResponse { - chatbotRepoVersion: ChatbotAskResponseChatbotDB - helpmeRepoVersion: ChatbotQuestionResponseHelpMeDB | null -} - -// comes from /ask from chatbot db -export interface ChatbotAskResponseChatbotDB { - question: string - answer: string - questionId: string - interactionId: number - sourceDocuments?: SourceDocument[] - verified: boolean - courseId: string - isPreviousQuestion: boolean +export class InteractionResponse { + @Expose() + @IsInt() + id!: number + + @Expose() + @IsDate() + timestamp!: Date + + @IsArray() + @Type(() => HelpMeChatbotQuestionResponse) + @ValidateNested({ each: true }) + @IsOptional() + questions?: HelpMeChatbotQuestionResponse[] } export enum ChatbotServiceType { @@ -545,20 +496,10 @@ export enum ChatbotServiceType { LATEST = 'latest', } -export interface AddChatbotQuestionParams { - question: string - answer: string - verified: boolean - suggested: boolean - sourceDocuments: SourceDocument[] -} - export interface OrganizationChatbotSettings { id: number - defaultProvider: ChatbotProvider providers: ChatbotProvider[] - default_prompt?: string default_temperature?: number default_topK?: number @@ -915,41 +856,6 @@ export class UpdateLLMTypeBody { additionalNotes?: string[] } -export interface ChatbotSettings { - id: string - pageContent: string - metadata: ChatbotSettingsMetadata -} - -export interface ProviderMetadata { - type: ChatbotServiceProvider - baseUrl: string - apiKey: string - defaultModelName: string - defaultVisionModelName: string - headers: ChatbotAllowedHeaders -} - -export interface ModelMetadata { - provider: ProviderMetadata - modelName: string -} - -export interface OrganizationChatbotSettingsMetadata { - defaultProvider: ProviderMetadata -} - -export interface ChatbotSettingsMetadata { - organizationSettings?: OrganizationChatbotSettingsMetadata - model?: ModelMetadata - modelName?: string - prompt: string - similarityThresholdDocuments: number - similarityThresholdQuestions: number - temperature: number - topK: number -} - export class UpsertCourseChatbotSettings { @IsInt() @IsOptional() @@ -972,30 +878,6 @@ export class UpsertCourseChatbotSettings { topK?: number } -export type ChatbotSettingsUpdateParams = Partial - -export interface InteractionResponse { - id: number - timestamp: Date - questions?: ChatbotQuestionResponseHelpMeDB[] -} - -export class ChatbotDocument { - id!: number - name!: number - type!: string - subDocumentIds!: string[] -} - -export type GetInteractionsAndQuestionsResponse = { - helpmeDB: InteractionResponse[] - chatbotDB: ChatbotQuestionResponseChatbotDB[] -} - -export type GetChatbotHistoryResponse = { - history: InteractionResponse[] -} - /** * A Queue that students can join with their tickets. * @param id - The unique id number for a Queue. @@ -2066,6 +1948,7 @@ export type LMSFileUploadResponse = { export type LMSSyncDocumentsResult = { itemsSynced: number itemsRemoved: number + resultIds: string[] errors: number } @@ -3907,7 +3790,7 @@ export function parseThinkBlock(answer: string) { if (!match) { // No block, return the text unchanged - return { thinkText: null, cleanAnswer: answer } + return { thinkText: undefined, cleanAnswer: answer } } const thinkText = match[1].trim() @@ -3973,6 +3856,8 @@ export const ERROR_MESSAGES = { 'Organization settings could not be created; organization not found.', }, chatbotController: { + textFileTooBig: + 'Text-only files (.txt, .csv, .md) must be less than 2 MB in size.', organizationSettingsAlreadyExists: 'Chatbot settings for this organization already exists.', organizationSettingsNotFound: @@ -3988,6 +3873,9 @@ export const ERROR_MESSAGES = { `Specified chatbot provider is not an ${provider} provider.`, }, chatbotService: { + missingVectorStoreId: + 'Cannot create question, corresponding vector store ID for question not specified', + interactionNotFound: 'Interaction with specified ID was not found.', defaultModelNotFound: 'Specified default model was not found in list of models', courseSettingsNotFound: @@ -4313,4 +4201,17 @@ export const ERROR_MESSAGES = { notAllowedToDeleteSemester: (role: OrganizationRole) => `Members with role ${role} are not allowed to delete semesters`, }, + webSocket: { + jwt: { + userNotFound: 'User not found', + disallowedPattern: 'Not authorized to use this route', + missingAuthHeader: 'Authorization header missing', + missingAuthToken: 'Authorization token missing', + malformedToken: 'Authorization token malformed', + invalidToken: 'Authorization token invalid', + }, + operations: { + timeout: 'Request timed out', + }, + }, } diff --git a/packages/common/tsconfig.json b/packages/common/tsconfig.json index 4fe490521..b45b4e4c8 100644 --- a/packages/common/tsconfig.json +++ b/packages/common/tsconfig.json @@ -1,4 +1,4 @@ { "extends": "../../tsconfig.json", - "include": ["index.ts"] + "include": ["index.ts","chatbot-api-types.ts"] } diff --git a/packages/frontend/.gitignore b/packages/frontend/.gitignore index 7c6929bcf..1cffbb2dd 100644 --- a/packages/frontend/.gitignore +++ b/packages/frontend/.gitignore @@ -15,6 +15,7 @@ # production /build +public/workers # misc .DS_Store diff --git a/packages/frontend/app/(dashboard)/components/CourseCloneForm.tsx b/packages/frontend/app/(dashboard)/components/CourseCloneForm.tsx index ae98bbe70..aa46b7ebb 100644 --- a/packages/frontend/app/(dashboard)/components/CourseCloneForm.tsx +++ b/packages/frontend/app/(dashboard)/components/CourseCloneForm.tsx @@ -275,9 +275,9 @@ const CourseCloneForm: React.FC = ({ = ({ > - - - ) diff --git a/packages/frontend/app/(dashboard)/components/CourseCloneFormModal.tsx b/packages/frontend/app/(dashboard)/components/CourseCloneFormModal.tsx index ce9246f22..a2c6e6613 100644 --- a/packages/frontend/app/(dashboard)/components/CourseCloneFormModal.tsx +++ b/packages/frontend/app/(dashboard)/components/CourseCloneFormModal.tsx @@ -100,6 +100,7 @@ const CourseCloneFormModal: React.FC = ({ = ({ form.getFieldValue([ 'toClone', 'chatbot', - 'manuallyCreatedChunks', + 'insertedDocuments', ]) || form.getFieldValue([ 'toClone', 'chatbot', 'insertedQuestions', - ]) || - form.getFieldValue([ - 'toClone', - 'chatbot', - 'insertedLMSData', ])) && (

Note that you may want to review and remove any out-of-date diff --git a/packages/frontend/app/(dashboard)/components/ImageCropperModal.tsx b/packages/frontend/app/(dashboard)/components/ImageCropperModal.tsx index 25f8e8918..73b170c4c 100644 --- a/packages/frontend/app/(dashboard)/components/ImageCropperModal.tsx +++ b/packages/frontend/app/(dashboard)/components/ImageCropperModal.tsx @@ -1,6 +1,6 @@ 'use client' -import { Button, Modal, Upload, message } from 'antd' +import { Button, message, Modal, Upload } from 'antd' import { UploadOutlined } from '@ant-design/icons' import React, { useCallback, useState } from 'react' import Cropper from 'react-easy-crop' @@ -128,6 +128,7 @@ const ImageCropperModal: React.FC = ({ return ( = ({ xl: '70%', xxl: '65%', }} - className="flex flex-col items-center justify-center" > - await handleUpload(file)} - beforeUpload={beforeUpload} - className="mb-2" - showUploadList={false} - maxCount={1} - accept=".jpg,.jpeg,.png,.webp,.avif,.gif,.svg,.tiff" - > - - -

- +
+ await handleUpload(file)} + beforeUpload={beforeUpload} + className="mb-2" + showUploadList={false} + maxCount={1} + accept=".jpg,.jpeg,.png,.webp,.avif,.gif,.svg,.tiff" + > + + +
+ +
) diff --git a/packages/frontend/app/(dashboard)/course/[cid]/(insights)/components/DashboardPresetComponent.tsx b/packages/frontend/app/(dashboard)/course/[cid]/(insights)/components/DashboardPresetComponent.tsx index b96187a9e..069cee2c5 100644 --- a/packages/frontend/app/(dashboard)/course/[cid]/(insights)/components/DashboardPresetComponent.tsx +++ b/packages/frontend/app/(dashboard)/course/[cid]/(insights)/components/DashboardPresetComponent.tsx @@ -139,6 +139,7 @@ const DashboardPresetComponent: React.FC = ({ <> {insightsList != undefined && ( = ({ queueDetails != undefined && ( <> = ({ staffDetails != undefined && ( <> = ({ studentDetails != undefined && ( <> void + onDeleteChunk: (documentId: string) => void + + generateQueries: (documentId: string, deleteOld: boolean) => Promise + onCreateQuery: (documentId: string, content: string) => Promise + onEditQuery: (queryId: string, content: string) => Promise + onDeleteQuery: (queryId: string) => Promise + onDeleteAllQueries: (documentId: string) => Promise + + isLoading: boolean +} + +const DocumentChunkRow: React.FC = ({ + documentChunk, + searchTerms, + + onEditChunk, + onDeleteChunk, + + generateQueries, + onCreateQuery, + onEditQuery, + onDeleteQuery, + onDeleteAllQueries, +}) => { + const [editingQueries, setEditingQueries] = useState< + { queryId: string; editingContent: string }[] + >([]) + const [creatingInput, setCreatingInput] = useState() + const [creatingQuery, setCreatingQuery] = useState(false) + const [isSaving, setIsSaving] = useState(false) + const [removeOld, setRemoveOld] = useState(false) + const [generateConfirmModalOpen, setGenerateConfirmModalOpen] = + useState(false) + + const [page, setPage] = useState(1) + + useEffect(() => { + const ids = documentChunk.queries.map((q) => q.id) + setEditingQueries((prev) => prev.filter((eq) => ids.includes(eq.queryId))) + }, [documentChunk.queries]) + + const toggleEditQuery = (record: ChatbotDocumentQueryResponse) => { + if (editingQueries.find((eq) => eq.queryId == record.id)) return + setEditingQueries((prev) => [ + ...prev, + { + queryId: record.id, + editingContent: record.query, + }, + ]) + } + + const onEditQueryInput = (queryId: string, text: string) => { + const index = editingQueries.findIndex((eq) => eq.queryId == queryId) + if (index < 0) { + message.warning('Edited query not found!') + return + } + + setEditingQueries((prev) => { + prev[index] = { + queryId, + editingContent: text, + } + return prev + }) + } + + const cancelEditQuery = (record: ChatbotDocumentQueryResponse) => { + setEditingQueries((prev) => prev.filter((p) => p.queryId != record.id)) + } + + const submitEditQuery = (queryId: string) => { + const entry = editingQueries.find((eq) => eq.queryId == queryId) + if (!entry) { + message.warning('Edited query not found!') + return + } + + setIsSaving(true) + onEditQuery(queryId, entry.editingContent) + .then((success) => { + if (success) { + setEditingQueries((prev) => prev.filter((p) => p.queryId != queryId)) + } + }) + .finally(() => setIsSaving(false)) + } + + const submitDeleteQuery = (queryId: string) => { + setIsSaving(true) + onDeleteQuery(queryId).finally(() => setIsSaving(false)) + } + + const gridCell = 'p-2 flex items-center justify-center' + + return ( +
+ + + Title + + + Content + + + {documentChunk.pageNumber ? 'Page Number' : ' '} + + + Type + + + Actions + + + + + + + {/* + In some environments, components which return Promises or arrays do not work. + This is due to some changes to react and @types/react, and the component + packages have not been updated to fix these issues. + */} + {/* @ts-expect-error Server Component */} + + + + + + {/* + In some environments, components which return Promises or arrays do not work. + This is due to some changes to react and @types/react, and the component + packages have not been updated to fix these issues. + */} + {/* @ts-expect-error Server Component */} + + + + {documentChunk.pageNumber ?? ''} + + +
+ {DocumentTypeDisplayMap[documentChunk.type] ?? + (DocumentType as any)[documentChunk.type]} +
+ + +
+ + +
+ + + setCreatingQuery(false)} + okButtonProps={{ + loading: isSaving, + }} + onOk={() => { + if (!documentChunk.id) { + setCreatingQuery(false) + return + } + if (!creatingInput) { + message.warning('Cannot create an empty query!') + return + } + setIsSaving(true) + onCreateQuery(documentChunk.id, creatingInput) + .then(() => { + setCreatingQuery(false) + setCreatingInput(undefined) + }) + .finally(() => setIsSaving(false)) + }} + okText={'Create'} + > +
+

+ Document queries are used to improve the retrieval stage of the RAG + algorithm, by closer matching to the potential questions asked by + users. +

+ setCreatingInput(evt.currentTarget?.value)} + /> +
+ + setGenerateConfirmModalOpen(false)} + footer={ +
+ + +
+ } + > +
+

+ This will generate a series of document queries that will match this + chunk, based on it and its surrounding chunks in the document. +

+

+ Document queries are used to improve the retrieval stage of the RAG + algorithm, by closer matching to the potential questions asked by + users. +

+

You have the option to remove previously generated queries:

+ setRemoveOld(!removeOld)} + checked={removeOld} + > + Remove Existing Queries + + {removeOld && ( + + + + {documentChunk.queries.length} queries will be removed + + + )} +
+ + {!documentChunk.questionId && !documentChunk.asyncQuestionId && ( + + +
+ + Document Queries{' '} + {(documentChunk.queries?.length ?? []) > 0 + ? `(${documentChunk.queries.length})` + : ''}{' '} + + +
+
+ + + +
+
+ ), + children: ( + <> + + { + setPage(page) + }} + /> + + + + Query Content + + + Actions + + + {documentChunk.queries + .slice((page - 1) * 10, page * 10) + .map((q, i) => ( + + ))} + + ), + }, + ]} + /> + + )} +
+ ) +} + +export type DocumentQueryRowProps = { + query: ChatbotDocumentQueryResponse + editingQueries: { queryId: string; editingContent: string }[] + onTextInput: (queryId: string, text: string) => void + + onDelete: (queryId: string) => void + submitEdit: (queryId: string) => void + + onEdit: (record: ChatbotDocumentQueryResponse) => void + cancelEdit: (record: ChatbotDocumentQueryResponse) => void + + isSaving: boolean +} +const DocumentQueryRow: React.FC = ({ + query, + editingQueries, + onTextInput, + onEdit, + onDelete, + submitEdit, + cancelEdit, + isSaving, +}) => { + const editingRow = useMemo( + () => editingQueries.find((eq) => eq.queryId == query.id), + [query, editingQueries], + ) + + return ( + + +
+ {editingRow ? ( + { + const val = event?.currentTarget?.value ?? '' + onTextInput(query.id, val) + }} + /> + ) : ( + + {query.query} + + )} +
+ + +
+ {editingRow ? ( + <> + + + + ) : ( + <> + + + + )} +
+ + + + ) +} + +export default DocumentChunkRow diff --git a/packages/frontend/app/(dashboard)/course/[cid]/(settings)/settings/chatbot_knowledge_base/components/EditChatbotDocumentChunkModal.tsx b/packages/frontend/app/(dashboard)/course/[cid]/(settings)/settings/chatbot_knowledge_base/components/EditChatbotDocumentChunkModal.tsx deleted file mode 100644 index 963aa2ba8..000000000 --- a/packages/frontend/app/(dashboard)/course/[cid]/(settings)/settings/chatbot_knowledge_base/components/EditChatbotDocumentChunkModal.tsx +++ /dev/null @@ -1,112 +0,0 @@ -'use client' - -import { ReactElement } from 'react' -import { Form, Input, message, Modal } from 'antd' -import TextArea from 'antd/es/input/TextArea' -import { getErrorMessage } from '@/app/utils/generalUtils' -import { SourceDocument } from '@koh/common' -import { API } from '@/app/api' - -interface FormValues { - documentName: string - content: string - source: string -} - -interface EditDocumentChunkModalProps { - editingRecord: SourceDocument - open: boolean - courseId: number - onSuccessfulUpdate: (value: SourceDocument[]) => void - onCancel: () => void -} - -const EditDocumentChunkModal: React.FC = ({ - editingRecord, - open, - courseId, - onSuccessfulUpdate, - onCancel, -}): ReactElement => { - const [form] = Form.useForm() - - const onFinish = async (values: FormValues) => { - await API.chatbot.staffOnly - .updateDocumentChunk(courseId, editingRecord.id || '', { - documentText: values.content, - metadata: { - name: values.documentName, - source: values.source, - }, - }) - .then((updatedDocs) => { - message.success('Document updated successfully.') - onSuccessfulUpdate(updatedDocs) - }) - .catch((e) => { - const errorMessage = getErrorMessage(e) - message.error('Failed to update document: ' + errorMessage) - }) - } - - return ( - -
Edit Document Chunk
- - } - okText="Save Changes" - cancelText="Cancel" - okButtonProps={{ - autoFocus: true, - htmlType: 'submit', - }} - onCancel={onCancel} - width={800} - destroyOnHidden - modalRender={(dom) => ( -
onFinish(values)} - > - {dom} -
- )} - > - - - - -