diff --git a/client/src/api.ts b/client/src/api.ts index 7b6bab15..c9a4cfbc 100644 --- a/client/src/api.ts +++ b/client/src/api.ts @@ -21,6 +21,7 @@ import { Homework, HomeworkSearchParams, PageDetailsResponse, + PageTag, PeerReview, Project, ProjectFile, @@ -33,6 +34,9 @@ import { Sender, ProjectSummary, InvitationsResponse, + PageSimpleWTags, + PageSimpleWOverview, + TableOfContentsDetailed, } from "./types"; import { AddableProjectTeamMember, @@ -343,6 +347,15 @@ class API { return res; } + async getBookPagesDetails(bookID: string) { + const res = await axios.get< + { + toc: TableOfContentsDetailed; + } & ConductorBaseResponse + >(`/commons/book/${bookID}/pages-details`); + return res; + } + async getPageDetails(pageID: string, coverPageID: string) { const res = await axios.get( `/commons/pages/${pageID}?coverPageID=${coverPageID}` @@ -369,12 +382,38 @@ class API { } /** - * Generates and applies an AI-generated summary to all pages in a book - * @param {string} pageID - the cover page of the book to apply the summaries to + * Generates and applies AI-generated summaries, tags, or both, to all pages in a book + * @param {string} bookID - the cover page of the book to apply the summaries to */ - async batchApplyPageAISummary(pageID: string) { - const res = await axios.patch( - `/commons/pages/${pageID}/ai-summary/batch` + async batchGenerateAIMetadata( + bookID: string, + summaries: boolean, + tags: boolean + ) { + const res = await axios.post( + `/commons/book/${bookID}/ai-metadata-batch`, + { + summaries, + tags, + } + ); + return res; + } + + /** + * Applies user-supplied summaries and tags to the respective pages in a book + * @param {string} bookID - the cover page of the book to apply the metadata to + * @param {Array<{ id: string; summary: string; tags: string[] }>} pages - the pages & data to update + */ + async batchUpdateBookMetadata( + bookID: string, + pages: { id: string; summary: string; tags: string[] }[] + ) { + const res = await axios.post( + `/commons/book/${bookID}/update-metadata-batch`, + { + pages, + } ); return res; } @@ -391,6 +430,21 @@ class API { return res; } + async bulkUpdatePageTags( + bookID: string, + pages: { id: string; tags: string[] }[] + ) { + const res = await axios.put< + { + failed: number; + processed: number; + } & ConductorBaseResponse + >(`/commons/book/${bookID}/page-tags`, { + pages, + }); + return res; + } + // Central Identity async getCentralIdentityOrgs({ activePage, @@ -1021,38 +1075,40 @@ class API { email: string, role: string ) { - const res = await axios.post<{ - responseInvitation: BaseInvitation - } & ConductorBaseResponse>( - `/project-invitations/${projectID}`, + const res = await axios.post< { - email, - role - } - ); + responseInvitation: BaseInvitation; + } & ConductorBaseResponse + >(`/project-invitations/${projectID}`, { + email, + role, + }); return res.data; } async getAllProjectInvitations( - projectID: string, - page: number = 1, + projectID: string, + page: number = 1, limit: number ) { - const res = await axios.get<{ - data: InvitationsResponse; - } & ConductorBaseResponse>(`/project-invitations/project/${projectID}`, { + const res = await axios.get< + { + data: InvitationsResponse; + } & ConductorBaseResponse + >(`/project-invitations/project/${projectID}`, { params: { page, limit }, }); return res.data; } - async getProjectInvitation( - inviteID: string, - token: string | null - ) { - const res = await axios.get<{ - invitation: BaseInvitation & {sender: Sender} & {project: ProjectSummary}; - } & ConductorBaseResponse>(`/project-invitations/${inviteID}`, { + async getProjectInvitation(inviteID: string, token: string | null) { + const res = await axios.get< + { + invitation: BaseInvitation & { sender: Sender } & { + project: ProjectSummary; + }; + } & ConductorBaseResponse + >(`/project-invitations/${inviteID}`, { params: { token }, }); return res.data; @@ -1064,34 +1120,35 @@ class API { deleted: boolean; } & ConductorBaseResponse >(`/project-invitations/${invitationId}`); - + return res.data; } - + async updateInvitationRole(inviteID: string, role: string) { const res = await axios.put< { updatedInvitation: BaseInvitation; } & ConductorBaseResponse >(`/project-invitations/${inviteID}/update`, { role }); - - return res.data; - } - - async acceptProjectInvitation(inviteID: string | null, token: string | null){ - const res = await axios.post<{ - data: string - } & ConductorBaseResponse>( + + return res.data; + } + + async acceptProjectInvitation(inviteID: string | null, token: string | null) { + const res = await axios.post< + { + data: string; + } & ConductorBaseResponse + >( `/project-invitation/${inviteID}/accept`, {}, { - params: {token}, + params: { token }, } ); return res.data; } - } export default new API(); diff --git a/client/src/components/ConfirmModal.tsx b/client/src/components/ConfirmModal.tsx new file mode 100644 index 00000000..fe9b919c --- /dev/null +++ b/client/src/components/ConfirmModal.tsx @@ -0,0 +1,40 @@ +import { Button, Modal, SemanticCOLORS } from "semantic-ui-react"; + +interface ConfirmModalProps { + text?: string; + onConfirm: () => void; + onCancel: () => void; + confirmText?: string; + cancelText?: string; + confirmColor?: SemanticCOLORS; + cancelColor?: SemanticCOLORS; +} + +const ConfirmModal: React.FC = ({ + text = "Are you sure?", + onConfirm, + onCancel, + confirmText = "Confirm", + cancelText = "Cancel", + confirmColor = "green", + cancelColor = undefined, +}) => { + return ( + + Confirm + +

{text}

+ + + + + + + ); +}; + +export default ConfirmModal; diff --git a/client/src/components/ControlledInputs/CtlTextArea.tsx b/client/src/components/ControlledInputs/CtlTextArea.tsx index 7ab84f7b..bf828d97 100644 --- a/client/src/components/ControlledInputs/CtlTextArea.tsx +++ b/client/src/components/ControlledInputs/CtlTextArea.tsx @@ -3,13 +3,14 @@ import { Form, FormTextAreaProps } from "semantic-ui-react"; import { ControlledInputProps } from "../../types"; import "../../styles/global.css"; -interface CtlTextAreaProps extends FormTextAreaProps { +interface CtlTextAreaProps extends React.HTMLProps { label?: string; required?: boolean; maxLength?: number; showRemaining?: boolean; fluid?: boolean; bordered?: boolean; + error?: string; } /** @@ -49,24 +50,48 @@ export default function CtlTextArea< field: { value, onChange, onBlur }, fieldState: { error }, }) => ( -
+
{label && ( )} - onChange(e.target.value)} onBlur={onBlur} error={error?.message} - className={`!m-0 ${fluid ? "fluid-textarea" : ""} ${bordered ? 'border border-slate-400 rounded-md padded-textarea': ''} ${bordered && showRemaining && maxLength && getRemainingChars(value) < 0 ? '!border-red-500' : ''}`} + className={`!m-0 ${fluid ? "!w-full" : ""} ${ + bordered + ? "border border-slate-400 rounded-md p-[0.5em]" + : "" + } ${ + bordered && + showRemaining && + maxLength && + getRemainingChars(value) < 0 + ? "!border-red-500" + : "" + }`} + rows={3} {...rest} /> + {/* */} {maxLength && showRemaining && typeof value === "string" && ( - + Characters remaining: {getRemainingChars(value)} )} diff --git a/client/src/components/LoadingSpinner/index.tsx b/client/src/components/LoadingSpinner/index.tsx index c0bfe931..908d3dba 100644 --- a/client/src/components/LoadingSpinner/index.tsx +++ b/client/src/components/LoadingSpinner/index.tsx @@ -1,9 +1,13 @@ -const LoadingSpinner: React.FC = () => { +interface LoadingSpinnerProps { + text?: string; +} + +const LoadingSpinner: React.FC = ({ text }) => { return (
- Loading + {text || "Loading"}
diff --git a/client/src/components/projects/TextbookCuration/BulkAIMetadataModal.tsx b/client/src/components/projects/TextbookCuration/BulkAIMetadataModal.tsx new file mode 100644 index 00000000..f0489b2d --- /dev/null +++ b/client/src/components/projects/TextbookCuration/BulkAIMetadataModal.tsx @@ -0,0 +1,167 @@ +import { Button, Checkbox, Icon, Modal } from "semantic-ui-react"; +import { useMemo, useState } from "react"; +import useGlobalError from "../../error/ErrorHooks"; +import "../Projects.css"; +import { useModals } from "../../../context/ModalContext"; +import { useNotifications } from "../../../context/NotificationContext"; +import api from "../../../api"; +import { Project } from "../../../types"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; + +interface BulkAIMetadataModalProps { + projectID: string; + library: string; + pageID: string; +} + +const BulkAIMetadataModal: React.FC = ({ + projectID, + library, + pageID, +}) => { + const { handleGlobalError } = useGlobalError(); + const { closeAllModals } = useModals(); + const { addNotification } = useNotifications(); + const queryClient = useQueryClient(); + const [loading, setLoading] = useState(false); + const [tags, setTags] = useState(false); + const [summaries, setSummaries] = useState(false); + const { data: projectData, isLoading: projectLoading } = useQuery< + Project | undefined + >({ + queryKey: ["project", projectID], + queryFn: async () => { + if (!projectID) return undefined; + const res = await api.getProject(projectID); + if (res.data.err) { + throw res.data.errMsg; + } + + return res.data.project; + }, + enabled: !!projectID, + }); + + const lastJob = useMemo(() => { + if (!projectData || !projectData.batchUpdateJobs) { + return null; + } + + return projectData.batchUpdateJobs.sort((a, b) => { + if (!a.startTimestamp || !b.startTimestamp) { + return 0; + } + return ( + new Date(b.startTimestamp).getTime() - + new Date(a.startTimestamp).getTime() + ); + })[0]; + }, [projectData]); + + async function handleSubmit() { + try { + if (!library || !pageID) { + throw new Error("Missing library or page ID"); + } + + if (!summaries && !tags) { + throw new Error("Please select summaries and/or tags to generate."); + } + + setLoading(true); + + const res = await api.batchGenerateAIMetadata( + `${library}-${pageID}`, + summaries, + tags + ); + if (res.data.err) { + throw new Error(res.data.errMsg); + } + + addNotification({ + type: "success", + message: + "We're generating AI metadata for this book. We'll send you an email when we've finished.", + }); + + queryClient.invalidateQueries(["project", projectID]); + + closeAllModals(); + } catch (err) { + handleGlobalError(err); + } finally { + setLoading(false); + } + } + + return ( + + Generate AI Metadata? + +

+ Are you sure you want to generate AI metadata for all pages in this + book? This will overwrite any existing summaries and/or tags. +

+ +
+ setSummaries(!summaries)} + toggle + /> + setTags(!tags)} + toggle + /> +
+

+ Note: Structural pages like the Table of Contents, + chapter cover pages, etc., will not be processed. This operation may + take some time, so we'll send you an email when it's complete. Only + one bulk operation can be run at a time per book. +

+

+ Caution: AI-generated output may not always be accurate. Please + thoroughly review content before publishing. LibreTexts is not + responsible for any inaccuracies in AI-generated content. +

+ + +
+

+ Last updated:{" "} + {lastJob?.endTimestamp + ? new Intl.DateTimeFormat("en-US", { + year: "numeric", + month: "long", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }).format(new Date(lastJob.endTimestamp)) + : "Never"} +

+
+ + +
+
+ + + ); +}; + +export default BulkAIMetadataModal; diff --git a/client/src/components/projects/TextbookCuration/BulkAddTagModal.tsx b/client/src/components/projects/TextbookCuration/BulkAddTagModal.tsx new file mode 100644 index 00000000..777829ba --- /dev/null +++ b/client/src/components/projects/TextbookCuration/BulkAddTagModal.tsx @@ -0,0 +1,183 @@ +import { useState } from "react"; +import { Button, Checkbox, Icon, Modal, Table } from "semantic-ui-react"; +import CtlTextInput from "../../ControlledInputs/CtlTextInput"; +import { useFieldArray, useForm } from "react-hook-form"; + +interface BulkAddTagModalProps { + pages: { id: string; title: string }[]; + onCancel: () => void; + onConfirm: (pages: string[], tags: string[]) => void; +} + +const BulkAddTagModal: React.FC = ({ + pages: availablePages, + onCancel, + onConfirm, +}) => { + const [pages, setPages] = useState([]); + + // We can't use a simple string array for useFieldArray, so we need to use an object with an id and value + const { control, watch, getValues, trigger } = useForm<{ + tags: { id: string; value: string }[]; + }>({ + defaultValues: { + tags: [ + { + id: crypto.randomUUID(), + value: "", + }, + ], + }, + }); + + const { fields, append, remove } = useFieldArray({ + control: control, + name: "tags", + }); + + const handleAddPage = (page: string) => { + if (!pages.includes(page)) { + setPages([...pages, page]); + } + }; + + const handleRemovePage = (page: string) => { + setPages(pages.filter((p) => p !== page)); + }; + + const handleToggleSelectAll = () => { + const anySelected = pages.length > 0; + if (anySelected) { + setPages([]); + } else { + setPages(availablePages.map((p) => p.id)); + } + }; + + const handleConfirm = async () => { + const isValid = await trigger(); + if (!isValid) return; + const tags = getValues().tags.map((t) => t.value.trim()); + onConfirm(pages, tags); + }; + + return ( + onCancel()} size="large"> + Bulk Add Tags + +
+

+ Add tags to multiple pages at once. Enter the tags you want to add, + then select the pages you want to add them to. This will not + immediately save the tags to the pages, but adds them to your + working changes so you can continue to edit before saving. +

+ + + + Tag + + + + + {fields.map((tag, index) => ( + + + + + +
+ { + // If it's the last tag show the + button instead of the trash can + index === fields.length - 1 ? ( + + ) : null + } + {index > 0 ? ( +
+ + + ))} + +
+
+
+ + + + + Selected + + + Page + + + + {availablePages.map((p) => ( + + + + pages.includes(p.id) + ? handleRemovePage(p.id) + : handleAddPage(p.id) + } + /> + + + {p.title} + + + ))} + +
+
+ + + + + + + ); +}; + +export default BulkAddTagModal; diff --git a/client/src/components/projects/TextbookCuration/ConfirmAISummariesModal.tsx b/client/src/components/projects/TextbookCuration/ConfirmAISummariesModal.tsx deleted file mode 100644 index b3cad3ab..00000000 --- a/client/src/components/projects/TextbookCuration/ConfirmAISummariesModal.tsx +++ /dev/null @@ -1,80 +0,0 @@ -import { Button, Icon, Modal } from "semantic-ui-react"; -import { useState } from "react"; -import useGlobalError from "../../error/ErrorHooks"; -import "../Projects.css"; -import { useModals } from "../../../context/ModalContext"; -import { useNotifications } from "../../../context/NotificationContext"; -import api from "../../../api"; - -interface ConfirmAISummariesModalProps { - library: string; - pageID: string; -} - -const ConfirmAISummariesModal: React.FC = ({ - library, - pageID, -}) => { - const { handleGlobalError } = useGlobalError(); - const { closeAllModals } = useModals(); - const { addNotification } = useNotifications(); - const [loading, setLoading] = useState(false); - - async function handleSubmit() { - try { - if (!library || !pageID) { - throw new Error("Missing library or page ID"); - } - - setLoading(true); - - const res = await api.batchApplyPageAISummary(`${library}-${pageID}`); - if (res.data.err) { - throw new Error(res.data.errMsg); - } - - addNotification({ - type: "success", - message: - "We're generating AI summaries for this book. We'll send you an email when they're ready.", - }); - closeAllModals(); - } catch (err) { - handleGlobalError(err); - } finally { - setLoading(false); - } - } - - return ( - - Generate AI Page Summaries? - -

- Are you sure you want to generate AI summaries for all pages in this - book? This will overwrite any existing summaries. -

-

- Note: Structural pages like the Table of Contents, - chapter cover pages, etc., will not be summarized. This operation may - take some time, so we'll send you an email when it's complete. -

-

- Caution: AI-generated output may not always be accurate. Please - thoroughly review content before publishing. LibreTexts is not - responsible for any inaccuracies in AI-generated content. -

- - - - - - - ); -}; - -export default ConfirmAISummariesModal; diff --git a/client/src/components/projects/TextbookCuration/EditMetadataModal.tsx b/client/src/components/projects/TextbookCuration/EditMetadataModal.tsx index dbe495dd..aeff0a9b 100644 --- a/client/src/components/projects/TextbookCuration/EditMetadataModal.tsx +++ b/client/src/components/projects/TextbookCuration/EditMetadataModal.tsx @@ -20,18 +20,8 @@ import useGlobalError from "../../error/ErrorHooks"; import LoadingSpinner from "../../LoadingSpinner"; import { useNotifications } from "../../../context/NotificationContext"; import "../Projects.css"; +import { DISABLED_PAGE_TAG_PREFIXES } from "../../../utils/misc"; const SUMMARY_MAX_LENGTH = 500; -const DISABLED_TAG_PREFIXES = [ - "article:", - "authorname:", - "license:", - "licenseversion:", - "source@", - "stage:", - "lulu@", - "author@", - "printoptions:" -]; type PageMetadata = { summary: string; @@ -45,6 +35,10 @@ interface EditMetadataModalProps { title: string; } +/** + * @deprecated + * This component is probably useless now, but leaving here for reference + */ const EditMetadataModal: React.FC = ({ library, pageID, @@ -192,7 +186,7 @@ const EditMetadataModal: React.FC = ({ }, [aiTags, watch("tags")]); const isDisabledTag = (value?: any): boolean => { - return DISABLED_TAG_PREFIXES.some((prefix) => + return DISABLED_PAGE_TAG_PREFIXES.some((prefix) => value?.toString().startsWith(prefix) ); }; diff --git a/client/src/components/projects/TextbookCuration/SingleAddTagModal.tsx b/client/src/components/projects/TextbookCuration/SingleAddTagModal.tsx new file mode 100644 index 00000000..709f78f0 --- /dev/null +++ b/client/src/components/projects/TextbookCuration/SingleAddTagModal.tsx @@ -0,0 +1,64 @@ +import { Button, Form, Icon, Modal } from "semantic-ui-react"; +import CtlTextInput from "../../ControlledInputs/CtlTextInput"; +import { useForm } from "react-hook-form"; + +interface SingleTagModalProps { + pageID: string; + onCancel: () => void; + onConfirm: (pageID: string, tag: string) => void; +} + +const SingleAddTagModal: React.FC = ({ + pageID, + onCancel, + onConfirm, +}) => { + const { control, watch, getValues, trigger } = useForm<{ + tag: string; + }>({ + defaultValues: { + tag: "", + }, + }); + + const handleConfirm = async () => { + const isValid = await trigger(); + if (!isValid) return; + const tag = getValues().tag.trim(); + onConfirm(pageID, tag); + }; + + return ( + onCancel()} size="large"> + Add Tag + +
{ + e.preventDefault(); + handleConfirm(); + }}> + + +
+ + + + + + ); +}; + +export default SingleAddTagModal; diff --git a/client/src/components/projects/TextbookCuration/ViewBulkUpdateHistoryModal.tsx b/client/src/components/projects/TextbookCuration/ViewBulkUpdateHistoryModal.tsx new file mode 100644 index 00000000..ad3b55c9 --- /dev/null +++ b/client/src/components/projects/TextbookCuration/ViewBulkUpdateHistoryModal.tsx @@ -0,0 +1,75 @@ +import { Button, Modal, Table } from "semantic-ui-react"; +import { Project } from "../../../types"; + +interface ViewBulkUpdateHistoryModalProps { + project: Project; + onClose: () => void; +} + +const ViewBulkUpdateHistoryModal: React.FC = ({ + project, + onClose, +}) => { + return ( + onClose()} size="large"> + View Bulk Update History + +
+ + + + Job ID + Timestamp + Type + Source + Processed Pages + Failed Pages + Status + + + + {project.batchUpdateJobs?.map((job, index) => ( + + {job.jobID.slice(0, 8)} + + {job.startTimestamp + ? new Intl.DateTimeFormat("en-US", { + year: "numeric", + month: "numeric", + day: "numeric", + hour: "numeric", + minute: "numeric", + second: "numeric", + }).format(new Date(job.startTimestamp?.toString())) + : "N/A"}{" "} + -{" "} + {job.endTimestamp + ? new Intl.DateTimeFormat("en-US", { + year: "numeric", + month: "numeric", + day: "numeric", + hour: "numeric", + minute: "numeric", + second: "numeric", + }).format(new Date(job.endTimestamp?.toString())) + : "N/A"} + + {job.type} + {job.dataSource} + {job.processedPages} + {job.failedPages} + {job.status} + + ))} + +
+
+ + + + + + ); +}; + +export default ViewBulkUpdateHistoryModal; diff --git a/client/src/screens/conductor/Projects/TextbookCuration.tsx b/client/src/screens/conductor/Projects/TextbookCuration.tsx deleted file mode 100644 index c59864b6..00000000 --- a/client/src/screens/conductor/Projects/TextbookCuration.tsx +++ /dev/null @@ -1,258 +0,0 @@ -import { - List, - Grid, - Header, - Segment, - Button, - Icon, - Breadcrumb, -} from "semantic-ui-react"; -import { Link } from "react-router-dom-v5-compat"; -import { useParams } from "react-router-dom"; -import { useQuery, useQueryClient } from "@tanstack/react-query"; -import api from "../../../api"; -import { Project, TableOfContents } from "../../../types"; -import { useModals } from "../../../context/ModalContext"; -import EditMetadataModal from "../../../components/projects/TextbookCuration/EditMetadataModal"; -import { useState } from "react"; -import ConfirmAISummariesModal from "../../../components/projects/TextbookCuration/ConfirmAISummariesModal"; - -type WithUIState = Omit & { - expanded: boolean; - isRoot: boolean; - children: WithUIState[]; -}; - -const TextbookCuration = () => { - const { id: projectID } = useParams<{ id: string }>(); - const { openModal } = useModals(); - const queryClient = useQueryClient(); - const [bookTitle, setBookTitle] = useState(""); - const { data: projectData, isLoading: projectLoading } = useQuery< - Project | undefined - >({ - queryKey: ["project", projectID], - queryFn: async () => { - if (!projectID) return undefined; - const res = await api.getProject(projectID); - if (res.data.err) { - throw res.data.errMsg; - } - - return res.data.project; - }, - enabled: !!projectID, - }); - - const { data, isLoading } = useQuery({ - queryKey: ["textbook-structure", projectID], - queryFn: async () => { - if (!projectData?.libreLibrary || !projectData.libreCoverID) { - return [] as WithUIState[]; - } - - const res = await api.getBookTOC( - `${projectData.libreLibrary}-${projectData.libreCoverID}` - ); - setBookTitle(res.data?.toc.title || "No Title"); - const content = res.data?.toc?.children; // Skip the first level of the TOC - - // Recursively add expanded state to each node - const addExpandedState = ( - nodes: TableOfContents[], - isRoot = false - ): WithUIState[] => { - return nodes.map((node) => { - return { - ...node, - isRoot, - expanded: false, - children: addExpandedState(node.children), - }; - }); - }; - - const withUIState = addExpandedState(content, true); - - return withUIState; - }, - refetchOnMount: false, - refetchOnWindowFocus: false, - retry: 1, - enabled: - !!projectData && !!projectData.libreLibrary && !!projectData.libreCoverID, - }); - - const handleToggle = (id: string) => { - const toggleNode = (nodes: WithUIState[]): WithUIState[] => { - return nodes.map((node) => { - if (node.id === id) { - return { ...node, expanded: !node.expanded }; - } - return { ...node, children: toggleNode(node.children) }; - }); - }; - - const updatedData = toggleNode(data!); - queryClient.setQueryData(["textbook-structure", projectID], updatedData); - }; - - const handleOpenEditModal = ( - library: string, - pageID: string, - title: string - ) => { - if (!projectData?.libreCoverID || !projectData?.libreLibrary) { - return; - } - - openModal( - - ); - }; - - const handleOpenBulkSummariesModal = () => { - if (!projectData?.libreLibrary || !projectData?.libreCoverID) { - return; - } - - openModal( - - ); - }; - - const renderNodes = (nodes: WithUIState[], indentLevel = 1) => { - if (!projectData?.libreLibrary || !projectData?.libreCoverID) { - return ( -
-

- This project does not have a textbook associated with it. Please - return to the main project page to create or connect one. -

-
- ); - } - return ( - - {nodes.map((node, idx) => { - return ( - - - -
- {node.children && node.children.length !== 0 && ( - { - e.preventDefault(); - handleToggle(node.id); - }} - /> - )} - - {node.title} - -
- {!node.isRoot && ( -
- -
- )} - - {node.children && - node.expanded && - renderNodes(node.children, indentLevel + 1)} - - - ); - })} - - ); - }; - - return ( - - -
- AI Co-Author: {projectData?.title} -
- - - - - Projects - - - - {projectData?.title || "Loading..."} - - - AI Co-Author - - - - {data ? ( - <> -
-

- Title: {bookTitle} -

-
-

Bulk Actions:

- - -
-
- {renderNodes(data)} - - ) : ( -
-

- No content available. -

-
- )} - - - - - ); -}; - -export default TextbookCuration; diff --git a/client/src/screens/conductor/Projects/TextbookCuration/index.tsx b/client/src/screens/conductor/Projects/TextbookCuration/index.tsx new file mode 100644 index 00000000..34ff08e3 --- /dev/null +++ b/client/src/screens/conductor/Projects/TextbookCuration/index.tsx @@ -0,0 +1,881 @@ +import { + Breadcrumb, + Button, + Grid, + Header, + Icon, + Label, + Segment, + Accordion, + Message, + Placeholder, + PlaceholderLine, + Dropdown, + DropdownMenu, + DropdownItem, + ButtonGroup, +} from "semantic-ui-react"; +import { useModals } from "../../../../context/ModalContext"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import api from "../../../../api"; +import { + Prettify, + Project, + ProjectBookBatchUpdateJob, + TableOfContentsDetailed, +} from "../../../../types"; +import useGlobalError from "../../../../components/error/ErrorHooks"; +import LoadingSpinner from "../../../../components/LoadingSpinner"; +import { useNotifications } from "../../../../context/NotificationContext"; +import "../../../../components/projects/Projects.css"; +import { DISABLED_PAGE_TAG_PREFIXES } from "../../../../utils/misc"; +import { useParams } from "react-router-dom"; +import { useEffect, useMemo, useState } from "react"; +import { Link } from "react-router-dom-v5-compat"; +import ConfirmModal from "../../../../components/ConfirmModal"; +import { useFieldArray, useForm } from "react-hook-form"; +import CtlTextArea from "../../../../components/ControlledInputs/CtlTextArea"; +import BulkAIMetadataModal from "../../../../components/projects/TextbookCuration/BulkAIMetadataModal"; +import BulkAddTagModal from "../../../../components/projects/TextbookCuration/BulkAddTagModal"; +import SingleAddTagModal from "../../../../components/projects/TextbookCuration/SingleAddTagModal"; +import ViewBulkUpdateHistoryModal from "../../../../components/projects/TextbookCuration/ViewBulkUpdateHistoryModal"; + +type WithUIState = Prettify< + Omit & { + expanded: boolean; + isRoot: boolean; + children: WithUIState[]; + } +>; + +type FormWorkingData = { + pages: { + pageID: string; + overview: string; + tags: string[]; + }[]; +}; + +const TextbookCuration = () => { + const { id: projectID } = useParams<{ id: string }>(); + const queryClient = useQueryClient(); + const { handleGlobalError } = useGlobalError(); + const { closeAllModals } = useModals(); + const { addNotification } = useNotifications(); + const { openModal } = useModals(); + const [hasMadeChanges, setHasMadeChanges] = useState(false); + const [loading, setLoading] = useState(false); + const [bulkActionsOpen, setBulkActionsOpen] = useState(false); + + const { control, setValue } = useForm(); + const { fields, append, update, replace } = useFieldArray({ + control, + name: "pages", // unique name for your Field Array + }); + + const { data: projectData } = useQuery({ + queryKey: ["project", projectID], + queryFn: async () => { + if (!projectID) return undefined; + const res = await api.getProject(projectID); + if (res.data.err) { + throw res.data.errMsg; + } + + return res.data.project; + }, + enabled: !!projectID, + refetchOnWindowFocus: false, + }); + + const activeBatchJob = useMemo(() => { + if (!projectData || !projectData.batchUpdateJobs) { + return null; + } + + const active = projectData.batchUpdateJobs.find((job) => + ["pending", "running"].includes(job.status) + ); + + return active || null; + }, [projectData]); + + const { data, isFetching } = useQuery({ + queryKey: ["textbook-structure-detailed", projectID], + queryFn: async () => { + const res = await api.getBookPagesDetails( + `${projectData?.libreLibrary}-${projectData?.libreCoverID}` + ); + + if (res.data.err) { + throw res.data.errMsg; + } + + const addExpandedState = ( + nodes: TableOfContentsDetailed[], + isRoot = false + ): WithUIState[] => { + return nodes.map((node) => { + return { + ...node, + expanded: node.children.length === 0 ? true : false, + isRoot, + children: addExpandedState(node.children), + }; + }); + }; + + const content = res.data.toc?.children || []; // Skip first level, it's the root + const withUIState = addExpandedState(content, true); + + return withUIState; + }, + retry: 1, + refetchOnMount: false, + refetchOnWindowFocus: false, + enabled: + !!projectData?.libreLibrary && + !!projectData?.libreCoverID && + !activeBatchJob, + }); + + useEffect(() => { + if (!data) return; + + // flatten the data recursively + const flatten = (nodes: WithUIState[]): void => { + nodes.forEach((node) => { + const field = fields.find((f) => f.pageID === node.id); + if (!field) { + append({ + pageID: node.id, + overview: node.overview, + tags: node.tags, + }); + } + flatten(node.children); + }); + }; + + flatten(data); + }, [data]); + + const updateBookPagesMutation = useMutation({ + mutationFn: async (data: FormWorkingData) => { + const simplified = data.pages.map((p) => ({ + id: p.pageID, + summary: p.overview, + tags: p.tags, + })); + + return api.batchUpdateBookMetadata( + `${projectData?.libreLibrary}-${projectData?.libreCoverID}`, + simplified + ); + }, + onSettled: () => { + queryClient.invalidateQueries(["project", projectID]); // Refresh the project data (with bulk update jobs) + queryClient.invalidateQueries(["textbook-structure-detailed", projectID]); + addNotification({ + type: "success", + message: "Bulk update job created successfully!", + }); + setHasMadeChanges(false); + }, + onError: (error) => { + handleGlobalError(error); + }, + }); + + async function fetchAISummary(pageID: string) { + try { + if (!projectData?.libreLibrary || !pageID) return null; + + setLoading(true); + const res = await api.getPageAISummary( + `${projectData?.libreLibrary}-${pageID}`, + `${projectData?.libreLibrary}-${projectData?.libreCoverID}` + ); + if (res.data.err) { + throw new Error( + res.data.errMsg || "Failed to generate AI summary for this page." + ); + } + if (!res.data.summary) { + throw new Error("Failed to generate AI summary for this page."); + } + + const fieldIdx = fields.findIndex((f) => f.pageID === pageID); + if (!fieldIdx) return; + + setValue(`pages.${fieldIdx}.overview`, res.data.summary); + setHasMadeChanges(true); + addNotification({ + type: "success", + message: "AI summary generated successfully!", + }); + } catch (error) { + console.error(error); + handleGlobalError(error); + } finally { + setLoading(false); + } + } + + async function fetchAITags(pageID: string) { + try { + if (!projectData?.libreLibrary || !pageID) return null; + + setLoading(true); + const res = await api.getPageAITags( + `${projectData?.libreLibrary}-${pageID}`, + `${projectData?.libreLibrary}-${projectData?.libreCoverID}` + ); + if (res.data.err) { + throw new Error( + res.data.errMsg || "Failed to generate AI tags for this page." + ); + } + if (!res.data.tags) { + throw new Error("Failed to generate AI tags for this page."); + } + + const field = fields.find((f) => f.pageID === pageID); + const fieldIdx = fields.findIndex((f) => f.pageID === pageID); + if (!field || !fieldIdx) return; + + update(fieldIdx, { + ...field, + tags: res.data.tags, + }); + + setHasMadeChanges(true); + addNotification({ + type: "success", + message: "AI tags generated successfully!", + }); + } catch (error) { + console.error(error); + handleGlobalError(error); + } finally { + setLoading(false); + } + } + + function handleRemoveSingleTag(pageID: string, tag: string) { + setLoading(true); + const field = fields.find((f) => f.pageID === pageID); + const fieldIdx = fields.findIndex((f) => f.pageID === pageID); + if (!field || !fieldIdx) { + setLoading(false); + return; + } + + const updatedTags = field.tags.filter((t) => t !== tag); + + update(fieldIdx, { + ...field, + tags: updatedTags, + }); + + setHasMadeChanges(true); + setLoading(false); + } + + function handleConfirmRemoveAllOccurrences(tag: string) { + openModal( + { + handleDoRemoveAllOccurrences(tag); + closeAllModals(); + }} + onCancel={closeAllModals} + confirmText="Remove" + confirmColor="red" + /> + ); + } + + function handleDoRemoveAllOccurrences(tag: string) { + setLoading(true); + const updated = fields.map((p) => { + return { + ...p, + tags: p.tags.filter((t) => t !== tag), + }; + }); + + replace(updated); + + setHasMadeChanges(true); + setLoading(false); + } + + const isDisabledTag = (value: string): boolean => { + return DISABLED_PAGE_TAG_PREFIXES.some((prefix) => + value?.toString().startsWith(prefix) + ); + }; + + const filterDisabledTags = (arr: string[]): string[] => { + return arr.filter((tag) => !isDisabledTag(tag)); + }; + + function handleConfirmReset() { + openModal( + { + closeAllModals(); + window.location.reload(); + }} + onCancel={closeAllModals} + confirmText="Reset" + confirmColor="red" + /> + ); + } + + function handleConfirmSave() { + openModal( + { + await updateBookPagesMutation.mutateAsync({ pages: fields }); + closeAllModals(); + }} + onCancel={closeAllModals} + confirmText="Save" + confirmColor="green" + /> + ); + } + + const handleOpenBulkAIMetadataModal = () => { + if (!projectData?.libreLibrary || !projectData?.libreCoverID) { + return; + } + + openModal( + + ); + }; + + const handleToggle = (id: string) => { + const toggleNode = (nodes: WithUIState[]): WithUIState[] => { + return nodes.map((node) => { + if (node.id === id) { + return { ...node, expanded: !node.expanded }; + } + return { + ...node, + children: toggleNode(node.children), + }; + }); + }; + + const updatedData = toggleNode(data!); + queryClient.setQueryData( + ["textbook-structure-detailed", projectID], + updatedData + ); + }; + + const handleJumpTo = (to: "top" | "bottom") => { + window.scrollTo(0, to === "top" ? 0 : document.body.scrollHeight); + }; + + const handleExpandCollapseAll = () => { + const anyExpanded = data?.some((node) => node.expanded); + + const expandAll = (nodes: WithUIState[]): WithUIState[] => { + return nodes.map((node) => { + return { + ...node, + expanded: !anyExpanded, + children: expandAll(node.children), + }; + }); + }; + + const updatedData = expandAll(data!); + queryClient.setQueryData( + ["textbook-structure-detailed", projectID], + updatedData + ); + }; + + const ActiveJobAlert = (job: ProjectBookBatchUpdateJob) => { + return ( + + + +
+
+ Bulk Update Job In Progress + +

+ AI-generated metadata is currently being applied. This may take + some time to complete. Last update: {job.processedPages || 0}{" "} + successful pages. +

+
+ +
+ + + ); + }; + + const handleOpenBulkUpdateHistoryModal = () => { + if (!projectData) return; + openModal( + + ); + }; + + const handleOpenBulkAddTagsModal = () => { + if (!data) return; + + // flatten all nodes recursively + const flatten = (nodes: WithUIState[]): WithUIState[] => { + return nodes.reduce((acc, node) => { + return [...acc, node, ...flatten(node.children)]; + }, [] as WithUIState[]); + }; + + const availablePages = flatten(data); + + openModal( + ({ id: p.id, title: p.title })) || []} + onCancel={closeAllModals} + onConfirm={(pages, tags) => { + handleBulkAddTags(pages, tags); + closeAllModals(); + }} + /> + ); + }; + + const handleBulkAddTags = (pages: string[], tags: string[]) => { + setLoading(true); + const updated = fields.map((p) => { + if (pages.includes(p.pageID)) { + return { + ...p, + tags: [...new Set([...p.tags, ...tags])], + }; + } + return p; + }); + + replace(updated); + setHasMadeChanges(true); + setLoading(false); + }; + + const handleOpenSingleAddTagModal = (pageID: string) => { + openModal( + { + handleSingleAddTag(pageID, tag); + closeAllModals(); + }} + /> + ); + }; + + const handleSingleAddTag = (pageID: string, tag: string) => { + setLoading(true); + const field = fields.find((f) => f.pageID === pageID); + const fieldIdx = fields.findIndex((f) => f.pageID === pageID); + if (!field || !fieldIdx) return; + + const updatedTags = [...field.tags, tag]; + update(fieldIdx, { + ...field, + tags: updatedTags, + }); + setHasMadeChanges(true); + setLoading(false); + }; + + const LoadingSkeleton = () => { + return ( + + + + + + + + + + + + + ); + }; + + const TagLabel = ({ pageID, tag }: { pageID: string; tag: string }) => { + return ( + + ); + }; + + const renderEditor = (node: WithUIState, indentLevel = 1) => { + const field = fields.find((f) => f.pageID === node.id); + const fieldIdx = fields.findIndex((f) => f.pageID === node.id); + if (!field) return null; + + const hasChildren = node.children && node.children.length !== 0; + const tags = filterDisabledTags(field.tags); + + const getIndent = () => { + if (indentLevel <= 2) return "!ml-4"; + if (indentLevel === 3 && !hasChildren) return "!ml-4"; + if (indentLevel === 3 && hasChildren) return "!ml-12"; + return `!ml-${indentLevel * 3}`; + }; + + const indent = getIndent(); + + return ( +
+
+

{node.title}

+
+
+ +
+ + +
+
+
+
+
+ {tags.map((t) => ( + + ))} +
+
+ + +
+
+
+
+

ID: {node.id}

+
+
+ ); + }; + + const renderNodes = (nodes: WithUIState[], indentLevel = 1) => { + if (!projectData?.libreLibrary || !projectData?.libreCoverID) { + return ( +
+

+ This project does not have a textbook associated with it. Please + return to the main project page to create or connect one. +

+
+ ); + } + return ( + + {nodes.map((node, idx) => { + const hasChildren = node.children && node.children.length !== 0; + return ( + <> + + {hasChildren && ( +
+ {node.children && node.children.length !== 0 && ( + { + e.preventDefault(); + handleToggle(node.id); + }} + /> + )} + + {node.title} + +
+ )} + + + {node.expanded && renderEditor(node, indentLevel + 1)} + {node.children && + node.expanded && + renderNodes(node.children, indentLevel + 1)} + + + ); + })} + + ); + }; + + return ( + + +
+ AI Co-Author: {projectData?.title} +
+ + + + + Projects + + + + {projectData?.title || "Loading..."} + + + AI Co-Author + + + +

+ Welcome to LibreTexts' AI Co-Author tool. Here, you can curate + AI-generated metadata for your textbook. You can generate and edit + metadata for individual pages below, or use the bulk actions to + generate metadata for all pages at once. +

+ Benny the LibreTexts Mascot + + {activeBatchJob && ( + + + + )} + + {(isFetching || updateBookPagesMutation.isLoading) && ( +
+ +
+ )} +
+
+

+ Editing Summaries: Use + the magic wand icon to generate an AI summary for the page. + This will replace the current summary with the AI-generated + one, which you can then edit further. +

+ +

+ Editing Tags: Use the + magic wand icon to generate AI tags for the page. You can then + left-click on a tag to remove it from that page, or + right-click on a tag to remove it from all pages. You + can also add individual tags to a page with the plus icon, or + use the bulk actions to add tags to multiple pages at once. +

+
+
+ + + setBulkActionsOpen(false)} + > + + + + + + + +
+
+ {!activeBatchJob && ( + <> +
+ + + + +
+ {isFetching && } + {data && renderNodes(data)} +
+ + +
+ + )} + + + + + ); +}; + +export default TextbookCuration; diff --git a/client/src/types/Book.ts b/client/src/types/Book.ts index 49ed93c8..eff7ad7b 100644 --- a/client/src/types/Book.ts +++ b/client/src/types/Book.ts @@ -1,3 +1,5 @@ +import { Prettify } from "./Misc"; + export type Book = { coverID: string; bookID: string; @@ -32,7 +34,7 @@ export type BookWithSourceData = Book & { sourceHarvestDate?: Date; sourceLastModifiedDate?: Date; sourceLanguage?: string; -} +}; export type BookLinks = { online: string; @@ -67,4 +69,30 @@ export type PageTag = { export type PageDetailsResponse = { overview: string; tags: PageTag[]; -} \ No newline at end of file +}; + +type _PageSimple = { + id: string; + title: string; + url: string; +}; + +export type PageSimpleWTags = Prettify< + _PageSimple & { + tags: PageTag[]; + } +>; + +export type PageSimpleWOverview = Prettify< + _PageSimple & { + overview: string; + } +>; + +export type TableOfContentsDetailed = Prettify< + Omit & { + overview: string; + tags: string[]; + children: TableOfContentsDetailed[]; + } +>; diff --git a/client/src/types/Misc.ts b/client/src/types/Misc.ts index dfcdf36b..755b9212 100644 --- a/client/src/types/Misc.ts +++ b/client/src/types/Misc.ts @@ -24,7 +24,10 @@ export type ConductorBaseResponse = | { err: false } | { err: true; errMsg: string }; -export type _MoveFile = Pick; +export type _MoveFile = Pick< + ProjectFile, + "fileID" | "name" | "storageType" | "description" +>; export type _MoveFileWithChildren = _MoveFile & { children: _MoveFileWithChildren[]; disabled: boolean; @@ -35,7 +38,7 @@ export type CloudflareCaptionData = { label: string; }; -export type SortDirection = 'ascending' | 'descending'; +export type SortDirection = "ascending" | "descending"; export type License = { name?: string; @@ -44,4 +47,13 @@ export type License = { sourceURL?: string; modifiedFromSource?: boolean; additionalTerms?: string; -} \ No newline at end of file +}; + +/** + * A TypeScript type alias called `Prettify`. + * It takes a type as its argument and returns a new type that has the same properties as the original type, + * but the properties are not intersected. This means that the new type is easier to read and understand. + */ +export type Prettify = { + [K in keyof T]: T[K]; +} & {}; diff --git a/client/src/types/Project.ts b/client/src/types/Project.ts index 7962ec0a..49bc83e2 100644 --- a/client/src/types/Project.ts +++ b/client/src/types/Project.ts @@ -103,6 +103,23 @@ export type ProjectModuleSettings = { tasks: ProjectModuleConfig; }; +export type ProjectBookBatchUpdateJob = { + jobID: string; + type: "summaries" | "tags" | "summaries+tags"; + status: "pending" | "running" | "completed" | "failed"; + processedPages: number; + failedPages: number; + totalPages: number; + dataSource: "user" | "generated"; + ranBy: string; // User UUID + startTimestamp?: Date; + endTimestamp?: Date; + error?: string; + results?: { + [key: string]: any; + }; +} + export type Project = { orgID: string; projectID: string; @@ -166,6 +183,7 @@ export type Project = { sourceHarvestDate?: Date; sourceLastModifiedDate?: Date; sourceLanguage?: string; + batchUpdateJobs?: ProjectBookBatchUpdateJob[]; }; export type AddableProjectTeamMember = Pick & { diff --git a/client/src/utils/misc.ts b/client/src/utils/misc.ts index 37ae6863..aee7769f 100644 --- a/client/src/utils/misc.ts +++ b/client/src/utils/misc.ts @@ -232,4 +232,22 @@ export function upperFirst(str: string): string { */ export async function unwrapAPIResponse(response: Promise>): Promise { return (await response).data; -} \ No newline at end of file +} + +/** + * CXOne page tags that should not be displayed/edited by users + */ +export const DISABLED_PAGE_TAG_PREFIXES = [ + "article:", + "authorname:", + "license:", + "licenseversion:", + "source@", + "stage:", + "lulu@", + "author@", + "printoptions:", + "showtoc:", + "coverpage:", + "columns:" +]; \ No newline at end of file diff --git a/server/api.js b/server/api.js index 351042e5..9a45b2f6 100644 --- a/server/api.js +++ b/server/api.js @@ -752,13 +752,6 @@ router booksAPI.deleteBook ); -router - .route("/commons/book/:bookID/summary") - .get( - middleware.validateZod(BookValidators.getWithBookIDParamSchema), - booksAPI.getBookSummary - ); - router .route("/commons/book/:bookID/files/:fileID/download") .get( @@ -780,6 +773,48 @@ router booksAPI.getLicenseReport ); +router + .route("/commons/book/:bookID/pages-details") + .get( + authAPI.verifyRequest, + authAPI.getUserAttributes, + middleware.validateZod(BookValidators.getWithBookIDParamSchema), + booksAPI.getBookPagesDetails + ) + .post( + authAPI.verifyRequest, + authAPI.getUserAttributes, + middleware.validateZod(BookValidators.bulkUpdatePageTagsSchema), + booksAPI.bulkUpdatePageTags + ); + +router + .route("/commons/book/:bookID/page-tags") + .put( + authAPI.verifyRequest, + authAPI.getUserAttributes, + middleware.validateZod(BookValidators.bulkUpdatePageTagsSchema), + booksAPI.bulkUpdatePageTags + ); + +router + .route("/commons/book/:bookID/ai-metadata-batch") + .post( + authAPI.verifyRequest, + authAPI.getUserAttributes, + middleware.validateZod(BookValidators.batchGenerateAIMetadataSchema), + booksAPI.batchGenerateAIMetadata + ); + +router + .route("/commons/book/:bookID/update-metadata-batch") + .post( + authAPI.verifyRequest, + authAPI.getUserAttributes, + middleware.validateZod(BookValidators.batchUpdateBookMetadataSchema), + booksAPI.batchUpdateBookMetadata + ); + router .route("/commons/book/:bookID/peerreviews") .get( @@ -815,15 +850,6 @@ router booksAPI.getPageAISummary ); -router - .route("/commons/pages/:pageID/ai-summary/batch") - .patch( - middleware.validateZod(BookValidators.getWithPageIDParamSchema), - authAPI.verifyRequest, - authAPI.getUserAttributes, - booksAPI.batchApplyAISummary - ); - router .route("/commons/pages/:pageID/ai-tags") .get( @@ -2280,48 +2306,60 @@ router .post( authAPI.verifyRequest, authAPI.getUserAttributes, - middleware.validateZod(ProjectInvitationValidators.createProjectInvitationSchema), + middleware.validateZod( + ProjectInvitationValidators.createProjectInvitationSchema + ), projectInvitationsAPI.createProjectInvitation - ) + ); router .route("/project-invitations/:inviteID") .get( - middleware.validateZod(ProjectInvitationValidators.getProjectInvitationSchema), + middleware.validateZod( + ProjectInvitationValidators.getProjectInvitationSchema + ), projectInvitationsAPI.getProjectInvitation ) .delete( authAPI.verifyRequest, authAPI.getUserAttributes, - middleware.validateZod(ProjectInvitationValidators.deleteProjectInvitationSchema), + middleware.validateZod( + ProjectInvitationValidators.deleteProjectInvitationSchema + ), projectInvitationsAPI.deleteProjectInvitation - ) + ); router .route("/project-invitations/project/:projectID") .get( authAPI.verifyRequest, authAPI.getUserAttributes, - middleware.validateZod(ProjectInvitationValidators.getAllProjectInvitationsSchema), + middleware.validateZod( + ProjectInvitationValidators.getAllProjectInvitationsSchema + ), projectInvitationsAPI.getAllInvitationsForProject - ) + ); router .route("/project-invitation/:inviteID/accept") .post( authAPI.verifyRequest, authAPI.getUserAttributes, - middleware.validateZod(ProjectInvitationValidators.acceptProjectInvitationSchema), + middleware.validateZod( + ProjectInvitationValidators.acceptProjectInvitationSchema + ), projectInvitationsAPI.acceptProjectInvitation - ) + ); router .route("/project-invitations/:inviteID/update") .put( authAPI.verifyRequest, authAPI.getUserAttributes, - middleware.validateZod(ProjectInvitationValidators.updateProjectInvitationSchema), + middleware.validateZod( + ProjectInvitationValidators.updateProjectInvitationSchema + ), projectInvitationsAPI.updateProjectInvitation - ) + ); export default router; diff --git a/server/api/books.ts b/server/api/books.ts index 9341c28f..a5ba6e1e 100644 --- a/server/api/books.ts +++ b/server/api/books.ts @@ -1,4 +1,3 @@ -// @ts-nocheck import { Request, Response } from "express"; import fs from "fs-extra"; import { debug, debugError, debugCommonsSync, debugServer } from "../debug.js"; @@ -7,25 +6,21 @@ import Book, { BookInterface } from "../models/book.js"; import Collection from "../models/collection.js"; import Organization, { OrganizationInterface } from "../models/organization.js"; import CustomCatalog from "../models/customcatalog.js"; -import Project from "../models/project.js"; +import Project, { ProjectBookBatchUpdateJob } from "../models/project.js"; import PeerReview from "../models/peerreview.js"; -import Tag from "../models/tag.js"; import CIDDescriptor from "../models/ciddescriptor.js"; import conductorErrors from "../conductor-errors.js"; import { getSubdomainFromUrl, - getPaginationOffset, isEmptyString, isValidDateObject, sleep, getRandomOffset, } from "../util/helpers.js"; import { - checkBookIDFormat, deleteBookFromAPI, extractLibFromID, getLibraryAndPageFromBookID, - isValidLibrary, genThumbnailLink, genPDFLink, genBookstoreLink, @@ -33,11 +28,8 @@ import { genPubFilesLink, genLMSFileLink, genPermalink, - getBookTOCFromAPI, - getBookTOCNew, } from "../util/bookutils.js"; import { - retrieveProjectFiles, downloadProjectFiles, updateTeamWorkbenchPermissions, } from "../util/projectutils.js"; @@ -51,7 +43,11 @@ import alertsAPI from "./alerts.js"; import mailAPI from "./mail.js"; import collectionsAPI from "./collections.js"; import axios from "axios"; -import { BookSortOption, TableOfContents } from "../types/Book.js"; +import { + BookSortOption, + TableOfContents, + TableOfContentsDetailed, +} from "../types/Book.js"; import { isBookSortOption } from "../util/typeHelpers.js"; import { z } from "zod"; import { @@ -75,13 +71,14 @@ import { getMasterCatalogSchema, getWithBookIDParamSchema, getWithBookIDBodySchema, - getBookFilesSchema, downloadBookFileSchema, - getWithPageIDParamSchema, updatePageDetailsSchema, -} from "../validators/book.js"; -import * as cheerio from "cheerio"; -import { getWithPageIDParamAndCoverPageIDSchema } from "./validators/book.js"; + batchGenerateAIMetadataSchema, + batchUpdateBookMetadataSchema, + bulkUpdatePageTagsSchema, + getWithPageIDParamAndCoverPageIDSchema, +} from "./validators/book.js"; +import BookService from "./services/book-service.js"; const BOOK_PROJECTION: Partial> = { _id: 0, @@ -1521,7 +1518,7 @@ async function deleteBook( } else { debug("Simulating book deletion from API."); } - } catch (err) { + } catch (err: any) { debugError(`[Delete Book] ${err.toString()}`); return conductor500Err(res); } @@ -1941,42 +1938,6 @@ const removeBookFromCustomCatalog = ( }); }; -/** - * Makes a request to a Book's respective library to retrieve the Book summary. If no summary has - * been set, an empty string is returned. - * NOTE: This function should only be called AFTER the validation chain. - * VALIDATION: 'getBookSummary' - * - * @param {z.infer} req - Incoming request object. - * @param {express.Response} res - Outgoing response object. - */ -async function getBookSummary( - req: z.infer, - res: Response -) { - try { - const { bookID } = req.params; - const book = await Book.findOne({ bookID }).lean(); - if (!book) { - return res.status(404).send({ - err: true, - errMsg: conductorErrors.err11, - }); - } - - return res.send({ - err: false, - summary: book.summary || "", - bookID, - }); - } catch (e) { - return res.status(500).send({ - err: true, - errMsg: conductorErrors.err6, - }); - } -} - /** * Makes a request to a Book's respective Project to retrieve a signed download URL for a given file * NOTE: This function should only be called AFTER the validation chain. @@ -2059,7 +2020,8 @@ async function getBookTOC( res: Response ) { try { - const toc = await getBookTOCNew(req.params.bookID); + const bookService = new BookService({ bookID: req.params.bookID }); + const toc = await bookService.getBookTOCNew(); return res.send({ err: false, toc, @@ -2120,13 +2082,62 @@ async function getLicenseReport( } } +async function getBookPagesDetails( + req: ZodReqWithUser>, + res: Response +) { + try { + const { bookID } = req.params; + + const bookService = new BookService({ bookID }); + const toc = await bookService.getBookTOCNew(); + + const [overviews, tags] = await Promise.all([ + bookService.getAllPageOverviews(toc), + bookService.getAllPageTags(toc), + ]); + + // Loop through table of contents and add overviews and tags to each page (based on ID) + // Table of contents is a nested array, so we need to loop through each level + const addOverviewsAndTags = ( + toc: TableOfContents + ): TableOfContentsDetailed => { + const pageOverview = overviews.find((o) => o.id === toc.id); + const pageTags = tags.find((t) => t.id === toc.id)?.tags || []; + + const page: TableOfContentsDetailed = { + ...toc, + overview: pageOverview?.overview || "", + tags: pageTags, + children: toc.children.map(addOverviewsAndTags), + }; + + return page; + }; + + const detailedToc = addOverviewsAndTags(toc); + + return res.send({ + err: false, + toc: detailedToc, + }); + } catch (err) { + debugError(err); + return res.status(500).send({ + err: true, + errMsg: conductorErrors.err6, + }); + } +} + async function getPageDetail( - req: ZodReqWithUser, + req: ZodReqWithUser>, res: Response ) { try { - const { pageID } = req.params; + const { pageID: fullPageID } = req.params; const { coverPageID } = req.query; + const [_, pageID] = getLibraryAndPageFromBookID(fullPageID); const canAccess = await _canAccessPage(coverPageID, req.user.decoded.uuid); if (!canAccess) { @@ -2136,66 +2147,56 @@ async function getPageDetail( }); } - const [subdomain, coverID] = getLibraryAndPageFromBookID(pageID); - if (!subdomain || !coverID) { - return res.status(400).send({ + const bookService = new BookService({ bookID: coverPageID }); + const details = await bookService.getPageDetails(pageID); + if (!details) { + return res.status(404).send({ err: true, - errMsg: conductorErrors.err2, + errMsg: conductorErrors.err11, }); } - const pagePropertiesRes = await CXOneFetch({ - scope: "page", - path: parseInt(coverID), - api: MindTouch.API.Page.GET_Page_Properties, - subdomain, - }).catch((err) => { - console.error(err); - throw new Error(`Error fetching page details: ${err}`); + return res.send({ + err: false, + overview: details.overview, + tags: details.tags, }); - - if (!pagePropertiesRes.ok) { - throw new Error( - `Error fetching page details: ${pagePropertiesRes.statusText}` - ); - } - - const pagePropertiesRaw = await pagePropertiesRes.json(); - const pageProperties = Array.isArray(pagePropertiesRaw?.property) - ? pagePropertiesRaw.property - : [pagePropertiesRaw?.property]; - console.log(pageProperties); - const overviewProperty = pageProperties - .filter((p) => !!p) - .find((prop: any) => prop["@name"] === MindTouch.PageProps.PageOverview); - const overviewText = overviewProperty?.contents?.["#text"] || ""; - - const pageTagsRes = await CXOneFetch({ - scope: "page", - path: parseInt(coverID), - api: MindTouch.API.Page.GET_Page_Tags, - subdomain, - }).catch((err) => { - console.error(err); - throw new Error(`Error fetching page tags: ${err}`); + } catch (e) { + debugError(e); + return res.status(500).send({ + err: true, + errMsg: conductorErrors.err6, }); + } +} - if (!pageTagsRes.ok) { - throw new Error(`Error fetching page tags: ${pageTagsRes.statusText}`); +async function getPageAISummary( + req: ZodReqWithUser>, + res: Response +) { + try { + const { pageID: fullPageID } = req.params; + const { coverPageID } = req.query; + const [_, pageID] = getLibraryAndPageFromBookID(fullPageID); + + const canAccess = await _canAccessPage(coverPageID, req.user.decoded.uuid); + if (!canAccess) { + return res.status(403).send({ + err: true, + errMsg: conductorErrors.err8, + }); } - const pageTagsData = await pageTagsRes.json(); - const pageTags = []; - if (Array.isArray(pageTagsData.tag)) { - pageTags.push(...pageTagsData.tag); - } else if (pageTagsData.tag) { - pageTags.push(pageTagsData.tag); + const bookService = new BookService({ bookID: coverPageID }); + const [error, summary] = await _generatePageAISummary(bookService, pageID); + + if (error) { + return _handleAIErrorResponse(res, error); } return res.send({ err: false, - overview: overviewText, - tags: pageTags, + summary, }); } catch (e) { debugError(e); @@ -2206,13 +2207,14 @@ async function getPageDetail( } } -async function getPageAISummary( - req: ZodReqWithUser, +async function getPageAITags( + req: ZodReqWithUser>, res: Response ) { try { - const { pageID } = req.params; + const { pageID: fullPageID } = req.params; const { coverPageID } = req.query; + const [_, pageID] = getLibraryAndPageFromBookID(fullPageID); const canAccess = await _canAccessPage(coverPageID, req.user.decoded.uuid); if (!canAccess) { @@ -2222,41 +2224,15 @@ async function getPageAISummary( }); } - const [error, summary] = await _generatePageAISummary(pageID); - + const bookService = new BookService({ bookID: coverPageID }); + const [error, tags] = await _generatePageAITags(bookService, pageID); if (error) { - switch (error) { - case "location": - return res.status(400).send({ - err: true, - errMsg: conductorErrors.err2, - }); - case "env": - return res.status(500).send({ - err: true, - errMsg: conductorErrors.err6, - }); - case "empty": - return res.send({ - err: false, - summary: "", - }); - case "badres": - return res.status(400).send({ - err: true, - errMsg: "Error generating page summary.", - }); - case "internal": - return res.status(500).send({ - err: true, - errMsg: conductorErrors.err6, - }); - } + return _handleAIErrorResponse(res, error); } return res.send({ err: false, - summary, + tags, }); } catch (e) { debugError(e); @@ -2267,13 +2243,45 @@ async function getPageAISummary( } } -async function batchApplyAISummary( - req: ZodReqWithUser, +function _handleAIErrorResponse(res: Response, error: string) { + switch (error) { + case "location": + return res.status(400).send({ + err: true, + errMsg: conductorErrors.err2, + }); + case "env": + return res.status(500).send({ + err: true, + errMsg: conductorErrors.err6, + }); + case "empty": + return res.send({ + err: true, + errMsg: + "No summary available for this page. There may be insufficient content.", + }); + case "badres": + return res.status(400).send({ + err: true, + errMsg: "Error generating page summary.", + }); + case "internal": + default: + return res.status(500).send({ + err: true, + errMsg: conductorErrors.err6, + }); + } +} + +async function batchGenerateAIMetadata( + req: ZodReqWithUser>, res: Response ) { try { const [coverPageLibrary, coverPageID] = getLibraryAndPageFromBookID( - req.params.pageID + req.params.bookID ); const project = await Project.findOne({ @@ -2288,7 +2296,7 @@ async function batchApplyAISummary( } const canAccess = await _canAccessPage( - req.params.pageID, + req.params.bookID, req.user.decoded.uuid ); if (!canAccess) { @@ -2298,26 +2306,64 @@ async function batchApplyAISummary( }); } - if (!canAccess) { - return res.status(403).send({ + const user = await User.findOne({ uuid: req.user.decoded.uuid }).orFail(); + if (!user || !user.email) { + return res.status(400).send({ err: true, - errMsg: conductorErrors.err8, + errMsg: conductorErrors.err9, }); } - const user = await User.findOne({ uuid: req.user.decoded.uuid }).orFail(); - if (!user || !user.email) { + const activeJob = project.batchUpdateJobs?.filter((j) => + ["pending", "running"].includes(j.status) + ); + if (activeJob && activeJob.length > 0) { return res.status(400).send({ err: true, - errMsg: conductorErrors.err9, + errMsg: "A batch AI summaries job is already running for this project.", }); } - _batchApplySummariesAndNotify( - coverPageLibrary, - coverPageID, + const jobType = + req.body.summaries && req.body.tags + ? "summaries+tags" + : req.body.summaries + ? "summaries" + : "tags"; + + const job: ProjectBookBatchUpdateJob = { + jobID: crypto.randomUUID(), + type: jobType, + status: "pending", + dataSource: "generated", + processedPages: 0, + failedPages: 0, + totalPages: 0, + startTimestamp: new Date(), + ranBy: req.user.decoded.uuid, + }; + + const jobs = project.batchUpdateJobs || []; + jobs.push(job); + + await Project.updateOne( + { + projectID: project.projectID, + }, + { + $set: { + batchUpdateJobs: jobs, + }, + } + ); + + _runBulkUpdateJob( + job.jobID, + job.type, project.projectID, - user.email + req.params.bookID, + job.dataSource, + [user.email] ); // Don't await, send response immediately return res.send({ @@ -2333,89 +2379,395 @@ async function batchApplyAISummary( } } -async function _batchApplySummariesAndNotify( - library: string, - pageID: string, - projectID: string, - emailToNotify: string +async function batchUpdateBookMetadata( + req: ZodReqWithUser>, + res: Response ) { try { - const fullID = `${library}-${pageID}`; - const toc = (await getBookTOCNew(fullID)) as TableOfContents; - const pageIDs: string[] = []; - const content = toc.children; // skip root pages - - // recursively get all page IDs - const getIDs = (content: TableOfContents[]) => { - content.forEach((item) => { - pageIDs.push(item.id); - if (item.children) { - getIDs(item.children); - } + const newPageData = req.body.pages; + if (!newPageData || !Array.isArray(newPageData) || newPageData.length < 1) { + return res.status(400).send({ + err: true, + errMsg: "No page data provided.", }); - }; + } + + const [coverPageLibrary, coverPageID] = getLibraryAndPageFromBookID( + req.params.bookID + ); - getIDs(content); + const project = await Project.findOne({ + libreCoverID: coverPageID, + libreLibrary: coverPageLibrary, + }); + if (!project) { + return res.status(404).send({ + err: true, + errMsg: conductorErrors.err11, + }); + } - const promises = pageIDs.map((p) => - _generatePageAISummary(`${library}-${p}`) + const canAccess = await _canAccessPage( + req.params.bookID, + req.user.decoded.uuid ); - const results = await Promise.allSettled(promises); - const settledPageIDs: { id: string; summary: string }[] = []; - - for (let i = 0; i < results.length; i++) { - const result = results[i]; - if (result.status === "rejected") continue; - if (result.value[0] !== null) continue; - settledPageIDs.push({ - id: pageIDs[i], - summary: result.value[1], + if (!canAccess) { + return res.status(403).send({ + err: true, + errMsg: conductorErrors.err8, }); } - const updatedPromises = settledPageIDs.map((p) => { - // delay 1s between each update to avoid rate limiting - return new Promise>((resolve) => { - setTimeout(async () => { - resolve(await _updatePageDetails(`${library}-${p.id}`, p.summary)); - }, 1000); + const user = await User.findOne({ uuid: req.user.decoded.uuid }).orFail(); + if (!user || !user.email) { + return res.status(400).send({ + err: true, + errMsg: conductorErrors.err9, }); - }); - const updateResults = await Promise.allSettled(updatedPromises); - const failedUpdates = updateResults.filter((r) => r.status === "rejected"); - failedUpdates.forEach((f) => { - debugError(f.reason); - }); + } - await mailAPI.sendBatchAISummariesFinished( - emailToNotify, - projectID, - updateResults.length - failedUpdates.length + const activeJob = project.batchUpdateJobs?.filter((j) => + ["pending", "running"].includes(j.status) ); + if (activeJob && activeJob.length > 0) { + return res.status(400).send({ + err: true, + errMsg: "A batch AI summaries job is already running for this project.", + }); + } + + const job: ProjectBookBatchUpdateJob = { + jobID: crypto.randomUUID(), + type: "summaries+tags", // Default to summaries+tags for user data source + status: "pending", + dataSource: "user", + processedPages: 0, + failedPages: 0, + totalPages: 0, + startTimestamp: new Date(), + ranBy: req.user.decoded.uuid, + }; + + const jobs = project.batchUpdateJobs || []; + jobs.push(job); + + await Project.updateOne( + { + projectID: project.projectID, + }, + { + $set: { + batchUpdateJobs: jobs, + }, + } + ); + + _runBulkUpdateJob( + job.jobID, + job.type, + project.projectID, + req.params.bookID, + job.dataSource, + [user.email], + newPageData + ); // Don't await, send response immediately + + return res.send({ + err: false, + msg: "Batch update started.", + }); } catch (e) { debugError(e); + return res.status(500).send({ + err: true, + errMsg: conductorErrors.err6, + }); } } +async function _runBulkUpdateJob( + jobID: string, + jobType: ProjectBookBatchUpdateJob["type"], + projectID: string, + bookID: string, + dataSource: ProjectBookBatchUpdateJob["dataSource"], + emailsToNotify: string[], + data?: { id: string; summary?: string; tags?: string[] }[] +) { + try { + // Outer catch-block will catch errors with updating a failed job + try { + // Inner catch-block will catch any errors and update job status + if (!data && dataSource === "user") { + throw new Error("No data provided for user data source"); + } + + // Create book service and get table of contents + const bookService = new BookService({ bookID }); + const toc = await bookService.getBookTOCNew(); + const pageIDs: string[] = []; + const content = toc.children; // skip root pages + + // recursively get all page IDs + const getIDs = (content: TableOfContents[]) => { + content.forEach((item) => { + pageIDs.push(item.id); + if (item.children) { + getIDs(item.children); + } + }); + }; + getIDs(content); + + // Update job with initial details + await Project.updateOne( + { + projectID, + }, + { + $set: { + "batchUpdateJobs.$[job].status": "running", + "batchUpdateJobs.$[job].totalPages": pageIDs.length, + }, + }, + { + arrayFilters: [{ "job.jobID": jobID }], + } + ); + + // Initialize new page details array + let newPageDetails: { id: string; summary?: string; tags?: string[] }[] = + []; + if (dataSource === "user") { + newPageDetails = data || []; + } + + // If data source is generated, get page text content and generate tags and/or summaries + if (dataSource === "generated") { + // Get pages text content + const pageTextPromises = pageIDs.map((p) => { + return new Promise((resolve) => { + setTimeout(async () => { + resolve(await bookService.getPageTextContent(p)); + }, 1000); // delay 1s between each page text fetch to avoid rate limiting + }); + }); + + const pageTexts = await Promise.allSettled(pageTextPromises); + const pageTextsMap = new Map(); + pageIDs.forEach((p, i) => { + if (pageTexts[i].status === "fulfilled") { + pageTextsMap.set(p, pageTexts[i].value); + } + }); + + if (["summaries", "summaries+tags"].includes(jobType)) { + const summaryPromises: Promise< + [ + "location" | "env" | "empty" | "badres" | "internal" | null, + string + ] + >[] = []; + + // Create AI summary for each page + pageTextsMap.forEach((pText, pID) => { + // delay 1s between each summary generation to avoid rate limiting + const promise = new Promise< + ReturnType + >((resolve) => { + setTimeout(async () => { + resolve(_generatePageAISummary(bookService, pID, pText)); + }, 1000); + }); + // @ts-ignore + summaryPromises.push(promise); + }); + + const results = await Promise.allSettled(summaryPromises); + + // Add summaries to newPageDetails + for (let i = 0; i < results.length; i++) { + const result = results[i]; + if (result.status === "rejected") continue; + if (result.value[0] !== null) continue; + newPageDetails.push({ + id: pageIDs[i], + summary: result.value[1], + }); + } + } + + if (["tags", "summaries+tags"].includes(jobType)) { + const tagPromises: Promise< + [ + "location" | "env" | "empty" | "badres" | "internal" | null, + string[] + ] + >[] = []; + + // Create AI tags for each page + pageTextsMap.forEach((pText, pID) => { + const promise = new Promise>( + (resolve) => { + setTimeout(async () => { + resolve(_generatePageAITags(bookService, pID, pText)); + }, 1000); // delay 1s between each tag generation to avoid rate limiting + } + ); + // @ts-ignore + tagPromises.push(promise); + }); + + const results = await Promise.allSettled(tagPromises); + + // Add tags to newPageDetails + for (let i = 0; i < results.length; i++) { + const result = results[i]; + if (result.status === "rejected") continue; + if (result.value[0] !== null) continue; + const found = newPageDetails.find((p) => p.id === pageIDs[i]); + if (found) { + found.tags = result.value[1]; + } else { + newPageDetails.push({ + id: pageIDs[i], + tags: result.value[1], + }); + } + } + } + } + + // Bulk update page details + let processed = 0; + let failed = 0; + const BATCH_SIZE = 5; + const resultMessages: string[] = []; + + const updatePromises = newPageDetails.map((p) => { + // delay 1s between each update to avoid rate limiting + return new Promise>( + (resolve) => { + setTimeout(async () => { + resolve(bookService.updatePageDetails(p.id, p.summary, p.tags)); + }, 1000); + } + ); + }); + + for (let i = 0; i < updatePromises.length; i += BATCH_SIZE) { + const batch = updatePromises.slice(i, i + BATCH_SIZE); + const batchResults = await Promise.allSettled(batch); + batchResults.forEach((r) => { + if (r.status === "rejected") { + failed++; + resultMessages.push(r.reason); + } else { + processed++; + resultMessages.push(`Successfully updated ${r.value[0]}.`); + } + }); + console.log( + `JOB ${jobID} Update: Processed ${processed} pages, failed ${failed}.` + ); + + // update job status + await Project.updateOne( + { + projectID, + }, + { + $set: { + "batchUpdateJobs.$[job].processedPages": processed, + "batchUpdateJobs.$[job].failedPages": failed, + }, + }, + { + arrayFilters: [{ "job.jobID": jobID }], + } + ); + } + + // Final update + await Project.updateOne( + { + projectID, + }, + { + $set: { + "batchUpdateJobs.$[job].status": "completed", + "batchUpdateJobs.$[job].endTimestamp": new Date(), + }, + }, + { + arrayFilters: [{ "job.jobID": jobID }], + } + ); + + if (dataSource === "generated") { + await mailAPI.sendBatchBookAIMetadataFinished( + emailsToNotify, + projectID, + jobID, + jobType, + processed + ); + } else { + await mailAPI.sendBatchBookUpdateFinished( + emailsToNotify, + projectID, + jobID, + processed + ); + } + } catch (e: any) { + // Catch any errors and update job status + await Project.updateOne( + { + projectID, + }, + { + $set: { + "batchUpdateJobs.$[job].status": "failed", + "batchUpdateJobs.$[job].endTimestamp": new Date(), + "batchUpdateJobs.$[job].error": e.message + ? e.message + : e.toString(), + }, + }, + { + arrayFilters: [{ "job.jobID": jobID }], + } + ); + } + } catch (err: any) { + debugError(err); + } +} + +/** + * Internal function to generate an AI summary for a page. + * @param pageID - The page ID to generate a summary for. + * @param _pageText - Text content of the page. Optional, and will be fetched if not provided. + * @returns [error, summary] - Error message or null, and the generated summary. + */ async function _generatePageAISummary( - pageID: number | string + bookService: BookService, + pageID: number | string, + _pageText?: string ): Promise< ["location" | "env" | "empty" | "badres" | "internal" | null, string] > { let error = null; let summary = ""; + let pageText = _pageText; try { - const [subdomain, parsedPageID] = getLibraryAndPageFromBookID( - pageID.toString() - ); - if (!subdomain || !parsedPageID) { - throw new Error("location"); - } - // Ensure OpenAI API key is set if (!process.env.OPENAI_API_KEY) throw new Error("env"); - const pageText = await _getPageTextContent(subdomain, parsedPageID); + if (!pageText) { + pageText = await bookService.getPageTextContent(pageID.toString()); + } if (!pageText || pageText.length < 50) throw new Error("empty"); const aiSummaryRes = await axios.post( @@ -2463,36 +2815,28 @@ async function _generatePageAISummary( return [error, summary]; } -async function getPageAITags( - req: ZodReqWithUser, - res: Response -) { +/** + * Internal function to generate AI tags for a page. + * @param pageID - The page ID to generate tags for. + * @param _pageText - Text content of the page. Optional, and will be fetched if not provided. + * @returns [error, tags] - Error message or null, and the generated tags. + */ +async function _generatePageAITags( + bookService: BookService, + pageID: number | string, + _pageText?: string +): Promise< + ["location" | "env" | "empty" | "badres" | "internal" | null, string[]] +> { + let error = null; + let tags = []; + let pageText = _pageText; try { - const { pageID } = req.params; - const { coverPageID } = req.query; - - const canAccess = await _canAccessPage(coverPageID, req.user.decoded.uuid); - if (!canAccess) { - return res.status(403).send({ - err: true, - errMsg: conductorErrors.err8, - }); + if (!pageText) { + pageText = await bookService.getPageTextContent(pageID.toString()); } - - const [subdomain, parsedPageID] = getLibraryAndPageFromBookID(pageID); - if (!subdomain || !parsedPageID) { - return res.status(400).send({ - err: true, - errMsg: conductorErrors.err2, - }); - } - - const pageText = await _getPageTextContent(subdomain, parsedPageID); if (!pageText || pageText.length < 50) { - return res.send({ - err: false, - tags: [], - }); + throw new Error("empty"); } const aiTagsRes = await axios.post( @@ -2526,10 +2870,7 @@ async function getPageAITags( : rawOutput : ""; if (!aiTagsOutput) { - return res.status(400).send({ - err: true, - errMsg: "Error generating page summary.", - }); + throw new Error("badres"); } const splitTags = @@ -2542,54 +2883,15 @@ async function getPageAITags( -1 ); } - - return res.send({ - err: false, - tags: splitTags, - }); - } catch (e) { - debugError(e); - return res.status(500).send({ - err: true, - errMsg: conductorErrors.err6, - }); - } -} - -async function _getPageTextContent( - subdomain: string, - pageID: string -): Promise { - const pageContentsRes = await CXOneFetch({ - scope: "page", - path: parseInt(pageID), - api: MindTouch.API.Page.GET_Page_Contents, - subdomain, - }).catch((err) => { - console.error(err); - throw new Error(`Error fetching page details: ${err}`); - }); - - if (!pageContentsRes.ok) { - throw new Error( - `Error fetching page details: ${pageContentsRes.statusText}` - ); - } - - const pageContent = await pageContentsRes.json(); - const pageRawBody = pageContent.body?.[0]; - if (!pageRawBody) { - return ""; + tags = splitTags; + } catch (err: any) { + error = err.message ?? "internal"; } - - const cheerioObj = cheerio.load(pageRawBody); - const pageText = cheerioObj.text(); // Extract text from HTML - - return pageText; + return [error, tags]; } async function updatePageDetails( - req: ZodReqWithUser, + req: ZodReqWithUser>, res: Response ) { try { @@ -2605,11 +2907,17 @@ async function updatePageDetails( }); } - const [error, success] = await _updatePageDetails(pageID, summary, tags); + const bookService = new BookService(coverPageID); + const [error, success] = await bookService.updatePageDetails( + pageID, + summary, + tags + ); + if (error) { switch (error) { case "location": - return res.status(400).send({ + return res.status(404).send({ err: true, errMsg: conductorErrors.err2, }); @@ -2641,99 +2949,51 @@ async function updatePageDetails( } } -async function _updatePageDetails( - pageID: string, - summary?: string, - tags?: string[] -): Promise<["location" | "internal" | null, boolean]> { - let error = null; - let success = false; +async function bulkUpdatePageTags( + req: ZodReqWithUser>, + res: Response +) { try { - const [subdomain, parsedPageID] = getLibraryAndPageFromBookID(pageID); - if (!subdomain || !parsedPageID) { - throw new Error("location"); - } - - // Get current page properties and find the overview property - const pagePropertiesRes = await CXOneFetch({ - scope: "page", - path: parseInt(parsedPageID), - api: MindTouch.API.Page.GET_Page_Properties, - subdomain, - }).catch((err) => { - console.error(err); - throw new Error("internal"); - }); + const { bookID } = req.params; + const { pages } = req.body; - if (!pagePropertiesRes.ok) { - throw new Error("internal"); - } + const bookService = new BookService({ bookID }); - const pagePropertiesRaw = await pagePropertiesRes.json(); - const pageProperties = Array.isArray(pagePropertiesRaw?.property) - ? pagePropertiesRaw.property - : [pagePropertiesRaw?.property]; - - // Check if there is an existing overview property - const overviewProperty = pageProperties - .filter((p) => !!p) - .find((prop: any) => prop["@name"] === MindTouch.PageProps.PageOverview); - - if (summary) { - // Update or set page overview property - const updatedOverviewRes = await CXOneFetch({ - scope: "page", - path: parseInt(parsedPageID), - api: MindTouch.API.Page.PUT_Page_Property( - MindTouch.PageProps.PageOverview - ), - subdomain, - options: { - method: "PUT", - headers: { - "Content-Type": "text/plain", - ...(overviewProperty && - overviewProperty["@etag"] && { - Etag: overviewProperty["@etag"], - }), - }, - body: summary, - }, + const updatePromises = []; + for (let i = 0; i < pages.length; i++) { + const promise = new Promise((resolve, reject) => { + setTimeout(async () => { + const page = pages[i]; + const [error, success] = await bookService.updatePageDetails( + page.id, + undefined, + page.tags + ); + if (error) { + reject(error); + } + resolve({ error, success }); + }, 1000); }); - - if (!updatedOverviewRes.ok) { - throw new Error("internal"); - } + updatePromises.push(promise); } - if (tags) { - // Update the page tags - const updatedTagsRes = await CXOneFetch({ - scope: "page", - path: parseInt(parsedPageID), - api: MindTouch.API.Page.PUT_Page_Tags, - subdomain, - options: { - method: "PUT", - headers: { - "Content-Type": "application/xml", - }, - body: MindTouch.Templates.PUT_PageTags(tags), - }, - }); + const results = await Promise.allSettled(updatePromises); + const failed = results.filter((r) => r.status === "rejected").length; + const processed = results.filter((r) => r.status === "fulfilled").length; - if (!updatedTagsRes.ok) { - throw new Error("internal"); - } - } - - success = true; - } catch (e: any) { - error = e.message ?? "internal"; - success = false; + return res.send({ + err: false, + failed, + processed, + }); + } catch (err) { + debugError(err); + return res.status(500).send({ + err: true, + errMsg: conductorErrors.err6, + }); } - - return [error, success]; } /** @@ -2973,7 +3233,7 @@ async function _canAccessPage( return false; } - return await projectsAPI.checkProjectMemberPermission(project, userID); + return projectsAPI.checkProjectMemberPermission(project, userID); } catch (err) { debugError(err); return false; @@ -2993,13 +3253,15 @@ export default { addBookToCustomCatalog, removeBookFromCustomCatalog, downloadBookFile, - getBookSummary, getBookTOC, getLicenseReport, + getBookPagesDetails, getPageDetail, getPageAISummary, - batchApplyAISummary, + batchGenerateAIMetadata, + batchUpdateBookMetadata, getPageAITags, updatePageDetails, + bulkUpdatePageTags, retrieveKBExport, }; diff --git a/server/api/mail.js b/server/api/mail.js index f1101f3e..c1311f45 100644 --- a/server/api/mail.js +++ b/server/api/mail.js @@ -759,21 +759,51 @@ const sendOrgEventRegistrationConfirmation = (addresses, orgEvent, participantNa }; /** - * Sends a notification to the specified email addresses that page AI summaries have been generated and applied. + * Sends a notification to the specified email addresses that page AI metadata has been generated and applied. * @param {string[]} recipientAddresses - the email addresses to send the notification to - * @param {string} projectID - the ID of the connect project - * @param {number} updated - the number of pages updated with AI summaries + * @param {string} projectID - the ID of the project + * @param {string} jobID - the ID of the job that was completed + * @param {string} jobType - the type of job that was completed (i.e. summaries, tags, or both) + * @param {number} updated - the number of pages updated */ -const sendBatchAISummariesFinished = (recipientAddresses, projectID, updated) => { +const sendBatchBookAIMetadataFinished = (recipientAddresses, projectID, jobID, jobType, updated) => { return mailgun.messages.create(process.env.MAILGUN_DOMAIN, { from: 'LibreTexts Support ', to: recipientAddresses, - subject: `AI Summaries Finished - Project ${projectID}`, + subject: `AI Metadata Generation Finished - Project ${projectID}`, html: `

Hi,

-

We're just writing to let you know that we've finished applying AI-generated page summaries to the textbook associated with this project.

+

We're just writing to let you know that we've finished applying AI-generated page ${jobType === 'summaries+tags' ? 'summaries and tags' : jobType === 'summaries' ? 'summaries' : 'tags'} to the textbook associated with this project.

Updated: ${updated} pages

-

Please note that page summaries are cached and may take a few minutes to appear in the library.

+

Job ID: ${jobID}

+

Please note that page summaries/tags are cached and may take a few minutes to appear in the library.

+

Sincerely,

+

The LibreTexts team

+
+ ${autoGenNoticeHTML} + `, + }); +}; + +/** + * Sends a notification to the specified email addresses that page AI metadata has been generated and applied. + * @param {string[]} recipientAddresses - the email addresses to send the notification to + * @param {string} projectID - the ID of the project + * @param {string} jobID - the ID of the job that was completed + * @param {string} jobType - the type of job that was completed (i.e. summaries, tags, or both) + * @param {number} updated - the number of pages updated + */ +const sendBatchBookUpdateFinished = (recipientAddresses, projectID, jobID, updated) => { + return mailgun.messages.create(process.env.MAILGUN_DOMAIN, { + from: 'LibreTexts Support ', + to: recipientAddresses, + subject: `Bulk Update Job Finished - Project ${projectID}`, + html: ` +

Hi,

+

We're just writing to let you know that we've finished applying your updates to the textbook associated with this project.

+

Updated: ${updated} pages

+

Job ID: ${jobID}

+

Please note that page summaries/tags are cached and may take a few minutes to appear in the library.

Sincerely,

The LibreTexts team


@@ -1097,7 +1127,8 @@ export default { sendAnalyticsInvite, sendAnalyticsInviteAccepted, sendOrgEventRegistrationConfirmation, - sendBatchAISummariesFinished, + sendBatchBookAIMetadataFinished, + sendBatchBookUpdateFinished, sendSupportTicketCreateConfirmation, sendSupportTicketCreateInternalNotification, sendNewTicketMessageNotification, diff --git a/server/api/projects.js b/server/api/projects.js index 94331cd6..39569ff6 100644 --- a/server/api/projects.js +++ b/server/api/projects.js @@ -3557,7 +3557,8 @@ const validate = (method) => { ] case 'getProject': return [ - query('projectID', conductorErrors.err1).exists().isString().isLength({ min: 10, max: 10 }) + query('projectID', conductorErrors.err1).exists().isString().isLength({ min: 10, max: 10 }), + query('include', conductorErrors.err1).optional({ checkFalsy: true }).isArray() ] case 'getUserProjectsAdmin': return [ diff --git a/server/api/services/book-service.ts b/server/api/services/book-service.ts new file mode 100644 index 00000000..1f699f85 --- /dev/null +++ b/server/api/services/book-service.ts @@ -0,0 +1,431 @@ +import { getLibraryAndPageFromBookID } from "../../util/bookutils"; +import { CXOneFetch } from "../../util/librariesclient"; +import MindTouch from "../../util/CXOne"; +import { + GetPageSubPagesResponse, + PageBase, + PageDetailsResponse, + PageSimpleWOverview, + PageSimpleWTags, + PageTag, + TableOfContents, +} from "../../types"; +import * as cheerio from "cheerio"; +import Book from "../../models/book"; + +export interface BookServiceParams { + bookID: string; +} + +export default class BookService { + private _bookID: string = ""; + private _library: string = ""; + private _coverID: string = ""; + constructor(params: BookServiceParams) { + if (!params.bookID) { + throw new Error("Missing bookID"); + } + this._bookID = params.bookID; + + const [library, coverID] = getLibraryAndPageFromBookID(params.bookID); + if (!library || !coverID) { + throw new Error("Invalid bookID"); + } + + this._library = library; + this._coverID = coverID; + } + + get bookID(): string { + return this._bookID; + } + + get library(): string { + return this._library; + } + + get coverID(): string { + return this._coverID; + } + + async getBookSummary(): Promise { + const book = await Book.findOne({ bookID: this._bookID }); + if (!book) { + return undefined; + } + return book?.summary || ""; + } + + async getBookTOCNew(): Promise { + const res = await CXOneFetch({ + scope: "page", + path: parseInt(this._coverID), + api: MindTouch.API.Page.GET_Page_Tree, + subdomain: this._library, + options: { + method: "GET", + }, + }); + const rawTree = (await res.json()) as GetPageSubPagesResponse; + + function _buildHierarchy( + page: GetPageSubPagesResponse["page"] | PageBase, + parentID?: number + ): (GetPageSubPagesResponse["page"] | PageBase) & { + parentID?: number; + subpages?: TableOfContents[]; + } { + const pageID = Number.parseInt(page["@id"], 10); + const subpages = []; + + // @ts-ignore + const processPage = (p) => ({ + ...p, + id: pageID, + url: p["uri.ui"], + }); + + if ("subpages" in page) { + if (Array.isArray(page?.subpages?.page)) { + page.subpages.page.forEach((p) => + subpages.push(_buildHierarchy(p, pageID)) + ); + } else if (typeof page?.subpages?.page === "object") { + // single page + subpages.push(_buildHierarchy(page.subpages.page, pageID)); + } + } + + return processPage({ + ...page, + ...(parentID && { parentID }), + ...(subpages.length && { subpages }), + }); + } + + const structured = _buildHierarchy(rawTree?.page); + const buildStructure = ( + page: GetPageSubPagesResponse["page"] | PageBase + ): TableOfContents => ({ + children: + "subpages" in page && Array.isArray(page.subpages) + ? page.subpages.map((s) => buildStructure(s)) + : [], + id: page["@id"], + title: page.title, + url: page["uri.ui"], + }); + + return buildStructure(structured); + } + + async getAllPageOverviews( + toc?: TableOfContents + ): Promise { + if (!toc) { + toc = await this.getBookTOCNew(); + } + + // Recursive function to collect all page IDs + const collectPageData = ( + toc: TableOfContents + ): { id: string; title: string; url: string }[] => { + return [ + { id: toc.id, title: toc.title, url: toc.url }, + ...toc.children.flatMap(collectPageData), + ]; + }; + + const flattenedPageData = collectPageData(toc); + + const overviewPromises: Promise[] = []; + for (const page of flattenedPageData) { + // Add a 1s delay between each fetch to avoid rate limiting + const _promise = new Promise((resolve) => { + setTimeout(async () => { + resolve(this.getPageOverview(page.id)); + }, 1000); + }); + overviewPromises.push(_promise); + } + + // Create an array of objects with the page ID, title, url, and its overview property + const results = await Promise.allSettled(overviewPromises); + const pageOverviews: PageSimpleWOverview[] = []; + for (let i = 0; i < results.length; i++) { + const _page = flattenedPageData[i]; + const _result = results[i]; + if (_result.status === "fulfilled") { + pageOverviews.push({ + id: _page.id, + title: _page.title, + url: _page.url, + overview: _result.value, + }); + } + } + + return pageOverviews; + } + + async getAllPageTags(toc?: TableOfContents): Promise { + if (!toc) { + toc = await this.getBookTOCNew(); + } + + // Recursive function to collect all page IDs + const collectPageData = ( + toc: TableOfContents + ): { id: string; title: string; url: string }[] => { + return [ + { id: toc.id, title: toc.title, url: toc.url }, + ...toc.children.flatMap(collectPageData), + ]; + }; + + const flattenedPageData = collectPageData(toc); + + const tagsPromises: Promise[] = []; + for (const page of flattenedPageData) { + // Add a 1s delay between each fetch to avoid rate limiting + const _promise = new Promise((resolve) => { + setTimeout(async () => { + resolve(this.getPageTags(page.id)); + }, 1000); + }); + tagsPromises.push(_promise); + } + + // Create an array of objects with the page ID, title, url, and its tags + const results = await Promise.allSettled(tagsPromises); + const pageTags: PageSimpleWTags[] = []; + for (let i = 0; i < results.length; i++) { + const _page = flattenedPageData[i]; + const _result = results[i]; + if (_result.status === "fulfilled") { + const tags = _result.value; + const valueOnly = tags.map((t) => t["@value"]); + pageTags.push({ + id: _page.id, + title: _page.title, + url: _page.url, + tags: valueOnly, + }); + continue; + } else { + const tags: PageTag[] = []; + pageTags.push({ + id: _page.id, + title: _page.title, + url: _page.url, + tags: [], + }); + } + } + + return pageTags; + } + + async getPageDetails( + pageID: string + ): Promise { + if (!pageID) { + throw new Error("Missing pageID"); + } + + const overview = await this.getPageOverview(pageID); + const tags = await this.getPageTags(pageID); + + return { + overview, + tags, + }; + } + + async getPageOverview(pageID: string): Promise { + if (!pageID) { + throw new Error("Missing page ID"); + } + + const pagePropertiesRes = await CXOneFetch({ + scope: "page", + path: parseInt(pageID), + api: MindTouch.API.Page.GET_Page_Properties, + subdomain: this._library, + }).catch((err) => { + console.error(err); + throw new Error(`Error fetching page details: ${err}`); + }); + + if (!pagePropertiesRes.ok) { + throw new Error( + `Error fetching page details: ${pagePropertiesRes.statusText}` + ); + } + + const pagePropertiesRaw = await pagePropertiesRes.json(); + const pageProperties = Array.isArray(pagePropertiesRaw?.property) + ? pagePropertiesRaw.property + : [pagePropertiesRaw?.property]; + const overviewProperty = pageProperties + .filter((p: any) => !!p) + .find((prop: any) => prop["@name"] === MindTouch.PageProps.PageOverview); + const overviewText = overviewProperty?.contents?.["#text"] || ""; + + return overviewText; + } + + async getPageTags(pageID: string): Promise { + if (!pageID) { + throw new Error("Missing page ID"); + } + + const pageTagsRes = await CXOneFetch({ + scope: "page", + path: parseInt(pageID), + api: MindTouch.API.Page.GET_Page_Tags, + subdomain: this._library, + }).catch((err) => { + console.error(err); + throw new Error(`Error fetching page tags: ${err}`); + }); + + if (!pageTagsRes.ok) { + throw new Error(`Error fetching page tags: ${pageTagsRes.statusText}`); + } + + const pageTagsData = await pageTagsRes.json(); + const pageTags = []; + if (Array.isArray(pageTagsData.tag)) { + pageTags.push(...pageTagsData.tag); + } else if (pageTagsData.tag) { + pageTags.push(pageTagsData.tag); + } + + return pageTags; + } + + async getPageTextContent(pageID: string): Promise { + const pageContentsRes = await CXOneFetch({ + scope: "page", + path: parseInt(pageID), + api: MindTouch.API.Page.GET_Page_Contents, + subdomain: this._library, + }).catch((err) => { + console.error(err); + throw new Error(`Error fetching page details: ${err}`); + }); + + if (!pageContentsRes.ok) { + throw new Error( + `Error fetching page details: ${pageContentsRes.statusText}` + ); + } + + const pageContent = await pageContentsRes.json(); + const pageRawBody = pageContent.body?.[0]; + if (!pageRawBody) { + return ""; + } + + const cheerioObj = cheerio.load(pageRawBody); + const pageText = cheerioObj.text(); // Extract text from HTML + + return pageText; + } + + async updatePageDetails( + pageID: string, + summary?: string, + tags?: string[] + ): Promise<["location" | "internal" | null, boolean]> { + let error = null; + let success = false; + try { + if (!pageID) { + throw new Error("location"); + } + // Get current page properties and find the overview property + const pagePropertiesRes = await CXOneFetch({ + scope: "page", + path: parseInt(pageID), + api: MindTouch.API.Page.GET_Page_Properties, + subdomain: this._library, + }).catch((err) => { + console.error(err); + throw new Error("internal"); + }); + + if (!pagePropertiesRes.ok) { + throw new Error("internal"); + } + + const pagePropertiesRaw = await pagePropertiesRes.json(); + const pageProperties = Array.isArray(pagePropertiesRaw?.property) + ? pagePropertiesRaw.property + : [pagePropertiesRaw?.property]; + + // Check if there is an existing overview property + const overviewProperty = pageProperties + .filter((p: any) => !!p) + .find( + (prop: any) => prop["@name"] === MindTouch.PageProps.PageOverview + ); + + if (summary) { + // Update or set page overview property + const updatedOverviewRes = await CXOneFetch({ + scope: "page", + path: parseInt(pageID), + api: MindTouch.API.Page.PUT_Page_Property( + MindTouch.PageProps.PageOverview + ), + subdomain: this._library, + options: { + method: "PUT", + headers: { + "Content-Type": "text/plain", + ...(overviewProperty && + overviewProperty["@etag"] && { + Etag: overviewProperty["@etag"], + }), + }, + body: summary, + }, + }); + + if (!updatedOverviewRes.ok) { + throw new Error("internal"); + } + } + + if (tags) { + // Update the page tags + const updatedTagsRes = await CXOneFetch({ + scope: "page", + path: parseInt(pageID), + api: MindTouch.API.Page.PUT_Page_Tags, + subdomain: this._library, + options: { + method: "PUT", + headers: { + "Content-Type": "application/xml", + }, + body: MindTouch.Templates.PUT_PageTags(tags), + }, + }); + + if (!updatedTagsRes.ok) { + throw new Error("internal"); + } + } + + success = true; + } catch (err: any) { + error = err.message ?? "internal"; + success = false; + } + + return [error, success]; + } +} diff --git a/server/api/validators/book.ts b/server/api/validators/book.ts index 842ebee6..e04fe0cc 100644 --- a/server/api/validators/book.ts +++ b/server/api/validators/book.ts @@ -118,3 +118,56 @@ export const updatePageDetailsSchema = z.object({ }), }), }); + +// For AI-generated metadata +export const batchGenerateAIMetadataSchema = z.object({ + params: z.object({ + bookID: z.string().refine(checkBookIDFormat, { + message: conductorErrors.err1, + }), + }), + body: z + .object({ + summaries: z.coerce.boolean().optional(), + tags: z.coerce.boolean().optional(), + }) + .refine((data) => data.summaries || data.tags, { + message: "At least one of 'summaries' or 'tags' must be true", + }), +}); + +// For user-defined metadata +export const batchUpdateBookMetadataSchema = z.object({ + params: z.object({ + bookID: z.string().refine(checkBookIDFormat, { + message: conductorErrors.err1, + }), + }), + body: z.object({ + pages: z.array( + z.object({ + id: z.string(), + summary: z.string().max(500).optional(), + tags: z.array(z.string().max(255)).max(100).optional(), + }) + ), + }), +}); + +const _ReducedPageSimpleWTagsSchema = z.array( + z.object({ + id: z.string(), + tags: z.array(z.string()), + }) +); + +export const bulkUpdatePageTagsSchema = z.object({ + params: z.object({ + bookID: z.string().refine(checkBookIDFormat, { + message: conductorErrors.err1, + }), + }), + body: z.object({ + pages: _ReducedPageSimpleWTagsSchema, + }), +}); diff --git a/server/models/project.ts b/server/models/project.ts index a938dbaa..cd59a586 100644 --- a/server/models/project.ts +++ b/server/models/project.ts @@ -15,6 +15,23 @@ export type ProjectModuleSettings = { tasks: ProjectModuleConfig; }; +export type ProjectBookBatchUpdateJob = { + jobID: string; + type: "summaries" | "tags" | "summaries+tags"; + status: "pending" | "running" | "completed" | "failed"; + processedPages: number; + failedPages: number; + totalPages: number; + dataSource: "user" | "generated"; + ranBy: string; // User UUID + startTimestamp?: Date; + endTimestamp?: Date; + error?: string; // root-level error message, not for individual pages + results?: { + [key: string]: any; + }; +} + export interface ProjectInterface extends Document { orgID: string; projectID: string; @@ -81,6 +98,7 @@ export interface ProjectInterface extends Document { sourceHarvestDate?: Date; sourceLastModifiedDate?: Date; sourceLanguage?: string; + batchUpdateJobs?: ProjectBookBatchUpdateJob[]; } const ProjectSchema = new Schema( @@ -429,6 +447,36 @@ const ProjectSchema = new Schema( * Language of the source material. */ sourceLanguage: String, + /** + * Batch Update Jobs (can be user-defined content or AI generated). + */ + batchUpdateJobs: [ + { + jobID: String, + type: { + type: String, + enum: ["summaries", "tags", "summaries+tags"], + }, + status: { + type: String, + enum: ["pending", "running", "completed", "failed"], + }, + processedPages: Number, + failedPages: Number, + totalPages: Number, + dataSource: { + type: String, + enum: ["user", "generated"], + }, + ranBy: String, + startTimestamp: Date, + endTimestamp: Date, + error: String, + results: { + type: Schema.Types.Mixed, + }, + } + ], }, { timestamps: true, diff --git a/server/types/Book.ts b/server/types/Book.ts index e5a8b834..ca54218f 100644 --- a/server/types/Book.ts +++ b/server/types/Book.ts @@ -1,3 +1,5 @@ +import { Prettify } from "./Misc"; + export type BookSortOption = "random" | "author" | "title"; export type TableOfContents = { id: string; @@ -5,3 +7,76 @@ export type TableOfContents = { url: string; children: TableOfContents[]; }; + +export type PageTag = { + "@value": string; + "@id": string; + "@href": string; + title: string; + type: string; + uri: string; +}; + +export type PageDetailsResponse = { + overview: string; + tags: PageTag[]; +}; + +export type PageBase = { + "@id": string; + "@guid": string; + "@draft.state": string; + "@href": string; + "@deleted": string; + "@revision": string; + article: string; + "date.created": string; + "date.modified": string; + language: string; + namespace: string; + path: PagePath; + security: Record; + title: string; + "uri.ui": string; +}; + +export type PagePath = { + "@seo": string; + "@type": string; + "#text": string; +}; + +export type GetPageSubPagesResponse = { + page: Omit & { + properties: Record; + subpages: { + page: PageBase | PageBase[]; + }; + }; +}; + +type _PageSimple = { + id: string; + title: string; + url: string; +}; + +export type PageSimpleWTags = Prettify< + _PageSimple & { + tags: string[]; + } +>; + +export type PageSimpleWOverview = Prettify< + _PageSimple & { + overview: string; + } +>; + +export type TableOfContentsDetailed = Prettify< + Omit & { + overview: string; + tags: string[]; + children: TableOfContentsDetailed[]; + } +>; diff --git a/server/types/Misc.ts b/server/types/Misc.ts index 45e55922..5cfb335e 100644 --- a/server/types/Misc.ts +++ b/server/types/Misc.ts @@ -15,4 +15,13 @@ export type License = { sourceURL?: string; modifiedFromSource?: boolean; additionalTerms?: string; -} \ No newline at end of file +}; + +/** + * A TypeScript type alias called `Prettify`. + * It takes a type as its argument and returns a new type that has the same properties as the original type, + * but the properties are not intersected. This means that the new type is easier to read and understand. + */ +export type Prettify = { + [K in keyof T]: T[K]; +} & {}; diff --git a/server/util/bookutils.js b/server/util/bookutils.js index 9b7fe0fc..a9a07ff0 100644 --- a/server/util/bookutils.js +++ b/server/util/bookutils.js @@ -259,58 +259,6 @@ export const getBookTOCFromAPI = async (bookID, bookURL) => { return buildStructure(tocRes.data.toc.structured); }; -export const getBookTOCNew = async (bookID) => { - if(!checkBookIDFormat(bookID)) throw new Error('bookid'); - const library = bookID.split("-")[0]; - const pageID = bookID.split("-")[1]; - - const res = await CXOneFetch({ - scope: "page", - path: pageID, - api: MindTouch.API.Page.GET_Page_Tree, - subdomain: library, - options: { - method: "GET", - } - }); - const rawTree = await res.json() - - function _buildHierarchy(page, parentID) { - const pageID = Number.parseInt(page['@id'], 10); - const subpages = []; - - const processPage = (p) => ({ - ...p, - id: pageID, - url: p['uri.ui'], - }); - - - if (Array.isArray(page?.subpages?.page)) { - page.subpages.page.forEach((p) => subpages.push(_buildHierarchy(p, pageID))); - } else if (typeof page?.subpages?.page === 'object') { - // single page - subpages.push(_buildHierarchy(page.subpages.page, pageID)); - } - - return processPage({ - ...page, - ...(parentID && { parentID }), - ...(subpages.length && { subpages }), - }); - } - - const structured = _buildHierarchy(rawTree?.page); - const buildStructure = (page) => ({ - children: Array.isArray(page.subpages) ? page.subpages.map((s) => buildStructure(s)) : [], - id: page['@id'], - title: page.title, - url: page.url, - }); - - return buildStructure(structured) -} - export const deleteBookFromAPI = async (bookID) => { if (!process.env.LIBRE_API_ENDPOINT_ACCESS) { throw new Error('missing API key'); diff --git a/server/util/librariesclient.ts b/server/util/librariesclient.ts index 51302bc4..f504be29 100644 --- a/server/util/librariesclient.ts +++ b/server/util/librariesclient.ts @@ -2,7 +2,7 @@ import { SSMClient, GetParametersByPathCommand } from "@aws-sdk/client-ssm"; import { debugError } from "../debug.js"; import { CXOneFetchParams, - LibrariesSSMClient, + LibrariesSSMClient as LibrariesSSMClientType, LibraryAPIRequestHeaders, LibraryTokenPair, CXOneGroup, @@ -12,85 +12,106 @@ import { createHmac } from "crypto"; import CXOne from "./CXOne/index.js"; import { libraryNameKeys, libraryNameKeysWDev } from "./librariesmap.js"; -export async function generateLibrariesSSMClient(): Promise { - try { - const libTokenPairPath = (process.env.AWS_SSM_LIB_TOKEN_PAIR_PATH || "/libkeys/production").replace(/['"]/g, ''); - const apiUsername = process.env.LIBRARIES_API_USERNAME || "LibreBot"; +/** + * Singleton class for interacting with AWS SSM service to retrieve library API tokens. + */ +class LibrariesSSMClient { + public apiUsername: string = "LibreBot"; + public libTokenPairPath: string = "/libkeys/production"; + public ssm: SSMClient = new SSMClient({ + credentials: { + accessKeyId: process.env.AWS_SSM_ACCESS_KEY_ID || "unknown", + secretAccessKey: process.env.AWS_SSM_SECRET_KEY || "unknown", + }, + region: process.env.AWS_SSM_REGION || "unknown", + }); + + public credentialsCache: Record< + string, + { keyPair: LibraryTokenPair; apiUsername: string; refreshAfter: Date } + > = {}; + + private static instance: LibrariesSSMClient; + + private constructor() { + this.apiUsername = process.env.LIBRARIES_API_USERNAME || "LibreBot"; + this.libTokenPairPath = ( + process.env.AWS_SSM_LIB_TOKEN_PAIR_PATH || "/libkeys/production" + ).replace(/['"]/g, ""); + } - const ssm = new SSMClient({ - credentials: { - accessKeyId: process.env.AWS_SSM_ACCESS_KEY_ID || "unknown", - secretAccessKey: process.env.AWS_SSM_SECRET_KEY || "unknown", - }, - region: process.env.AWS_SSM_REGION || "unknown", - }); + public static getInstance() { + if (!LibrariesSSMClient.instance) { + LibrariesSSMClient.instance = new LibrariesSSMClient(); + } - return { - apiUsername, - libTokenPairPath, - ssm, - }; - } catch (err) { - debugError(err); - return null; + return LibrariesSSMClient.instance; } -} -/** - * Retrieves the token pair requried to interact with a library's API. - */ -export async function getLibraryCredentials( - lib: string -): Promise<{ keyPair: LibraryTokenPair; apiUsername: string } | null> { - try { - const ssmClient = await generateLibrariesSSMClient(); - if (!ssmClient) { - console.error("Failed to generate SSMClient - Null value returned."); - throw new Error("Error generating SSMClient."); - } + public async getLibraryCredentials(lib: string) { + try { + // Check if credentials are cached and still valid + if (this.credentialsCache[lib]) { + const cached = this.credentialsCache[lib]; + if (cached.refreshAfter > new Date()) { + return cached; + } + } - const basePath = ssmClient.libTokenPairPath.endsWith("/") - ? ssmClient.libTokenPairPath - : `${ssmClient.libTokenPairPath}/`; - const pairResponse = await ssmClient.ssm.send( - new GetParametersByPathCommand({ - Path: `${basePath}${lib}`, - MaxResults: 10, - Recursive: true, - WithDecryption: true, - }) - ); - - if (pairResponse.$metadata.httpStatusCode !== 200) { - console.error(pairResponse.$metadata); - throw new Error("Error retrieving library token pair."); - } - if (!pairResponse.Parameters) { - console.error("No data returned from token pair retrieval. Lib: " + lib); - throw new Error("Error retrieving library token pair."); - } + // If not, retrieve from SSM + const basePath = this.libTokenPairPath.endsWith("/") + ? this.libTokenPairPath + : `${this.libTokenPairPath}/`; + + const pairResponse = await this.ssm.send( + new GetParametersByPathCommand({ + Path: `${basePath}${lib}`, + MaxResults: 10, + Recursive: true, + WithDecryption: true, + }) + ); - const libKey = pairResponse.Parameters.find((p) => - p.Name?.includes(`${lib}/key`) - ); - const libSec = pairResponse.Parameters.find((p) => - p.Name?.includes(`${lib}/secret`) - ); - if (!libKey?.Value || !libSec?.Value) { - console.error("Key param not found in token pair retrieval. Lib: " + lib); - throw new Error("Error retrieving library token pair."); - } + if (pairResponse.$metadata.httpStatusCode !== 200) { + console.error(pairResponse.$metadata); + throw new Error("Error retrieving library token pair."); + } + if (!pairResponse.Parameters) { + console.error( + "No data returned from token pair retrieval. Lib: " + lib + ); + throw new Error("Error retrieving library token pair."); + } - return { - keyPair: { - key: libKey.Value, - secret: libSec.Value, - }, - apiUsername: ssmClient.apiUsername, - }; - } catch (err) { - debugError(err); - return null; + const libKey = pairResponse.Parameters.find((p) => + p.Name?.includes(`${lib}/key`) + ); + const libSec = pairResponse.Parameters.find((p) => + p.Name?.includes(`${lib}/secret`) + ); + if (!libKey?.Value || !libSec?.Value) { + console.error( + "Key param not found in token pair retrieval. Lib: " + lib + ); + throw new Error("Error retrieving library token pair."); + } + + // Push to cache and return + const creds = { + keyPair: { + key: libKey.Value, + secret: libSec.Value, + }, + apiUsername: this.apiUsername, + refreshAfter: new Date(Date.now() + 30 * 60 * 1000) // 30 minutes + }; + + this.credentialsCache[lib] = creds; + return creds; + } catch (err) { + debugError(err); + return null; + } } } @@ -102,7 +123,12 @@ export async function generateAPIRequestHeaders( lib: string ): Promise { try { - const creds = await getLibraryCredentials(lib); + const libClient = LibrariesSSMClient.getInstance(); + if (!libClient) { + throw new Error("Error generating library token pair. LibrariesSSMClient is null."); + } + + const creds = await libClient.getLibraryCredentials(lib); if (!creds || !creds.keyPair || !creds.apiUsername) { console.log("Failed attempt to generate library token pair."); throw new Error("Error generating library token pair.");