Merge branch 'feature-new-attachments'

Fixes #251

pages/link/share_target.tsx

@@ -75,11 +75,11 @@ function AppShareTarget() {
     if (intentURL) {
       setIsDownloading(true);
       callBrowseFetchPage(intentURL)
-        .then(pageContent => {
-          if (pageContent)
-            queueComposerTextAndLaunchApp('\n\n```' + intentURL + '\n' + pageContent + '\n```\n');
+        .then(page => {
+          if (page.stopReason !== 'error')
+            queueComposerTextAndLaunchApp('\n\n```' + intentURL + '\n' + page.content + '\n```\n');
           else
-            setErrorMessage('Could not read any data');
+            setErrorMessage('Could not read any data' + page.error ? ': ' + page.error : '');
         })
         .catch(error => setErrorMessage(error?.message || error || 'Unknown error'))
         .finally(() => setIsDownloading(false));

src/apps/call/CallUI.tsx

@@ -113,7 +113,7 @@ export function CallUI(props: {
         setCallMessages(messages => [...messages, createDMessage('user', transcribed)]);
     }
   }, []);
-  const { isSpeechEnabled, isRecording, isRecordingAudio, isRecordingSpeech, startRecording, stopRecording, toggleRecording } = useSpeechRecognition(onSpeechResultCallback, 1000, false);
+  const { isSpeechEnabled, isRecording, isRecordingAudio, isRecordingSpeech, startRecording, stopRecording, toggleRecording } = useSpeechRecognition(onSpeechResultCallback, 1000);
 
   // derived state
   const isRinging = stage === 'ring';

src/apps/chat/AppChat.tsx

@@ -12,7 +12,7 @@ import { TradeConfig, TradeModal } from '~/modules/trade/TradeModal';
 import { imaginePromptFromText } from '~/modules/aifn/imagine/imaginePromptFromText';
 import { speakText } from '~/modules/elevenlabs/elevenlabs.client';
 import { useBrowseStore } from '~/modules/browse/store-module-browsing';
-import { useModelsStore } from '~/modules/llms/store-llms';
+import { useChatLLM, useModelsStore } from '~/modules/llms/store-llms';
 
 import { ConfirmationModal } from '~/common/components/ConfirmationModal';
 import { GlobalShortcutItem, ShortcutKeyName, useGlobalShortcuts } from '~/common/components/useGlobalShortcut';
@@ -21,6 +21,7 @@ import { createDMessage, DConversationId, DMessage, getConversation, useConversa
 import { openLayoutLLMOptions, useLayoutPluggable } from '~/common/layout/store-applayout';
 import { useUXLabsStore } from '~/common/state/store-ux-labs';
 
+import type { ComposerOutputMultiPart } from './components/composer/composer.types';
 import { ChatDrawerItemsMemo } from './components/applayout/ChatDrawerItems';
 import { ChatDropdowns } from './components/applayout/ChatDropdowns';
 import { ChatMenuItems } from './components/applayout/ChatMenuItems';
@@ -57,6 +58,8 @@ export function AppChat() {
   const composerTextAreaRef = React.useRef<HTMLTextAreaElement>(null);
 
   // external state
+  const { chatLLM } = useChatLLM();
+
   const {
     chatPanes,
     focusedConversationId,
@@ -180,13 +183,33 @@ export function AppChat() {
     setMessages(conversationId, history);
   }, [focusedSystemPurposeId, setMessages]);
 
-  const handleComposerNewMessage = async (chatModeId: ChatModeId, conversationId: DConversationId, userText: string) => {
+  const handleComposerAction = (chatModeId: ChatModeId, conversationId: DConversationId, multiPartMessage: ComposerOutputMultiPart): boolean => {
+
+    // validate inputs
+    if (multiPartMessage.length !== 1 || multiPartMessage[0].type !== 'text-block') {
+      addSnackbar({
+        key: 'chat-composer-action-invalid',
+        message: 'Only a single text part is supported for now.',
+        type: 'issue',
+        overrides: {
+          autoHideDuration: 2000,
+        },
+      });
+      return false;
+    }
+    const userText = multiPartMessage[0].text;
+
+    // find conversation
     const conversation = getConversation(conversationId);
-    if (conversation)
-      return await _handleExecute(chatModeId, conversationId, [
-        ...conversation.messages,
-        createDMessage('user', userText),
-      ]);
+    if (!conversation)
+      return false;
+
+    // start execution (async)
+    void _handleExecute(chatModeId, conversationId, [
+      ...conversation.messages,
+      createDMessage('user', userText),
+    ]);
+    return true;
   };
 
   const handleConversationExecuteHistory = async (conversationId: DConversationId, history: DMessage[]) =>
@@ -364,6 +387,7 @@ export function AppChat() {
 
           <ChatMessageList
             conversationId={_conversationId}
+            chatLLMContextTokens={chatLLM?.contextTokens}
             isMessageSelectionMode={isMessageSelectionMode}
             setIsMessageSelectionMode={setIsMessageSelectionMode}
             onConversationBranch={handleConversationBranch}
@@ -402,10 +426,11 @@ export function AppChat() {
     </Box>
 
     <Composer
+      chatLLM={chatLLM}
+      composerTextAreaRef={composerTextAreaRef}
       conversationId={focusedConversationId}
       isDeveloperMode={focusedSystemPurposeId === 'Developer'}
-      composerTextAreaRef={composerTextAreaRef}
-      onNewMessage={handleComposerNewMessage}
+      onAction={handleComposerAction}
       sx={{
         zIndex: 21, // position: 'sticky', bottom: 0,
         backgroundColor: 'background.surface',
@@ -434,7 +459,7 @@ export function AppChat() {
     {/* [confirmation] Reset Conversation */}
     {!!clearConversationId && <ConfirmationModal
       open onClose={() => setClearConversationId(null)} onPositive={handleConfirmedClearConversation}
-      confirmationText={'Are you sure you want to discard all the messages?'} positiveActionText={'Clear conversation'}
+      confirmationText={'Are you sure you want to discard all messages?'} positiveActionText={'Clear conversation'}
     />}
 
     {/* [confirmation] Delete All */}

src/apps/chat/components/ChatMessageList.tsx

@@ -4,8 +4,7 @@ import { shallow } from 'zustand/shallow';
 import { Box, List } from '@mui/joy';
 import { SxProps } from '@mui/joy/styles/types';
 
-import { DiagramConfig } from '~/modules/aifn/digrams/DiagramsModal';
-import { useChatLLM } from '~/modules/llms/store-llms';
+import type { DiagramConfig } from '~/modules/aifn/digrams/DiagramsModal';
 
 import { ShortcutKeyName, useGlobalShortcut } from '~/common/components/useGlobalShortcut';
 import { InlineError } from '~/common/components/InlineError';
@@ -24,6 +23,7 @@ import { useChatShowSystemMessages } from '../store-app-chat';
  */
 export function ChatMessageList(props: {
   conversationId: DConversationId | null,
+  chatLLMContextTokens?: number,
   isMessageSelectionMode: boolean, setIsMessageSelectionMode: (isMessageSelectionMode: boolean) => void,
   onConversationBranch: (conversationId: DConversationId, messageId: string) => void,
   onConversationExecuteHistory: (conversationId: DConversationId, history: DMessage[]) => void,
@@ -50,7 +50,6 @@ export function ChatMessageList(props: {
       setMessages: state.setMessages,
     };
   }, shallow);
-  const { chatLLM } = useChatLLM();
   const { mayWork: isImaginable } = useCapabilityProdia();
   const { mayWork: isSpeakable } = useCapabilityElevenLabs();
 
@@ -188,7 +187,7 @@ export function ChatMessageList(props: {
           <CleanerMessage
             key={'sel-' + message.id}
             message={message}
-            isBottom={idx === 0} remainingTokens={(chatLLM ? chatLLM.contextTokens : 0) - historyTokenCount}
+            isBottom={idx === 0} remainingTokens={(props.chatLLMContextTokens || 0) - historyTokenCount}
             selected={selectedMessages.has(message.id)} onToggleSelected={handleSelectMessage}
           />
 

src/apps/chat/components/composer/ButtonAttachCamera.tsx

@@ -0,0 +1,45 @@
+import * as React from 'react';
+
+import { Box, Button, IconButton, Tooltip } from '@mui/joy';
+import AddAPhotoIcon from '@mui/icons-material/AddAPhoto';
+
+import { CameraCaptureModal } from './CameraCaptureModal';
+
+
+const attachCameraLegend = (isMobile: boolean) =>
+  <Box sx={{ px: 1, py: 0.75, lineHeight: '1.5rem' }}>
+    <b>Attach photo</b><br />
+    {isMobile ? 'Auto-OCR to read text' : 'See the world, on the go'}
+  </Box>;
+
+
+export function ButtonAttachCamera(props: { isMobile?: boolean, onAttachImage: (file: File) => void }) {
+  // state
+  const [open, setOpen] = React.useState(false);
+
+  return <>
+
+    {/* The Button */}
+    {props.isMobile ? (
+      <IconButton variant='plain' color='neutral' onClick={() => setOpen(true)}>
+        <AddAPhotoIcon />
+      </IconButton>
+    ) : (
+      <Tooltip variant='solid' placement='top-start' title={attachCameraLegend(!!props.isMobile)}>
+        <Button fullWidth variant='plain' color='neutral' onClick={() => setOpen(true)} startDecorator={<AddAPhotoIcon />}
+                sx={{ justifyContent: 'flex-start' }}>
+          Camera
+        </Button>
+      </Tooltip>
+    )}
+
+    {/* The actual capture dialog, which will stream the video */}
+    {open && (
+      <CameraCaptureModal
+        onCloseModal={() => setOpen(false)}
+        onAttachImage={props.onAttachImage}
+      />
+    )}
+
+  </>;
+}

src/apps/chat/components/composer/ButtonAttachClipboard.tsx

@@ -0,0 +1,30 @@
+import * as React from 'react';
+
+import { Box, Button, IconButton, Tooltip } from '@mui/joy';
+import ContentPasteGoIcon from '@mui/icons-material/ContentPasteGo';
+
+import { KeyStroke } from '~/common/components/KeyStroke';
+
+
+const pasteClipboardLegend =
+  <Box sx={{ px: 1, py: 0.75, lineHeight: '1.5rem' }}>
+    <b>Attach clipboard 📚</b><br />
+    Auto-converts to the best types<br />
+    <KeyStroke combo='Ctrl + Shift + V' sx={{ mt: 1, mb: 0.5 }} />
+  </Box>;
+
+
+export function ButtonAttachClipboard(props: { isMobile?: boolean, onClick: () => void }) {
+  return props.isMobile ? (
+    <IconButton onClick={props.onClick}>
+      <ContentPasteGoIcon />
+    </IconButton>
+  ) : (
+    <Tooltip variant='solid' placement='top-start' title={pasteClipboardLegend}>
+      <Button fullWidth variant='plain' color='neutral' startDecorator={<ContentPasteGoIcon />} onClick={props.onClick}
+              sx={{ justifyContent: 'flex-start' }}>
+        Paste
+      </Button>
+    </Tooltip>
+  );
+}

src/apps/chat/components/composer/ButtonAttachFile.tsx

@@ -0,0 +1,27 @@
+import * as React from 'react';
+
+import { Box, Button, IconButton, Tooltip } from '@mui/joy';
+import AttachFileOutlinedIcon from '@mui/icons-material/AttachFileOutlined';
+
+
+const attachFileLegend =
+  <Box sx={{ px: 1, py: 0.75, lineHeight: '1.5rem' }}>
+    <b>Attach files</b><br />
+    Drag & drop in chat for faster loads ⚡
+  </Box>;
+
+
+export function ButtonAttachFile(props: { isMobile?: boolean, onAttachFilePicker: () => void }) {
+  return props.isMobile ? (
+    <IconButton onClick={props.onAttachFilePicker}>
+      <AttachFileOutlinedIcon />
+    </IconButton>
+  ) : (
+    <Tooltip variant='solid' placement='top-start' title={attachFileLegend}>
+      <Button fullWidth variant='plain' color='neutral' onClick={props.onAttachFilePicker} startDecorator={<AttachFileOutlinedIcon />}
+              sx={{ justifyContent: 'flex-start' }}>
+        File
+      </Button>
+    </Tooltip>
+  );
+}

src/apps/chat/components/composer/ButtonCall.tsx

@@ -0,0 +1,25 @@
+import * as React from 'react';
+
+import { Box, Button, IconButton, Tooltip } from '@mui/joy';
+import { SxProps } from '@mui/joy/styles/types';
+import CallIcon from '@mui/icons-material/Call';
+
+
+const callConversationLegend =
+  <Box sx={{ px: 1, py: 0.75, lineHeight: '1.5rem' }}>
+    Quick call regarding this chat
+  </Box>;
+
+export function ButtonCall(props: { isMobile?: boolean, disabled?: boolean, onClick: () => void, sx?: SxProps }) {
+  return props.isMobile ? (
+    <IconButton variant='soft' color='primary' disabled={props.disabled} onClick={props.onClick} sx={props.sx}>
+      <CallIcon />
+    </IconButton>
+  ) : (
+    <Tooltip variant='solid' arrow placement='right' title={callConversationLegend}>
+      <Button variant='soft' color='primary' disabled={props.disabled} onClick={props.onClick} endDecorator={<CallIcon />} sx={props.sx}>
+        Call
+      </Button>
+    </Tooltip>
+  );
+}