diff --git a/README.md b/README.md index 0a04140..d5371af 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,11 @@ Autoshow automates the processing of audio and video content from various source The Autoshow workflow includes the following steps: -1. The user provides input (video URL, playlist, RSS feed, or local file). -2. The system downloads the audio (if necessary). +1. The user provides a content input (video URL, playlist, RSS feed, or local file) and front matter is created based on the content's metadata. +2. The audio is downloaded (if necessary). 3. Transcription is performed using the selected service. 4. A customizable prompt is inserted containing instructions for the contents of the show notes. 5. The transcript is processed by the chosen LLM to generate show notes based on the selected prompts. -6. Results are saved in markdown format with front matter. ### Key Features diff --git a/docs/examples.md b/docs/examples.md index 18b0d3b..beeba2b 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -10,10 +10,13 @@ - [Process All Videos from a YouTube Channel](#process-all-videos-from-a-youtube-channel) - [Process Podcast RSS Feed](#process-podcast-rss-feed) - [Transcription Options](#transcription-options) + - [Get Transcription Cost](#get-transcription-cost) - [Whisper](#whisper) - [Deepgram](#deepgram) - [Assembly](#assembly) - [Language Model (LLM) Options](#language-model-llm-options) + - [Run Only LLM Process Step](#run-only-llm-process-step) + - [Get LLM Cost](#get-llm-cost) - [Ollama Local Models](#ollama-local-models) - [OpenAI ChatGPT Models](#openai-chatgpt-models) - [Anthropic Claude Models](#anthropic-claude-models) @@ -260,6 +263,13 @@ npm run as -- \ ## Transcription Options +### Get Transcription Cost + +```bash +npm run as -- --transcriptCost "content/audio.mp3" --deepgram +npm run as -- --transcriptCost "content/audio.mp3" --assembly +``` + ### Whisper If neither the `--deepgram` or `--assembly` option is included for transcription, `autoshow` will default to running the largest Whisper.cpp model. To configure the size of the Whisper model, use the `--model` option and select one of the following: @@ -359,6 +369,19 @@ For each model available for each provider, I have collected the following detai - Cost of input and output tokens per million tokens. - Some model providers also offer a Batch API with input/output tokens at half the price. +### Run Only LLM Process Step + +```bash +npm run as -- --runLLM "content/audio-prompt.md" --chatgpt +``` + +### Get LLM Cost + +```bash +npm run as -- --llmCost "content/audio-prompt.md" --chatgpt +npm run as -- --llmCost "content/audio-prompt.md" --claude +``` + ### Ollama Local Models ```bash diff --git a/package-lock.json b/package-lock.json index e0194dd..fb4e7d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,9 +48,9 @@ } }, "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { - "version": "18.19.71", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.71.tgz", - "integrity": "sha512-evXpcgtZm8FY4jqBSN8+DmOTcVkkvTmAayeo4Wf3m1xAruyVGzGuDh/Fb/WWX2yLItUiho42ozyJjB0dw//Tkw==", + "version": "18.19.74", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.74.tgz", + "integrity": "sha512-HMwEkkifei3L605gFdV+/UwtpxP6JSzM+xFk2Ia6DNFSwSVBRh9qp5Tgf4lNFOMfPVuU0WnkcWpXZpgn5ufO4A==", "license": "MIT", "dependencies": { "undici-types": "~5.26.4" @@ -216,48 +216,48 @@ } }, "node_modules/@aws-sdk/client-cognito-identity": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.730.0.tgz", - "integrity": "sha512-iJt2pL6RqWg7R3pja1WfcC2+oTjwaKFYndNE9oUQqyc6RN24XWUtGy9JnWqTUOy8jYzaP2eoF00fGeasSBX+Dw==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.734.0.tgz", + "integrity": "sha512-qfieHPeLga8MXH1PabQtb9+5etkViUcI4t33dndLHNn46vMhzfTztzkNtjUgN3fg4SrOjnxZd33pR9lRquMRvA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.730.0", - "@aws-sdk/credential-provider-node": "3.730.0", - "@aws-sdk/middleware-host-header": "3.723.0", - "@aws-sdk/middleware-logger": "3.723.0", - "@aws-sdk/middleware-recursion-detection": "3.723.0", - "@aws-sdk/middleware-user-agent": "3.730.0", - "@aws-sdk/region-config-resolver": "3.723.0", - "@aws-sdk/types": "3.723.0", - "@aws-sdk/util-endpoints": "3.730.0", - "@aws-sdk/util-user-agent-browser": "3.723.0", - "@aws-sdk/util-user-agent-node": "3.730.0", - "@smithy/config-resolver": "^4.0.0", - "@smithy/core": "^3.0.0", - "@smithy/fetch-http-handler": "^5.0.0", - "@smithy/hash-node": "^4.0.0", - "@smithy/invalid-dependency": "^4.0.0", - "@smithy/middleware-content-length": "^4.0.0", - "@smithy/middleware-endpoint": "^4.0.0", - "@smithy/middleware-retry": "^4.0.0", - "@smithy/middleware-serde": "^4.0.0", - "@smithy/middleware-stack": "^4.0.0", - "@smithy/node-config-provider": "^4.0.0", - "@smithy/node-http-handler": "^4.0.0", - "@smithy/protocol-http": "^5.0.0", - "@smithy/smithy-client": "^4.0.0", - "@smithy/types": "^4.0.0", - "@smithy/url-parser": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/credential-provider-node": "3.734.0", + "@aws-sdk/middleware-host-header": "3.734.0", + "@aws-sdk/middleware-logger": "3.734.0", + "@aws-sdk/middleware-recursion-detection": "3.734.0", + "@aws-sdk/middleware-user-agent": "3.734.0", + "@aws-sdk/region-config-resolver": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@aws-sdk/util-endpoints": "3.734.0", + "@aws-sdk/util-user-agent-browser": "3.734.0", + "@aws-sdk/util-user-agent-node": "3.734.0", + "@smithy/config-resolver": "^4.0.1", + "@smithy/core": "^3.1.1", + "@smithy/fetch-http-handler": "^5.0.1", + "@smithy/hash-node": "^4.0.1", + "@smithy/invalid-dependency": "^4.0.1", + "@smithy/middleware-content-length": "^4.0.1", + "@smithy/middleware-endpoint": "^4.0.2", + "@smithy/middleware-retry": "^4.0.3", + "@smithy/middleware-serde": "^4.0.1", + "@smithy/middleware-stack": "^4.0.1", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/node-http-handler": "^4.0.2", + "@smithy/protocol-http": "^5.0.1", + "@smithy/smithy-client": "^4.1.2", + "@smithy/types": "^4.1.0", + "@smithy/url-parser": "^4.0.1", "@smithy/util-base64": "^4.0.0", "@smithy/util-body-length-browser": "^4.0.0", "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.0", - "@smithy/util-defaults-mode-node": "^4.0.0", - "@smithy/util-endpoints": "^3.0.0", - "@smithy/util-middleware": "^4.0.0", - "@smithy/util-retry": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.3", + "@smithy/util-defaults-mode-node": "^4.0.3", + "@smithy/util-endpoints": "^3.0.1", + "@smithy/util-middleware": "^4.0.1", + "@smithy/util-retry": "^4.0.1", "@smithy/util-utf8": "^4.0.0", "tslib": "^2.6.2" }, @@ -266,50 +266,50 @@ } }, "node_modules/@aws-sdk/client-sagemaker": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker/-/client-sagemaker-3.730.0.tgz", - "integrity": "sha512-pWMqNIplv5WSybONuQ/8CV8vguIW9X4wqPj49lRy0XSLS5XdiXAFVyegOh1Vk+vMWJIfyUgr3b/SoePCgVyuKA==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker/-/client-sagemaker-3.734.0.tgz", + "integrity": "sha512-K9rUtAT3haVxFflCi/E1XMZHG9YK0txVeuoInDYlS3LQSd8zroIvxZoWrCMEvN1QWulC9qVCJZhPOCnPuo8LLA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.730.0", - "@aws-sdk/credential-provider-node": "3.730.0", - "@aws-sdk/middleware-host-header": "3.723.0", - "@aws-sdk/middleware-logger": "3.723.0", - "@aws-sdk/middleware-recursion-detection": "3.723.0", - "@aws-sdk/middleware-user-agent": "3.730.0", - "@aws-sdk/region-config-resolver": "3.723.0", - "@aws-sdk/types": "3.723.0", - "@aws-sdk/util-endpoints": "3.730.0", - "@aws-sdk/util-user-agent-browser": "3.723.0", - "@aws-sdk/util-user-agent-node": "3.730.0", - "@smithy/config-resolver": "^4.0.0", - "@smithy/core": "^3.0.0", - "@smithy/fetch-http-handler": "^5.0.0", - "@smithy/hash-node": "^4.0.0", - "@smithy/invalid-dependency": "^4.0.0", - "@smithy/middleware-content-length": "^4.0.0", - "@smithy/middleware-endpoint": "^4.0.0", - "@smithy/middleware-retry": "^4.0.0", - "@smithy/middleware-serde": "^4.0.0", - "@smithy/middleware-stack": "^4.0.0", - "@smithy/node-config-provider": "^4.0.0", - "@smithy/node-http-handler": "^4.0.0", - "@smithy/protocol-http": "^5.0.0", - "@smithy/smithy-client": "^4.0.0", - "@smithy/types": "^4.0.0", - "@smithy/url-parser": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/credential-provider-node": "3.734.0", + "@aws-sdk/middleware-host-header": "3.734.0", + "@aws-sdk/middleware-logger": "3.734.0", + "@aws-sdk/middleware-recursion-detection": "3.734.0", + "@aws-sdk/middleware-user-agent": "3.734.0", + "@aws-sdk/region-config-resolver": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@aws-sdk/util-endpoints": "3.734.0", + "@aws-sdk/util-user-agent-browser": "3.734.0", + "@aws-sdk/util-user-agent-node": "3.734.0", + "@smithy/config-resolver": "^4.0.1", + "@smithy/core": "^3.1.1", + "@smithy/fetch-http-handler": "^5.0.1", + "@smithy/hash-node": "^4.0.1", + "@smithy/invalid-dependency": "^4.0.1", + "@smithy/middleware-content-length": "^4.0.1", + "@smithy/middleware-endpoint": "^4.0.2", + "@smithy/middleware-retry": "^4.0.3", + "@smithy/middleware-serde": "^4.0.1", + "@smithy/middleware-stack": "^4.0.1", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/node-http-handler": "^4.0.2", + "@smithy/protocol-http": "^5.0.1", + "@smithy/smithy-client": "^4.1.2", + "@smithy/types": "^4.1.0", + "@smithy/url-parser": "^4.0.1", "@smithy/util-base64": "^4.0.0", "@smithy/util-body-length-browser": "^4.0.0", "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.0", - "@smithy/util-defaults-mode-node": "^4.0.0", - "@smithy/util-endpoints": "^3.0.0", - "@smithy/util-middleware": "^4.0.0", - "@smithy/util-retry": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.3", + "@smithy/util-defaults-mode-node": "^4.0.3", + "@smithy/util-endpoints": "^3.0.1", + "@smithy/util-middleware": "^4.0.1", + "@smithy/util-retry": "^4.0.1", "@smithy/util-utf8": "^4.0.0", - "@smithy/util-waiter": "^4.0.0", + "@smithy/util-waiter": "^4.0.2", "@types/uuid": "^9.0.1", "tslib": "^2.6.2", "uuid": "^9.0.1" @@ -319,47 +319,47 @@ } }, "node_modules/@aws-sdk/client-sso": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.730.0.tgz", - "integrity": "sha512-mI8kqkSuVlZklewEmN7jcbBMyVODBld3MsTjCKSl5ztduuPX69JD7nXLnWWPkw1PX4aGTO24AEoRMGNxntoXUg==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.734.0.tgz", + "integrity": "sha512-oerepp0mut9VlgTwnG5Ds/lb0C0b2/rQ+hL/rF6q+HGKPfGsCuPvFx1GtwGKCXd49ase88/jVgrhcA9OQbz3kg==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.730.0", - "@aws-sdk/middleware-host-header": "3.723.0", - "@aws-sdk/middleware-logger": "3.723.0", - "@aws-sdk/middleware-recursion-detection": "3.723.0", - "@aws-sdk/middleware-user-agent": "3.730.0", - "@aws-sdk/region-config-resolver": "3.723.0", - "@aws-sdk/types": "3.723.0", - "@aws-sdk/util-endpoints": "3.730.0", - "@aws-sdk/util-user-agent-browser": "3.723.0", - "@aws-sdk/util-user-agent-node": "3.730.0", - "@smithy/config-resolver": "^4.0.0", - "@smithy/core": "^3.0.0", - "@smithy/fetch-http-handler": "^5.0.0", - "@smithy/hash-node": "^4.0.0", - "@smithy/invalid-dependency": "^4.0.0", - "@smithy/middleware-content-length": "^4.0.0", - "@smithy/middleware-endpoint": "^4.0.0", - "@smithy/middleware-retry": "^4.0.0", - "@smithy/middleware-serde": "^4.0.0", - "@smithy/middleware-stack": "^4.0.0", - "@smithy/node-config-provider": "^4.0.0", - "@smithy/node-http-handler": "^4.0.0", - "@smithy/protocol-http": "^5.0.0", - "@smithy/smithy-client": "^4.0.0", - "@smithy/types": "^4.0.0", - "@smithy/url-parser": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/middleware-host-header": "3.734.0", + "@aws-sdk/middleware-logger": "3.734.0", + "@aws-sdk/middleware-recursion-detection": "3.734.0", + "@aws-sdk/middleware-user-agent": "3.734.0", + "@aws-sdk/region-config-resolver": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@aws-sdk/util-endpoints": "3.734.0", + "@aws-sdk/util-user-agent-browser": "3.734.0", + "@aws-sdk/util-user-agent-node": "3.734.0", + "@smithy/config-resolver": "^4.0.1", + "@smithy/core": "^3.1.1", + "@smithy/fetch-http-handler": "^5.0.1", + "@smithy/hash-node": "^4.0.1", + "@smithy/invalid-dependency": "^4.0.1", + "@smithy/middleware-content-length": "^4.0.1", + "@smithy/middleware-endpoint": "^4.0.2", + "@smithy/middleware-retry": "^4.0.3", + "@smithy/middleware-serde": "^4.0.1", + "@smithy/middleware-stack": "^4.0.1", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/node-http-handler": "^4.0.2", + "@smithy/protocol-http": "^5.0.1", + "@smithy/smithy-client": "^4.1.2", + "@smithy/types": "^4.1.0", + "@smithy/url-parser": "^4.0.1", "@smithy/util-base64": "^4.0.0", "@smithy/util-body-length-browser": "^4.0.0", "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.0", - "@smithy/util-defaults-mode-node": "^4.0.0", - "@smithy/util-endpoints": "^3.0.0", - "@smithy/util-middleware": "^4.0.0", - "@smithy/util-retry": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.3", + "@smithy/util-defaults-mode-node": "^4.0.3", + "@smithy/util-endpoints": "^3.0.1", + "@smithy/util-middleware": "^4.0.1", + "@smithy/util-retry": "^4.0.1", "@smithy/util-utf8": "^4.0.0", "tslib": "^2.6.2" }, @@ -368,20 +368,20 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.730.0.tgz", - "integrity": "sha512-jonKyR+2GcqbZj2WDICZS0c633keLc9qwXnePu83DfAoFXMMIMyoR/7FOGf8F3OrIdGh8KzE9VvST+nZCK9EJA==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.734.0.tgz", + "integrity": "sha512-SxnDqf3vobdm50OLyAKfqZetv6zzwnSqwIwd3jrbopxxHKqNIM/I0xcYjD6Tn+mPig+u7iRKb9q3QnEooFTlmg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "3.723.0", - "@smithy/core": "^3.0.0", - "@smithy/node-config-provider": "^4.0.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/protocol-http": "^5.0.0", - "@smithy/signature-v4": "^5.0.0", - "@smithy/smithy-client": "^4.0.0", - "@smithy/types": "^4.0.0", - "@smithy/util-middleware": "^4.0.0", + "@aws-sdk/types": "3.734.0", + "@smithy/core": "^3.1.1", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/property-provider": "^4.0.1", + "@smithy/protocol-http": "^5.0.1", + "@smithy/signature-v4": "^5.0.1", + "@smithy/smithy-client": "^4.1.2", + "@smithy/types": "^4.1.0", + "@smithy/util-middleware": "^4.0.1", "fast-xml-parser": "4.4.1", "tslib": "^2.6.2" }, @@ -412,15 +412,15 @@ } }, "node_modules/@aws-sdk/credential-provider-cognito-identity": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.730.0.tgz", - "integrity": "sha512-Ynp67VkpaaFubqPrqGxLbg5XuS+QTjR7JVhZvjNO6Su4tQVKBFSfQpDIXTyggD9UVixXy4NB9cqg30uvebDeiw==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.734.0.tgz", + "integrity": "sha512-9/5SZsg7aZVssWFPWedRv9UNFMI3Vjf83DqVQUCzsSgpIQNtqlxC30WeFXtC/rP5ulOqmF5xHs9zv3bcETAzsg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-cognito-identity": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/client-cognito-identity": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/property-provider": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -428,15 +428,15 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.730.0.tgz", - "integrity": "sha512-fFXgo3jBXLWqu8I07Hd96mS7RjrtpDgm3bZShm0F3lKtqDQF+hObFWq9A013SOE+RjMLVfbABhToXAYct3FcBw==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.734.0.tgz", + "integrity": "sha512-gtRkzYTGafnm1FPpiNO8VBmJrYMoxhDlGPYDVcijzx3DlF8dhWnowuSBCxLSi+MJMx5hvwrX2A+e/q0QAeHqmw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/property-provider": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -444,20 +444,20 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.730.0.tgz", - "integrity": "sha512-1aF3elbCzpVhWLAuV63iFElfLOqLGGTp4fkf2VAFIDO3hjshpXUQssTgIWiBwwtJYJdOSxaFrCU7u8frjr/5aQ==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.734.0.tgz", + "integrity": "sha512-JFSL6xhONsq+hKM8xroIPhM5/FOhiQ1cov0lZxhzZWj6Ai3UAjucy3zyIFDr9MgP1KfCYNdvyaUq9/o+HWvEDg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/fetch-http-handler": "^5.0.0", - "@smithy/node-http-handler": "^4.0.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/protocol-http": "^5.0.0", - "@smithy/smithy-client": "^4.0.0", - "@smithy/types": "^4.0.0", - "@smithy/util-stream": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/fetch-http-handler": "^5.0.1", + "@smithy/node-http-handler": "^4.0.2", + "@smithy/property-provider": "^4.0.1", + "@smithy/protocol-http": "^5.0.1", + "@smithy/smithy-client": "^4.1.2", + "@smithy/types": "^4.1.0", + "@smithy/util-stream": "^4.0.2", "tslib": "^2.6.2" }, "engines": { @@ -465,23 +465,23 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.730.0.tgz", - "integrity": "sha512-zwsxkBuQuPp06o45ATAnznHzj3+ibop/EaTytNzSv0O87Q59K/jnS/bdtv1n6bhe99XCieRNTihvtS7YklzK7A==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.734.0.tgz", + "integrity": "sha512-HEyaM/hWI7dNmb4NhdlcDLcgJvrilk8G4DQX6qz0i4pBZGC2l4iffuqP8K6ZQjUfz5/6894PzeFuhTORAMd+cg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "3.730.0", - "@aws-sdk/credential-provider-env": "3.730.0", - "@aws-sdk/credential-provider-http": "3.730.0", - "@aws-sdk/credential-provider-process": "3.730.0", - "@aws-sdk/credential-provider-sso": "3.730.0", - "@aws-sdk/credential-provider-web-identity": "3.730.0", - "@aws-sdk/nested-clients": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/credential-provider-imds": "^4.0.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/shared-ini-file-loader": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/credential-provider-env": "3.734.0", + "@aws-sdk/credential-provider-http": "3.734.0", + "@aws-sdk/credential-provider-process": "3.734.0", + "@aws-sdk/credential-provider-sso": "3.734.0", + "@aws-sdk/credential-provider-web-identity": "3.734.0", + "@aws-sdk/nested-clients": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/credential-provider-imds": "^4.0.1", + "@smithy/property-provider": "^4.0.1", + "@smithy/shared-ini-file-loader": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -489,22 +489,22 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.730.0.tgz", - "integrity": "sha512-ztRjh1edY7ut2wwrj1XqHtqPY/NXEYIk5fYf04KKsp8zBi81ScVqP7C+Cst6PFKixjgLSG6RsqMx9GSAalVv0Q==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.734.0.tgz", + "integrity": "sha512-9NOSNbkPVb91JwaXOhyfahkzAwWdMsbWHL6fh5/PHlXYpsDjfIfT23I++toepNF2nODAJNLnOEHGYIxgNgf6jQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "3.730.0", - "@aws-sdk/credential-provider-http": "3.730.0", - "@aws-sdk/credential-provider-ini": "3.730.0", - "@aws-sdk/credential-provider-process": "3.730.0", - "@aws-sdk/credential-provider-sso": "3.730.0", - "@aws-sdk/credential-provider-web-identity": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/credential-provider-imds": "^4.0.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/shared-ini-file-loader": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/credential-provider-env": "3.734.0", + "@aws-sdk/credential-provider-http": "3.734.0", + "@aws-sdk/credential-provider-ini": "3.734.0", + "@aws-sdk/credential-provider-process": "3.734.0", + "@aws-sdk/credential-provider-sso": "3.734.0", + "@aws-sdk/credential-provider-web-identity": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/credential-provider-imds": "^4.0.1", + "@smithy/property-provider": "^4.0.1", + "@smithy/shared-ini-file-loader": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -512,16 +512,16 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.730.0.tgz", - "integrity": "sha512-cNKUQ81eptfZN8MlSqwUq3+5ln8u/PcY57UmLZ+npxUHanqO1akpgcpNsLpmsIkoXGbtSQrLuDUgH86lS/SWOw==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.734.0.tgz", + "integrity": "sha512-zvjsUo+bkYn2vjT+EtLWu3eD6me+uun+Hws1IyWej/fKFAqiBPwyeyCgU7qjkiPQSXqk1U9+/HG9IQ6Iiz+eBw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/shared-ini-file-loader": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/property-provider": "^4.0.1", + "@smithy/shared-ini-file-loader": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -529,18 +529,18 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.730.0.tgz", - "integrity": "sha512-SdI2xrTbquJLMxUh5LpSwB8zfiKq3/jso53xWRgrVfeDlrSzZuyV6QghaMs3KEEjcNzwEnTfSIjGQyRXG9VrEw==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.734.0.tgz", + "integrity": "sha512-cCwwcgUBJOsV/ddyh1OGb4gKYWEaTeTsqaAK19hiNINfYV/DO9r4RMlnWAo84sSBfJuj9shUNsxzyoe6K7R92Q==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-sso": "3.730.0", - "@aws-sdk/core": "3.730.0", - "@aws-sdk/token-providers": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/shared-ini-file-loader": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/client-sso": "3.734.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/token-providers": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/property-provider": "^4.0.1", + "@smithy/shared-ini-file-loader": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -548,16 +548,16 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.730.0.tgz", - "integrity": "sha512-l5vdPmvF/d890pbvv5g1GZrdjaSQkyPH/Bc8dO/ZqkWxkIP8JNgl48S2zgf4DkP3ik9K2axWO828L5RsMDQzdA==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.734.0.tgz", + "integrity": "sha512-t4OSOerc+ppK541/Iyn1AS40+2vT/qE+MFMotFkhCgCJbApeRF2ozEdnDN6tGmnl4ybcUuxnp9JWLjwDVlR/4g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "3.730.0", - "@aws-sdk/nested-clients": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/nested-clients": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/property-provider": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -565,26 +565,27 @@ } }, "node_modules/@aws-sdk/credential-providers": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.730.0.tgz", - "integrity": "sha512-Z25yfmHOehgIDVyY8h7GmAEbodHD2iLgNmrBBkkJXCE6d4GwDet3Qeyw4bQPPyuycBtYOUiz5Oco03+YGOEhYA==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.734.0.tgz", + "integrity": "sha512-3q76ngVxwX/kSRA0bjH7hUkIOVf/38aACmYpbwwr7jyRU3Cpbsj57W9YtRd7zS9/A4Jt6fYx7VFEA52ajyoGAQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-cognito-identity": "3.730.0", - "@aws-sdk/core": "3.730.0", - "@aws-sdk/credential-provider-cognito-identity": "3.730.0", - "@aws-sdk/credential-provider-env": "3.730.0", - "@aws-sdk/credential-provider-http": "3.730.0", - "@aws-sdk/credential-provider-ini": "3.730.0", - "@aws-sdk/credential-provider-node": "3.730.0", - "@aws-sdk/credential-provider-process": "3.730.0", - "@aws-sdk/credential-provider-sso": "3.730.0", - "@aws-sdk/credential-provider-web-identity": "3.730.0", - "@aws-sdk/nested-clients": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/credential-provider-imds": "^4.0.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/client-cognito-identity": "3.734.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/credential-provider-cognito-identity": "3.734.0", + "@aws-sdk/credential-provider-env": "3.734.0", + "@aws-sdk/credential-provider-http": "3.734.0", + "@aws-sdk/credential-provider-ini": "3.734.0", + "@aws-sdk/credential-provider-node": "3.734.0", + "@aws-sdk/credential-provider-process": "3.734.0", + "@aws-sdk/credential-provider-sso": "3.734.0", + "@aws-sdk/credential-provider-web-identity": "3.734.0", + "@aws-sdk/nested-clients": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/core": "^3.1.1", + "@smithy/credential-provider-imds": "^4.0.1", + "@smithy/property-provider": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -592,14 +593,14 @@ } }, "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.723.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.723.0.tgz", - "integrity": "sha512-LLVzLvk299pd7v4jN9yOSaWDZDfH0SnBPb6q+FDPaOCMGBY8kuwQso7e/ozIKSmZHRMGO3IZrflasHM+rI+2YQ==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.734.0.tgz", + "integrity": "sha512-LW7RRgSOHHBzWZnigNsDIzu3AiwtjeI2X66v+Wn1P1u+eXssy1+up4ZY/h+t2sU4LU36UvEf+jrZti9c6vRnFw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "3.723.0", - "@smithy/protocol-http": "^5.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/types": "3.734.0", + "@smithy/protocol-http": "^5.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -607,13 +608,13 @@ } }, "node_modules/@aws-sdk/middleware-logger": { - "version": "3.723.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.723.0.tgz", - "integrity": "sha512-chASQfDG5NJ8s5smydOEnNK7N0gDMyuPbx7dYYcm1t/PKtnVfvWF+DHCTrRC2Ej76gLJVCVizlAJKM8v8Kg3cg==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.734.0.tgz", + "integrity": "sha512-mUMFITpJUW3LcKvFok176eI5zXAUomVtahb9IQBwLzkqFYOrMJvWAvoV4yuxrJ8TlQBG8gyEnkb9SnhZvjg67w==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "3.723.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/types": "3.734.0", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -621,14 +622,14 @@ } }, "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.723.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.723.0.tgz", - "integrity": "sha512-7usZMtoynT9/jxL/rkuDOFQ0C2mhXl4yCm67Rg7GNTstl67u7w5WN1aIRImMeztaKlw8ExjoTyo6WTs1Kceh7A==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.734.0.tgz", + "integrity": "sha512-CUat2d9ITsFc2XsmeiRQO96iWpxSKYFjxvj27Hc7vo87YUHRnfMfnc8jw1EpxEwMcvBD7LsRa6vDNky6AjcrFA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "3.723.0", - "@smithy/protocol-http": "^5.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/types": "3.734.0", + "@smithy/protocol-http": "^5.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -636,17 +637,17 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.730.0.tgz", - "integrity": "sha512-aPMZvNmf2a42B41au3bA3ODU4HfHka2nYT/SAIhhVXH1ENYfAmZo7FraFPxetKepFMCtL7j4QE6/LDucK6liIw==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.734.0.tgz", + "integrity": "sha512-MFVzLWRkfFz02GqGPjqSOteLe5kPfElUrXZft1eElnqulqs6RJfVSpOV7mO90gu293tNAeggMWAVSGRPKIYVMg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@aws-sdk/util-endpoints": "3.730.0", - "@smithy/core": "^3.0.0", - "@smithy/protocol-http": "^5.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@aws-sdk/util-endpoints": "3.734.0", + "@smithy/core": "^3.1.1", + "@smithy/protocol-http": "^5.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -654,47 +655,47 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.730.0.tgz", - "integrity": "sha512-vilIgf1/7kre8DdE5zAQkDOwHFb/TahMn/6j2RZwFLlK7cDk91r19deSiVYnKQkupDMtOfNceNqnorM4I3PDzw==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.734.0.tgz", + "integrity": "sha512-iph2XUy8UzIfdJFWo1r0Zng9uWj3253yvW9gljhtu+y/LNmNvSnJxQk1f3D2BC5WmcoPZqTS3UsycT3mLPSzWA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "3.730.0", - "@aws-sdk/middleware-host-header": "3.723.0", - "@aws-sdk/middleware-logger": "3.723.0", - "@aws-sdk/middleware-recursion-detection": "3.723.0", - "@aws-sdk/middleware-user-agent": "3.730.0", - "@aws-sdk/region-config-resolver": "3.723.0", - "@aws-sdk/types": "3.723.0", - "@aws-sdk/util-endpoints": "3.730.0", - "@aws-sdk/util-user-agent-browser": "3.723.0", - "@aws-sdk/util-user-agent-node": "3.730.0", - "@smithy/config-resolver": "^4.0.0", - "@smithy/core": "^3.0.0", - "@smithy/fetch-http-handler": "^5.0.0", - "@smithy/hash-node": "^4.0.0", - "@smithy/invalid-dependency": "^4.0.0", - "@smithy/middleware-content-length": "^4.0.0", - "@smithy/middleware-endpoint": "^4.0.0", - "@smithy/middleware-retry": "^4.0.0", - "@smithy/middleware-serde": "^4.0.0", - "@smithy/middleware-stack": "^4.0.0", - "@smithy/node-config-provider": "^4.0.0", - "@smithy/node-http-handler": "^4.0.0", - "@smithy/protocol-http": "^5.0.0", - "@smithy/smithy-client": "^4.0.0", - "@smithy/types": "^4.0.0", - "@smithy/url-parser": "^4.0.0", + "@aws-sdk/core": "3.734.0", + "@aws-sdk/middleware-host-header": "3.734.0", + "@aws-sdk/middleware-logger": "3.734.0", + "@aws-sdk/middleware-recursion-detection": "3.734.0", + "@aws-sdk/middleware-user-agent": "3.734.0", + "@aws-sdk/region-config-resolver": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@aws-sdk/util-endpoints": "3.734.0", + "@aws-sdk/util-user-agent-browser": "3.734.0", + "@aws-sdk/util-user-agent-node": "3.734.0", + "@smithy/config-resolver": "^4.0.1", + "@smithy/core": "^3.1.1", + "@smithy/fetch-http-handler": "^5.0.1", + "@smithy/hash-node": "^4.0.1", + "@smithy/invalid-dependency": "^4.0.1", + "@smithy/middleware-content-length": "^4.0.1", + "@smithy/middleware-endpoint": "^4.0.2", + "@smithy/middleware-retry": "^4.0.3", + "@smithy/middleware-serde": "^4.0.1", + "@smithy/middleware-stack": "^4.0.1", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/node-http-handler": "^4.0.2", + "@smithy/protocol-http": "^5.0.1", + "@smithy/smithy-client": "^4.1.2", + "@smithy/types": "^4.1.0", + "@smithy/url-parser": "^4.0.1", "@smithy/util-base64": "^4.0.0", "@smithy/util-body-length-browser": "^4.0.0", "@smithy/util-body-length-node": "^4.0.0", - "@smithy/util-defaults-mode-browser": "^4.0.0", - "@smithy/util-defaults-mode-node": "^4.0.0", - "@smithy/util-endpoints": "^3.0.0", - "@smithy/util-middleware": "^4.0.0", - "@smithy/util-retry": "^4.0.0", + "@smithy/util-defaults-mode-browser": "^4.0.3", + "@smithy/util-defaults-mode-node": "^4.0.3", + "@smithy/util-endpoints": "^3.0.1", + "@smithy/util-middleware": "^4.0.1", + "@smithy/util-retry": "^4.0.1", "@smithy/util-utf8": "^4.0.0", "tslib": "^2.6.2" }, @@ -742,16 +743,16 @@ } }, "node_modules/@aws-sdk/region-config-resolver": { - "version": "3.723.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.723.0.tgz", - "integrity": "sha512-tGF/Cvch3uQjZIj34LY2mg8M2Dr4kYG8VU8Yd0dFnB1ybOEOveIK/9ypUo9ycZpB9oO6q01KRe5ijBaxNueUQg==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.734.0.tgz", + "integrity": "sha512-Lvj1kPRC5IuJBr9DyJ9T9/plkh+EfKLy+12s/mykOy1JaKHDpvj+XGy2YO6YgYVOb8JFtaqloid+5COtje4JTQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "3.723.0", - "@smithy/node-config-provider": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/types": "3.734.0", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/types": "^4.1.0", "@smithy/util-config-provider": "^4.0.0", - "@smithy/util-middleware": "^4.0.0", + "@smithy/util-middleware": "^4.0.1", "tslib": "^2.6.2" }, "engines": { @@ -878,16 +879,16 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.730.0.tgz", - "integrity": "sha512-BSPssGj54B/AABWXARIPOT/1ybFahM1ldlfmXy9gRmZi/afe9geWJGlFYCCt3PmqR+1Ny5XIjSfue+kMd//drQ==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.734.0.tgz", + "integrity": "sha512-2U6yWKrjWjZO8Y5SHQxkFvMVWHQWbS0ufqfAIBROqmIZNubOL7jXCiVdEFekz6MZ9LF2tvYGnOW4jX8OKDGfIw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/nested-clients": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/property-provider": "^4.0.0", - "@smithy/shared-ini-file-loader": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/nested-clients": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/property-provider": "^4.0.1", + "@smithy/shared-ini-file-loader": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -895,12 +896,12 @@ } }, "node_modules/@aws-sdk/types": { - "version": "3.723.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.723.0.tgz", - "integrity": "sha512-LmK3kwiMZG1y5g3LGihT9mNkeNOmwEyPk6HGcJqh0wOSV4QpWoKu2epyKE4MLQNUUlz2kOVbVbOrwmI6ZcteuA==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.734.0.tgz", + "integrity": "sha512-o11tSPTT70nAkGV1fN9wm/hAIiLPyWX6SuGf+9JyTp7S/rC2cFWhR26MvA69nplcjNaXVzB0f+QFrLXXjOqCrg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.0.0", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -908,14 +909,14 @@ } }, "node_modules/@aws-sdk/util-endpoints": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.730.0.tgz", - "integrity": "sha512-1KTFuVnk+YtLgWr6TwDiggcDqtPpOY2Cszt3r2lkXfaEAX6kHyOZi1vdvxXjPU5LsOBJem8HZ7KlkmrEi+xowg==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.734.0.tgz", + "integrity": "sha512-w2+/E88NUbqql6uCVAsmMxDQKu7vsKV0KqhlQb0lL+RCq4zy07yXYptVNs13qrnuTfyX7uPXkXrlugvK9R1Ucg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "3.723.0", - "@smithy/types": "^4.0.0", - "@smithy/util-endpoints": "^3.0.0", + "@aws-sdk/types": "3.734.0", + "@smithy/types": "^4.1.0", + "@smithy/util-endpoints": "^3.0.1", "tslib": "^2.6.2" }, "engines": { @@ -935,27 +936,27 @@ } }, "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.723.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.723.0.tgz", - "integrity": "sha512-Wh9I6j2jLhNFq6fmXydIpqD1WyQLyTfSxjW9B+PXSnPyk3jtQW8AKQur7p97rO8LAUzVI0bv8kb3ZzDEVbquIg==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.734.0.tgz", + "integrity": "sha512-xQTCus6Q9LwUuALW+S76OL0jcWtMOVu14q+GoLnWPUM7QeUw963oQcLhF7oq0CtaLLKyl4GOUfcwc773Zmwwng==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "3.723.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/types": "3.734.0", + "@smithy/types": "^4.1.0", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.730.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.730.0.tgz", - "integrity": "sha512-yBvkOAjqsDEl1va4eHNOhnFBk0iCY/DBFNyhvtTMqPF4NO+MITWpFs3J9JtZKzJlQ6x0Yb9TLQ8NhDjEISz5Ug==", + "version": "3.734.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.734.0.tgz", + "integrity": "sha512-c6Iinh+RVQKs6jYUFQ64htOU2HUXFQ3TVx+8Tu3EDF19+9vzWi9UukhIMH9rqyyEXIAkk9XL7avt8y2Uyw2dGA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-user-agent": "3.730.0", - "@aws-sdk/types": "3.723.0", - "@smithy/node-config-provider": "^4.0.0", - "@smithy/types": "^4.0.0", + "@aws-sdk/middleware-user-agent": "3.734.0", + "@aws-sdk/types": "3.734.0", + "@smithy/node-config-provider": "^4.0.1", + "@smithy/types": "^4.1.0", "tslib": "^2.6.2" }, "engines": { @@ -1009,9 +1010,9 @@ } }, "node_modules/@deepgram/sdk/node_modules/@types/node": { - "version": "18.19.71", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.71.tgz", - "integrity": "sha512-evXpcgtZm8FY4jqBSN8+DmOTcVkkvTmAayeo4Wf3m1xAruyVGzGuDh/Fb/WWX2yLItUiho42ozyJjB0dw//Tkw==", + "version": "18.19.74", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.74.tgz", + "integrity": "sha512-HMwEkkifei3L605gFdV+/UwtpxP6JSzM+xFk2Ia6DNFSwSVBRh9qp5Tgf4lNFOMfPVuU0WnkcWpXZpgn5ufO4A==", "license": "MIT", "dependencies": { "undici-types": "~5.26.4" @@ -1562,20 +1563,20 @@ } }, "node_modules/@shikijs/engine-oniguruma": { - "version": "1.27.2", - "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-1.27.2.tgz", - "integrity": "sha512-FZYKD1KN7srvpkz4lbGLOYWlyDU4Rd+2RtuKfABTkafAPOFr+J6umfIwY/TzOQqfNtWjL7SAwPAO0dcOraRLaQ==", + "version": "1.29.1", + "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-1.29.1.tgz", + "integrity": "sha512-gSt2WhLNgEeLstcweQOSp+C+MhOpTsgdNXRqr3zP6M+BUBZ8Md9OU2BYwUYsALBxHza7hwaIWtFHjQ/aOOychw==", "dev": true, "license": "MIT", "dependencies": { - "@shikijs/types": "1.27.2", + "@shikijs/types": "1.29.1", "@shikijs/vscode-textmate": "^10.0.1" } }, "node_modules/@shikijs/types": { - "version": "1.27.2", - "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-1.27.2.tgz", - "integrity": "sha512-DM9OWUyjmdYdnKDpaGB/GEn9XkToyK1tqxuqbmc5PV+5K8WjjwfygL3+cIvbkSw2v1ySwHDgqATq/+98pJ4Kyg==", + "version": "1.29.1", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-1.29.1.tgz", + "integrity": "sha512-aBqAuhYRp5vSir3Pc9+QPu9WESBOjUo03ao0IHLC4TyTioSsp/SkbAZSrIH4ghYYC1T1KTEpRSBa83bas4RnPA==", "dev": true, "license": "MIT", "dependencies": { @@ -2928,9 +2929,9 @@ } }, "node_modules/fast-uri": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.5.tgz", - "integrity": "sha512-5JnBCWpFlMo0a3ciDy/JckMzzv1U9coZrIhedq+HXxxUfDTAiS0LA8OKVao4G9BxmCVck/jtA5r3KAtRWEyD8Q==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", "funding": [ { "type": "github", @@ -3158,9 +3159,9 @@ } }, "node_modules/get-tsconfig": { - "version": "4.8.1", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz", - "integrity": "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==", + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.0.tgz", + "integrity": "sha512-kGzZ3LWWQcGIAmg6iWvXn0ei6WDtV26wzHRMwDSzmAbcXrTEXxHy6IehI6/4eT6VRKyMP1eF1VqwrVUmE/LR7A==", "dev": true, "license": "MIT", "dependencies": { @@ -3463,9 +3464,9 @@ "license": "MIT" }, "node_modules/napi-build-utils": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", - "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", "license": "MIT" }, "node_modules/node-abi": { @@ -3591,9 +3592,9 @@ } }, "node_modules/openai/node_modules/@types/node": { - "version": "18.19.71", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.71.tgz", - "integrity": "sha512-evXpcgtZm8FY4jqBSN8+DmOTcVkkvTmAayeo4Wf3m1xAruyVGzGuDh/Fb/WWX2yLItUiho42ozyJjB0dw//Tkw==", + "version": "18.19.74", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.74.tgz", + "integrity": "sha512-HMwEkkifei3L605gFdV+/UwtpxP6JSzM+xFk2Ia6DNFSwSVBRh9qp5Tgf4lNFOMfPVuU0WnkcWpXZpgn5ufO4A==", "license": "MIT", "dependencies": { "undici-types": "~5.26.4" @@ -3656,9 +3657,9 @@ "license": "MIT" }, "node_modules/prebuild-install": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", - "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==", + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", "license": "MIT", "dependencies": { "detect-libc": "^2.0.0", @@ -3666,7 +3667,7 @@ "github-from-package": "0.0.0", "minimist": "^1.2.3", "mkdirp-classic": "^0.5.3", - "napi-build-utils": "^1.0.1", + "napi-build-utils": "^2.0.0", "node-abi": "^3.3.0", "pump": "^3.0.0", "rc": "^1.2.7", diff --git a/scripts/setup.sh b/scripts/setup.sh index d3175fb..223d276 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -1,86 +1,194 @@ -#!/bin/bash +#!/usr/bin/env bash # scripts/setup.sh -# Function to check if a command exists +# ------------------------------------------------------------------------------ +# A single script to set up your environment on macOS (brew) or Linux (apt). +# Installs yt-dlp, ffmpeg, and ollama if they are missing, plus sets up whisper.cpp. +# ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# 1. OS DETECTION +# ------------------------------------------------------------------------------ +IS_MAC=false +IS_LINUX=false + +case "$OSTYPE" in + darwin*) + IS_MAC=true + ;; + linux*) + IS_LINUX=true + ;; + *) + echo "Unsupported OS: $OSTYPE" + echo "Please install dependencies manually." + exit 1 + ;; +esac + +# ------------------------------------------------------------------------------ +# 2. HELPER FUNCTIONS +# ------------------------------------------------------------------------------ + +# Check if a command is available command_exists() { - command -v "$1" >/dev/null 2>&1 + command -v "$1" &>/dev/null } -# Check if .env file exists -if [ -f ".env" ]; then - echo ".env file already exists. Skipping copy of .env.example." -else - echo ".env file does not exist. Copying .env.example to .env." - cp .env.example .env -fi +# Ensure Homebrew on macOS +ensure_homebrew() { + if ! command_exists brew; then + echo "Homebrew is not installed on your system." + echo "Please install Homebrew from https://brew.sh/ then rerun this script." + exit 1 + fi +} -# Check if yt-dlp is installed, if not, provide installation instructions -if ! command_exists yt-dlp; then - echo "yt-dlp could not be found, refer to installation instructions here:" - echo "https://github.com/yt-dlp/yt-dlp/wiki/Installation" -else - echo "yt-dlp is already installed." -fi +# Ensure apt on Linux +ensure_apt() { + if ! command_exists apt-get; then + echo "This script requires apt-get, but it was not found." + echo "Please install dependencies manually, or add logic for your package manager." + exit 1 + fi +} + +# Install package if missing (macOS) +install_if_missing_brew() { + local pkg=$1 + if ! command_exists "$pkg"; then + echo "$pkg not found. Installing with Homebrew..." + brew install "$pkg" + else + echo "$pkg is already installed." + fi +} + +# Install package if missing (Linux/apt) +install_if_missing_apt() { + local pkg=$1 + if ! command_exists "$pkg"; then + echo "$pkg not found. Installing with apt-get..." + sudo apt-get update -y + sudo apt-get install -y "$pkg" + else + echo "$pkg is already installed." + fi +} -# Function to check if Ollama server is running +# Check if Ollama server is running check_ollama_server() { - if curl -s "http://127.0.0.1:11434" &> /dev/null; then - echo "Ollama server is already running." - else - echo "Ollama server is not running. Starting Ollama server..." - ollama serve > ollama.log 2>&1 & - OLLAMA_PID=$! - echo "Ollama server started with PID $OLLAMA_PID" - sleep 5 - fi + if curl -s "http://127.0.0.1:11434" &> /dev/null; then + echo "Ollama server is already running." + else + echo "Ollama server is not running. Starting Ollama server..." + ollama serve > ollama.log 2>&1 & + OLLAMA_PID=$! + echo "Ollama server started with PID $OLLAMA_PID" + # Allow server a few seconds to initialize + sleep 5 + fi } -# Function to check if a model is available, and pull it if not +# Check if a model is available, and pull it if not check_and_pull_model() { - local model=$1 - if ollama list | grep -q "$model"; then - echo "Model $model is already available." - else - echo "Model $model is not available. Pulling the model..." - ollama pull "$model" - fi + local model=$1 + if ollama list | grep -q "$model"; then + echo "Model '$model' is already available." + else + echo "Model '$model' is not available. Pulling the model..." + ollama pull "$model" + fi } -# Check if Ollama is installed -if ! command_exists ollama; then - echo "Ollama is not installed, refer to installation instructions here:" - echo "https://github.com/ollama/ollama" +# ------------------------------------------------------------------------------ +# 3. INSTALL DEPENDENCIES (yt-dlp, ffmpeg, ollama) +# ------------------------------------------------------------------------------ + +# On macOS, make sure Homebrew is installed and install packages +if [ "$IS_MAC" = true ]; then + ensure_homebrew + BREW_PACKAGES=("yt-dlp" "ffmpeg" "ollama") + for pkg in "${BREW_PACKAGES[@]}"; do + install_if_missing_brew "$pkg" + done +fi + +# On Linux, we’ll assume apt and install packages +if [ "$IS_LINUX" = true ]; then + ensure_apt + + # There's no official apt package for "ollama" at the time of writing. + + APT_PACKAGES=("yt-dlp" "ffmpeg") + for pkg in "${APT_PACKAGES[@]}"; do + install_if_missing_apt "$pkg" + done + + # Check if Ollama is installed + if ! command_exists ollama; then + echo "Ollama is not installed. There's no official apt package yet." + echo "Please follow instructions here: https://github.com/jmorganca/ollama" + echo "After installing Ollama, re-run this script." + exit 1 + else + echo "Ollama is already installed." + fi +fi + +# ------------------------------------------------------------------------------ +# 4. SETUP .ENV +# ------------------------------------------------------------------------------ +if [ -f ".env" ]; then + echo "" + echo ".env file already exists. Skipping copy of .env.example." + echo "" else - echo "Ollama is installed." - - # Check if Ollama server is running - check_ollama_server - - # Check and pull required models - check_and_pull_model "llama3.2:1b" && check_and_pull_model "qwen2.5:0.5b" + echo "" + echo ".env file does not exist. Copying .env.example to .env." + echo "" + cp .env.example .env +fi + +# ------------------------------------------------------------------------------ +# 5. OLLAMA SERVER AND MODELS +# ------------------------------------------------------------------------------ +# If Ollama is installed, let's start the server and pull models +if command_exists ollama; then + check_ollama_server +# check_and_pull_model "llama3.2:1b" + check_and_pull_model "qwen2.5:0.5b" fi -# Install npm dependencies -npm i +# ------------------------------------------------------------------------------ +# 6. NPM DEPENDENCIES +# ------------------------------------------------------------------------------ +echo "" +echo "Installing npm dependencies..." +echo "" +npm install -# Check if whisper.cpp directory exists +# ------------------------------------------------------------------------------ +# 7. WHISPER.CPP SETUP +# ------------------------------------------------------------------------------ if [ -d "whisper.cpp" ]; then - echo "whisper.cpp directory already exists. Skipping clone and setup." + echo "whisper.cpp directory already exists. Skipping clone and setup." else - echo "Cloning whisper.cpp repository..." - git clone https://github.com/ggerganov/whisper.cpp.git - - # Download whisper models - echo "Downloading whisper models..." - bash ./whisper.cpp/models/download-ggml-model.sh tiny - bash ./whisper.cpp/models/download-ggml-model.sh base - bash ./whisper.cpp/models/download-ggml-model.sh large-v3-turbo - - # Compile whisper.cpp - echo "Compiling whisper.cpp..." - cmake -B whisper.cpp/build -S whisper.cpp - cmake --build whisper.cpp/build --config Release - rm -rf whisper.cpp/.git + echo "Cloning whisper.cpp repository..." + git clone https://github.com/ggerganov/whisper.cpp.git + + echo "Downloading whisper models..." + bash ./whisper.cpp/models/download-ggml-model.sh tiny + bash ./whisper.cpp/models/download-ggml-model.sh base + bash ./whisper.cpp/models/download-ggml-model.sh large-v3-turbo + + echo "Compiling whisper.cpp..." + cmake -B whisper.cpp/build -S whisper.cpp + cmake --build whisper.cpp/build --config Release + + # Optionally remove .git folder to keep workspace clean + rm -rf whisper.cpp/.git fi +echo "" echo "Setup completed successfully!" \ No newline at end of file diff --git a/src/commander.ts b/src/commander.ts index 6bce4f7..1bfee76 100644 --- a/src/commander.ts +++ b/src/commander.ts @@ -17,17 +17,20 @@ import { Command } from 'commander' import { selectPrompts } from './process-steps/04-select-prompt' import { processAction, validateCLIOptions } from './utils/validate-option' import { l, err, logSeparator } from './utils/logging' -import { envVarsMap } from './utils/globals/llms' +import { envVarsMap, estimateLLMCost } from './utils/llm-utils' +import { estimateTranscriptCost } from './utils/transcription-utils' +import { runLLMFromPromptFile } from './process-steps/05-run-llm' import type { ProcessingOptions } from './utils/types/process' -// Initialize the command-line interface using Commander.js -const program = new Command() - /** * Defines the command-line interface options and descriptions. * Sets up all available commands and their respective flags. */ + +// Initialize the command-line interface using Commander.js +const program = new Command() + program .name('autoshow') .version('0.0.1') @@ -53,6 +56,8 @@ program .option('--deepgram [model]', 'Use Deepgram for transcription with optional model specification') .option('--assembly [model]', 'Use AssemblyAI for transcription with optional model specification') .option('--speakerLabels', 'Use speaker labels for AssemblyAI transcription') + .option('--transcriptCost ', 'Estimate transcription cost for the given file') + .option('--llmCost ', 'Estimate LLM cost for the given prompt and transcript file') // LLM service options .option('--ollama [model]', 'Use Ollama for processing with optional model specification') .option('--chatgpt [model]', 'Use ChatGPT for processing with optional model specification') @@ -69,6 +74,7 @@ program .option('--prompt ', 'Specify prompt sections to include') .option('--printPrompt ', 'Print the prompt sections without processing') .option('--customPrompt ', 'Use a custom prompt from a markdown file') + .option('--runLLM ', 'Run Step 5 with a prompt file') .option('--saveAudio', 'Do not delete intermediary files after processing') // Options to override environment variables from CLI .option('--openaiApiKey ', 'Specify OpenAI API key (overrides .env variable)') @@ -109,6 +115,106 @@ program.action(async (options: ProcessingOptions) => { exit(0) } + // Handle transcript cost estimation + if (options.transcriptCost) { + let transcriptServices: 'deepgram' | 'assembly' | 'whisper' | undefined = undefined + + if (typeof options.deepgram !== 'undefined') { + transcriptServices = 'deepgram' + } else if (typeof options.assembly !== 'undefined') { + transcriptServices = 'assembly' + } else if (typeof options.whisper !== 'undefined') { + transcriptServices = 'whisper' + } + + if (!transcriptServices) { + err('Please specify which transcription service to use (e.g., --deepgram, --assembly, --whisper).') + exit(1) + } + + await estimateTranscriptCost(options, transcriptServices) + exit(0) + } + + // Handle LLM cost estimation + if (options.llmCost) { + let llmService: 'ollama' | 'chatgpt' | 'claude' | 'gemini' | 'cohere' | 'mistral' | 'deepseek' | 'grok' | 'fireworks' | 'together' | 'groq' | undefined = undefined + + if (typeof options.ollama !== 'undefined') { + llmService = 'ollama' + } else if (typeof options.chatgpt !== 'undefined') { + llmService = 'chatgpt' + } else if (typeof options.claude !== 'undefined') { + llmService = 'claude' + } else if (typeof options.gemini !== 'undefined') { + llmService = 'gemini' + } else if (typeof options.cohere !== 'undefined') { + llmService = 'cohere' + } else if (typeof options.mistral !== 'undefined') { + llmService = 'mistral' + } else if (typeof options.deepseek !== 'undefined') { + llmService = 'deepseek' + } else if (typeof options.grok !== 'undefined') { + llmService = 'grok' + } else if (typeof options.fireworks !== 'undefined') { + llmService = 'fireworks' + } else if (typeof options.together !== 'undefined') { + llmService = 'together' + } else if (typeof options.groq !== 'undefined') { + llmService = 'groq' + } + + if (!llmService) { + err('Please specify which LLM service to use (e.g., --chatgpt, --claude, --ollama, etc.).') + exit(1) + } + + await estimateLLMCost(options, llmService) + exit(0) + } + + /** + * Handle running Step 5 (LLM) directly with a prompt file + * + * Example usage: + * npm run as -- --runLLM "content/audio-prompt.md" --chatgpt + */ + if (options.runLLM) { + let llmService: 'ollama' | 'chatgpt' | 'claude' | 'gemini' | 'cohere' | 'mistral' | 'deepseek' | 'grok' | 'fireworks' | 'together' | 'groq' | undefined = undefined + + if (typeof options.ollama !== 'undefined') { + llmService = 'ollama' + } else if (typeof options.chatgpt !== 'undefined') { + llmService = 'chatgpt' + } else if (typeof options.claude !== 'undefined') { + llmService = 'claude' + } else if (typeof options.gemini !== 'undefined') { + llmService = 'gemini' + } else if (typeof options.cohere !== 'undefined') { + llmService = 'cohere' + } else if (typeof options.mistral !== 'undefined') { + llmService = 'mistral' + } else if (typeof options.deepseek !== 'undefined') { + llmService = 'deepseek' + } else if (typeof options.grok !== 'undefined') { + llmService = 'grok' + } else if (typeof options.fireworks !== 'undefined') { + llmService = 'fireworks' + } else if (typeof options.together !== 'undefined') { + llmService = 'together' + } else if (typeof options.groq !== 'undefined') { + llmService = 'groq' + } + + if (!llmService) { + err('Please specify which LLM service to use (e.g., --chatgpt, --claude, --ollama, etc.).') + exit(1) + } + + await runLLMFromPromptFile(options.runLLM, options, llmService) + exit(0) + } + // Validate action, LLM, and transcription inputs const { action, llmServices, transcriptServices } = validateCLIOptions(options) diff --git a/src/llms/chatgpt.ts b/src/llms/chatgpt.ts index 45f697c..5f03861 100644 --- a/src/llms/chatgpt.ts +++ b/src/llms/chatgpt.ts @@ -2,8 +2,9 @@ import { env } from 'node:process' import { OpenAI } from 'openai' -import { GPT_MODELS } from '../utils/globals/llms' -import { err, logLLMCost } from '../utils/logging' +import { GPT_MODELS } from '../utils/llm-utils' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' import type { ChatGPTModelType } from '../utils/types/llms' /** diff --git a/src/llms/claude.ts b/src/llms/claude.ts index 4b9b033..71358c0 100644 --- a/src/llms/claude.ts +++ b/src/llms/claude.ts @@ -2,8 +2,9 @@ import { env } from 'node:process' import { Anthropic } from '@anthropic-ai/sdk' -import { CLAUDE_MODELS } from '../utils/globals/llms' -import { err, logLLMCost } from '../utils/logging' +import { CLAUDE_MODELS } from '../utils/llm-utils' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' import type { ClaudeModelType } from '../utils/types/llms' /** diff --git a/src/llms/cohere.ts b/src/llms/cohere.ts index bc42ee4..e091f87 100644 --- a/src/llms/cohere.ts +++ b/src/llms/cohere.ts @@ -2,8 +2,9 @@ import { env } from 'node:process' import { CohereClient } from 'cohere-ai' -import { COHERE_MODELS } from '../utils/globals/llms' -import { err, logLLMCost } from '../utils/logging' +import { COHERE_MODELS } from '../utils/llm-utils' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' import type { CohereModelType } from '../utils/types/llms' /** diff --git a/src/llms/deepseek.ts b/src/llms/deepseek.ts index b7c3093..5c6c12b 100644 --- a/src/llms/deepseek.ts +++ b/src/llms/deepseek.ts @@ -7,8 +7,9 @@ import { env } from 'node:process' import { OpenAI } from 'openai' -import { err, logLLMCost } from '../utils/logging' -import { DEEPSEEK_MODELS } from '../utils/globals/llms' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' +import { DEEPSEEK_MODELS } from '../utils/llm-utils' import type { DeepSeekModelType } from '../utils/types/llms' /** diff --git a/src/llms/fireworks.ts b/src/llms/fireworks.ts index c4b5da9..373f1e2 100644 --- a/src/llms/fireworks.ts +++ b/src/llms/fireworks.ts @@ -1,8 +1,9 @@ // src/llms/fireworks.ts import { env } from 'node:process' -import { FIREWORKS_MODELS } from '../utils/globals/llms' -import { err, logLLMCost } from '../utils/logging' +import { FIREWORKS_MODELS } from '../utils/llm-utils' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' import type { FireworksModelType, FireworksResponse } from '../utils/types/llms' /** diff --git a/src/llms/gemini.ts b/src/llms/gemini.ts index 5342015..837a3fe 100644 --- a/src/llms/gemini.ts +++ b/src/llms/gemini.ts @@ -2,8 +2,9 @@ import { env } from 'node:process' import { GoogleGenerativeAI } from "@google/generative-ai" -import { GEMINI_MODELS } from '../utils/globals/llms' -import { err, logLLMCost } from '../utils/logging' +import { GEMINI_MODELS } from '../utils/llm-utils' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' import type { GeminiModelType } from '../utils/types/llms' /** diff --git a/src/llms/grok.ts b/src/llms/grok.ts index e3b7ead..9e1ffdd 100644 --- a/src/llms/grok.ts +++ b/src/llms/grok.ts @@ -7,7 +7,8 @@ import { env } from 'node:process' import { OpenAI } from 'openai' -import { err, logLLMCost } from '../utils/logging' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' import type { GroqChatCompletionResponse, GrokModelType } from '../utils/types/llms' /** diff --git a/src/llms/groq.ts b/src/llms/groq.ts index e5f607a..eb46826 100644 --- a/src/llms/groq.ts +++ b/src/llms/groq.ts @@ -1,8 +1,9 @@ // src/llms/groq.ts import { env } from 'node:process' -import { GROQ_MODELS } from '../utils/globals/llms' -import { err, logLLMCost } from '../utils/logging' +import { GROQ_MODELS } from '../utils/llm-utils' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' import type { GroqModelType, GroqChatCompletionResponse } from '../utils/types/llms' /** diff --git a/src/llms/mistral.ts b/src/llms/mistral.ts index 29b3b27..ff2290e 100644 --- a/src/llms/mistral.ts +++ b/src/llms/mistral.ts @@ -2,8 +2,9 @@ import { env } from 'node:process' import { Mistral } from '@mistralai/mistralai' -import { MISTRAL_MODELS } from '../utils/globals/llms' -import { err, logLLMCost } from '../utils/logging' +import { MISTRAL_MODELS } from '../utils/llm-utils' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' import type { MistralModelType } from '../utils/types/llms' /** diff --git a/src/llms/ollama.ts b/src/llms/ollama.ts index a7de3f5..c13c58a 100644 --- a/src/llms/ollama.ts +++ b/src/llms/ollama.ts @@ -1,9 +1,9 @@ // src/llms/ollama.ts import { env } from 'node:process' -import { OLLAMA_MODELS } from '../utils/globals/llms' -import { l, err, logLLMCost } from '../utils/logging' -import { checkOllamaServerAndModel } from '../utils/validate-option' +import { OLLAMA_MODELS } from '../utils/llm-utils' +import { l, err } from '../utils/logging' +import { logLLMCost, checkOllamaServerAndModel } from '../utils/llm-utils' import type { OllamaModelType, OllamaResponse } from '../utils/types/llms' /** @@ -21,8 +21,8 @@ export const callOllama = async ( transcript: string, model: string | OllamaModelType = 'QWEN_2_5_0B' ) => { - l.wait('\n callOllama called with arguments:') - l.wait(` - model: ${model}`) + l.dim('\n callOllama called with arguments:') + l.dim(` - model: ${model}`) try { // Determine the final modelKey from the argument @@ -30,12 +30,12 @@ export const callOllama = async ( const modelConfig = OLLAMA_MODELS[modelKey as OllamaModelType] || OLLAMA_MODELS.QWEN_2_5_0B const ollamaModelName = modelConfig.modelId - l.wait(` - modelName: ${modelKey}\n - ollamaModelName: ${ollamaModelName}`) + l.dim(` - modelName: ${modelKey}\n - ollamaModelName: ${ollamaModelName}`) // Determine host/port from environment or fallback const ollamaHost = env['OLLAMA_HOST'] || 'localhost' const ollamaPort = env['OLLAMA_PORT'] || '11434' - l.info(`\n [callOllama] OLLAMA_HOST=${ollamaHost}, OLLAMA_PORT=${ollamaPort}`) + l.dim(`\n [callOllama] OLLAMA_HOST=${ollamaHost}, OLLAMA_PORT=${ollamaPort}`) // Combine prompt + transcript const combinedPrompt = `${prompt}\n${transcript}` @@ -43,7 +43,7 @@ export const callOllama = async ( // Ensure Ollama server is running and that the model is pulled await checkOllamaServerAndModel(ollamaHost, ollamaPort, ollamaModelName) - l.wait(`\n Sending chat request to http://${ollamaHost}:${ollamaPort} using model '${ollamaModelName}'`) + l.dim(`\n Sending chat request to http://${ollamaHost}:${ollamaPort} using model '${ollamaModelName}'`) // Make the actual request to Ollama const response = await fetch(`http://${ollamaHost}:${ollamaPort}/api/chat`, { diff --git a/src/llms/together.ts b/src/llms/together.ts index ccd05d8..a066d09 100644 --- a/src/llms/together.ts +++ b/src/llms/together.ts @@ -1,8 +1,9 @@ // src/llms/together.ts import { env } from 'node:process' -import { TOGETHER_MODELS } from '../utils/globals/llms' -import { err, logLLMCost } from '../utils/logging' +import { TOGETHER_MODELS } from '../utils/llm-utils' +import { err } from '../utils/logging' +import { logLLMCost } from '../utils/llm-utils' import type { TogetherModelType, TogetherResponse } from '../utils/types/llms' /** diff --git a/src/process-commands/channel.ts b/src/process-commands/channel.ts index 4a94973..c179e27 100644 --- a/src/process-commands/channel.ts +++ b/src/process-commands/channel.ts @@ -15,9 +15,9 @@ */ import { processVideo } from './video' -import { execFilePromise } from '../utils/globals/process' -import { validateChannelOptions, saveInfo } from '../utils/validate-option' -import { l, err, logSeparator, logChannelProcessingStatus, logInitialFunctionCall } from '../utils/logging' +import { saveInfo, execFilePromise } from '../utils/validate-option' +import { l, err, logSeparator, logInitialFunctionCall } from '../utils/logging' +import { validateChannelOptions, logChannelProcessingStatus } from '../utils/channel-utils' import type { ProcessingOptions, VideoInfo } from '../utils/types/process' import type { TranscriptServices } from '../utils/types/transcription' diff --git a/src/process-commands/file.ts b/src/process-commands/file.ts index e25a5ef..39badd7 100644 --- a/src/process-commands/file.ts +++ b/src/process-commands/file.ts @@ -72,7 +72,7 @@ export async function processFile( await saveAudio(finalPath) } - l.wait(' processFile command completed successfully.') + l.dim('\n processFile command completed successfully.') return { frontMatter, diff --git a/src/process-commands/playlist.ts b/src/process-commands/playlist.ts index 2721459..332d52c 100644 --- a/src/process-commands/playlist.ts +++ b/src/process-commands/playlist.ts @@ -6,8 +6,7 @@ */ import { processVideo } from './video' -import { saveInfo } from '../utils/validate-option' -import { execFilePromise } from '../utils/globals/process' +import { saveInfo, execFilePromise } from '../utils/validate-option' import { l, err, logSeparator, logInitialFunctionCall } from '../utils/logging' import type { ProcessingOptions } from '../utils/types/process' import type { TranscriptServices } from '../utils/types/transcription' diff --git a/src/process-commands/rss.ts b/src/process-commands/rss.ts index 4ace0f9..6c40996 100644 --- a/src/process-commands/rss.ts +++ b/src/process-commands/rss.ts @@ -19,9 +19,10 @@ import { downloadAudio } from '../process-steps/02-download-audio' import { runTranscription } from '../process-steps/03-run-transcription' import { selectPrompts } from '../process-steps/04-select-prompt' import { runLLM } from '../process-steps/05-run-llm' -import { filterRSSItems, saveAudio, saveInfo } from '../utils/validate-option' -import { l, err, logSeparator, logInitialFunctionCall, logRSSProcessingStatus } from '../utils/logging' -import { parser } from '../utils/globals/process' +import { saveAudio, saveInfo, parser } from '../utils/validate-option' +import { filterRSSItems } from '../utils/rss-utils' +import { l, err, logSeparator, logInitialFunctionCall } from '../utils/logging' +import { logRSSProcessingStatus } from '../utils/rss-utils' import { retryRSSFetch } from '../utils/retry' import type { ProcessingOptions, RSSItem } from '../utils/types/process' @@ -114,12 +115,12 @@ export async function processRSS( logInitialFunctionCall('processRSS', { llmServices, transcriptServices }) if (options.item && options.item.length > 0) { - l.wait('\nProcessing specific items:') - options.item.forEach((url) => l.wait(` - ${url}`)) + l.dim('\nProcessing specific items:') + options.item.forEach((url) => l.dim(` - ${url}`)) } else if (options.last) { - l.wait(`\nProcessing the last ${options.last} items`) + l.dim(`\nProcessing the last ${options.last} items`) } else if (options.skip) { - l.wait(` - Skipping first ${options.skip || 0} items`) + l.dim(` - Skipping first ${options.skip || 0} items`) } try { @@ -134,7 +135,7 @@ export async function processRSS( } if (items.length === 0) { - l.wait('\nNo items found matching the provided criteria for this feed. Skipping...') + l.dim('\nNo items found matching the provided criteria for this feed. Skipping...') return } diff --git a/src/process-commands/video.ts b/src/process-commands/video.ts index 5bf7d4f..6f8ddd7 100644 --- a/src/process-commands/video.ts +++ b/src/process-commands/video.ts @@ -11,7 +11,7 @@ import { runTranscription } from '../process-steps/03-run-transcription' import { selectPrompts } from '../process-steps/04-select-prompt' import { runLLM } from '../process-steps/05-run-llm' import { saveAudio } from '../utils/validate-option' -import { l, err, logInitialFunctionCall } from '../utils/logging' +import { err, logInitialFunctionCall } from '../utils/logging' import type { ProcessingOptions } from '../utils/types/process' import type { TranscriptServices } from '../utils/types/transcription' import type { LLMServices } from '../utils/types/llms' @@ -70,8 +70,6 @@ export async function processVideo( await saveAudio(finalPath) } - l.wait('\n processVideo command completed successfully.') - return { frontMatter, prompt: selectedPrompts, diff --git a/src/process-steps/01-generate-markdown.ts b/src/process-steps/01-generate-markdown.ts index 4b39df7..79c5fb2 100644 --- a/src/process-steps/01-generate-markdown.ts +++ b/src/process-steps/01-generate-markdown.ts @@ -7,8 +7,7 @@ */ import { basename, extname } from 'node:path' -import { execFilePromise } from '../utils/globals/process' -import { sanitizeTitle, buildFrontMatter } from '../utils/validate-option' +import { sanitizeTitle, buildFrontMatter, execFilePromise } from '../utils/validate-option' import { l, err, logInitialFunctionCall } from '../utils/logging' import type { MarkdownData, ProcessingOptions, RSSItem } from '../utils/types/process' @@ -59,6 +58,7 @@ export async function generateMarkdown( options: ProcessingOptions, input: string | RSSItem ): Promise { + l.step(`\nStep 1 - Generate Markdown\n`) logInitialFunctionCall('generateMarkdown', { options, input }) const { filename, metadata } = await (async () => { @@ -68,7 +68,7 @@ export async function generateMarkdown( case !!options.urls: case !!options.channel: try { - l.wait('\n Extracting metadata with yt-dlp. Parsing output...\n') + l.dim(' Extracting metadata with yt-dlp. Parsing output...') const { stdout } = await execFilePromise('yt-dlp', [ '--restrict-filenames', '--print', '%(webpage_url)s', @@ -109,7 +109,7 @@ export async function generateMarkdown( } case !!options.file: - l.wait('\n Generating markdown for a local file...') + l.dim('\n Generating markdown for a local file...') const originalFilename = basename(input as string) const filenameWithoutExt = originalFilename.replace(extname(originalFilename), '') const localFilename = sanitizeTitle(filenameWithoutExt) @@ -128,7 +128,7 @@ export async function generateMarkdown( } case !!options.rss: - l.wait('\n Generating markdown for an RSS item...\n') + l.dim('\n Generating markdown for an RSS item...\n') const item = input as RSSItem const { publishDate, @@ -163,6 +163,7 @@ export async function generateMarkdown( const frontMatter = buildFrontMatter(metadata) const frontMatterContent = frontMatter.join('\n') - l.wait(` generateMarkdown returning:\n\n - finalPath: ${finalPath}\n - filename: ${filename}\n`) + l.dim(`\n generateMarkdown returning:\n\n - finalPath: ${finalPath}\n - filename: ${filename}\n`) + l.dim(`frontMatterContent:\n\n${frontMatterContent}\n`) return { frontMatter: frontMatterContent, finalPath, filename, metadata } } \ No newline at end of file diff --git a/src/process-steps/02-download-audio.ts b/src/process-steps/02-download-audio.ts index db6754c..a77afda 100644 --- a/src/process-steps/02-download-audio.ts +++ b/src/process-steps/02-download-audio.ts @@ -12,7 +12,7 @@ import { readFile, access } from 'node:fs/promises' import { fileTypeFromBuffer } from 'file-type' import { l, err, logInitialFunctionCall } from '../utils/logging' import { executeWithRetry } from '../utils/retry' -import { execPromise } from '../utils/globals/process' +import { execPromise } from '../utils/validate-option' import type { SupportedFileType, ProcessingOptions } from '../utils/types/process' /** @@ -79,6 +79,7 @@ export async function downloadAudio( input: string, filename: string ): Promise { + l.step(`\nStep 2 - Download Audio\n`) logInitialFunctionCall('downloadAudio', { options, input, filename }) const finalPath = `content/${filename}` @@ -100,7 +101,6 @@ export async function downloadAudio( ], 5 ) - l.wait(`\n Audio downloaded successfully:\n - ${outputPath}`) } catch (error) { err(`Error downloading audio: ${error instanceof Error ? error.message : String(error)}`) throw error @@ -111,26 +111,25 @@ export async function downloadAudio( 'mp4', 'mkv', 'avi', 'mov', 'webm', ]) try { - l.wait(`\n Checking file access:\n - ${input}`) await access(input) - l.wait(`\n File ${input} is accessible. Attempting to read file data for type detection...`) + l.dim(`\n File ${input} is accessible. Attempting to read file data for type detection...\n`) const buffer = await readFile(input) - l.wait(`\n Successfully read file: ${buffer.length} bytes`) + l.dim(` - Successfully read file: ${buffer.length} bytes`) const fileType = await fileTypeFromBuffer(buffer) - l.wait(`\n File type detection result: ${fileType?.ext ?? 'unknown'}`) + l.dim(` - File type detection result: ${fileType?.ext ?? 'unknown'}`) if (!fileType || !supportedFormats.has(fileType.ext as SupportedFileType)) { throw new Error( fileType ? `Unsupported file type: ${fileType.ext}` : 'Unable to determine file type' ) } - l.wait(` - Running ffmpeg command for ${input} -> ${outputPath}\n`) + l.dim(` - Running ffmpeg command for ${input} -> ${outputPath}\n`) await execPromise( `ffmpeg -i "${input}" -ar 16000 -ac 1 -c:a pcm_s16le "${outputPath}"` ) - l.wait(` File converted to WAV format successfully:\n - ${outputPath}`) + l.dim(` File converted to WAV format successfully:\n - ${outputPath}`) } catch (error) { err(`Error processing local file: ${error instanceof Error ? error.message : String(error)}`) throw error @@ -139,6 +138,6 @@ export async function downloadAudio( throw new Error('Invalid option provided for audio download/processing.') } - l.wait(`\n downloadAudio returning:\n - outputPath: ${outputPath}\n`) + l.dim(`\n downloadAudio returning:\n - outputPath: ${outputPath}\n`) return outputPath } \ No newline at end of file diff --git a/src/process-steps/03-run-transcription.ts b/src/process-steps/03-run-transcription.ts index 1757e09..ea8f1ca 100644 --- a/src/process-steps/03-run-transcription.ts +++ b/src/process-steps/03-run-transcription.ts @@ -23,6 +23,7 @@ export async function runTranscription( finalPath: string, transcriptServices?: TranscriptServices ) { + l.step(`\nStep 3 - Run Transcription\n`) logInitialFunctionCall('runTranscription', { options, finalPath, transcriptServices }) try { @@ -34,8 +35,8 @@ export async function runTranscription( 5, 5000 ) - l.wait('\n Deepgram transcription completed successfully.\n') - l.wait(`\n - deepgramModel: ${deepgramModel}`) + l.dim('\n Deepgram transcription completed successfully.\n') + l.dim(`\n - deepgramModel: ${deepgramModel}`) return deepgramTranscript case 'assembly': @@ -45,8 +46,8 @@ export async function runTranscription( 5, 5000 ) - l.wait('\n AssemblyAI transcription completed successfully.\n') - l.wait(`\n - assemblyModel: ${assemblyModel}`) + l.dim('\n AssemblyAI transcription completed successfully.\n') + l.dim(`\n - assemblyModel: ${assemblyModel}`) return assemblyTranscript case 'whisper': @@ -55,7 +56,7 @@ export async function runTranscription( 5, 5000 ) - l.wait('\n Whisper transcription completed successfully.\n') + l.dim('\n Whisper transcription completed successfully.\n') return whisperTranscript default: diff --git a/src/process-steps/04-select-prompt.ts b/src/process-steps/04-select-prompt.ts index c5c04ea..863dc8e 100644 --- a/src/process-steps/04-select-prompt.ts +++ b/src/process-steps/04-select-prompt.ts @@ -15,21 +15,22 @@ import type { ProcessingOptions } from '../utils/types/process' * @throws {Error} If the file cannot be read or is invalid */ export async function selectPrompts(options: ProcessingOptions) { + l.step(`\nStep 4 - Select Prompts\n`) logInitialFunctionCall('selectPrompts', { options }) let customPrompt = '' if (options.customPrompt) { - l.wait(`\n Custom prompt path provided, attempting to read: ${options.customPrompt}`) + l.dim(`\n Custom prompt path provided, attempting to read: ${options.customPrompt}`) - l.wait('\n readCustomPrompt called with arguments:\n') - l.wait(` - filePath: ${options.customPrompt}`) + l.dim('\n readCustomPrompt called with arguments:\n') + l.dim(` - filePath: ${options.customPrompt}`) try { - l.wait(`\n Reading custom prompt file:\n - ${options.customPrompt}`) + l.dim(`\n Reading custom prompt file:\n - ${options.customPrompt}`) const customPromptFileContents = await readFile(options.customPrompt, 'utf8') - l.wait(`\n Successfully read custom prompt file, character length:\n\n - ${customPromptFileContents.length}`) + l.dim(`\n Successfully read custom prompt file, character length:\n\n - ${customPromptFileContents.length}`) customPrompt = customPromptFileContents.trim() - l.wait('\n Custom prompt file successfully processed.') + l.dim('\n Custom prompt file successfully processed.') } catch (error) { err(`Error reading custom prompt file: ${(error as Error).message}`) customPrompt = '' @@ -46,7 +47,7 @@ export async function selectPrompts(options: ProcessingOptions) { const validSections = prompt.filter((section): section is keyof typeof sections => Object.hasOwn(sections, section) ) - l.wait(`\n Valid sections identified:\n\n ${JSON.stringify(validSections)}`) + l.dim(`${JSON.stringify(validSections, null, 2)}`) validSections.forEach((section) => { text += sections[section].instruction + "\n" @@ -56,5 +57,6 @@ export async function selectPrompts(options: ProcessingOptions) { validSections.forEach((section) => { text += ` ${sections[section].example}\n` }) + // l.dim(`\n selectPrompts returning:\n\n${text}`) return text } \ No newline at end of file diff --git a/src/process-steps/05-run-llm.ts b/src/process-steps/05-run-llm.ts index bdc668a..36543c4 100644 --- a/src/process-steps/05-run-llm.ts +++ b/src/process-steps/05-run-llm.ts @@ -6,11 +6,11 @@ * @packageDocumentation */ -import { writeFile } from 'node:fs/promises' +import { writeFile, readFile } from 'node:fs/promises' import { insertShowNote } from '../server/db' import { l, err, logInitialFunctionCall } from '../utils/logging' import { retryLLMCall } from '../utils/retry' -import { LLM_FUNCTIONS } from '../utils/globals/llms' +import { LLM_FUNCTIONS } from '../utils/llm-utils' import type { ProcessingOptions, EpisodeMetadata } from '../utils/types/process' import type { LLMServices } from '../utils/types/llms' @@ -50,12 +50,13 @@ export async function runLLM( metadata: EpisodeMetadata, llmServices?: LLMServices, ) { - logInitialFunctionCall('runLLM', { options, finalPath, frontMatter, prompt, transcript, metadata, llmServices }) + l.step(`\nStep 5 - Run Language Model\n`) + logInitialFunctionCall('runLLM', { llmServices, metadata }) try { let showNotesResult = '' if (llmServices) { - l.wait(`\n Preparing to process with '${llmServices}' Language Model...\n`) + l.dim(`\n Preparing to process with '${llmServices}' Language Model...\n`) const llmFunction = LLM_FUNCTIONS[llmServices] if (!llmFunction) { @@ -73,12 +74,12 @@ export async function runLLM( const outputFilename = `${finalPath}-${llmServices}-shownotes.md` await writeFile(outputFilename, `${frontMatter}\n${showNotes}\n\n## Transcript\n\n${transcript}`) - l.wait(`\n LLM processing completed, combined front matter + LLM output + transcript written to:\n - ${outputFilename}`) + l.dim(`\n LLM processing completed, combined front matter + LLM output + transcript written to:\n - ${outputFilename}`) showNotesResult = showNotes } else { - l.wait('\n No LLM selected, skipping processing...') + l.dim(' No LLM selected, skipping processing...') const noLLMFile = `${finalPath}-prompt.md` - l.wait(`\n Writing front matter + prompt + transcript to file:\n - ${noLLMFile}`) + l.dim(`\n Writing front matter + prompt + transcript to file:\n - ${noLLMFile}`) await writeFile(noLLMFile, `${frontMatter}\n${prompt}\n## Transcript\n\n${transcript}`) } @@ -101,4 +102,135 @@ export async function runLLM( err(`Error running Language Model: ${(error as Error).message}`) throw error } +} + +/** + * @public + * @typedef {Object} ParsedPromptFile + * @property {string} frontMatter - The extracted front matter (including --- lines). + * @property {string} prompt - The prompt text to be processed. + * @property {string} transcript - The transcript text to be processed (if any). + * @property {EpisodeMetadata} metadata - The metadata object parsed from front matter. + */ + +/** + * Utility function to parse a markdown file that may contain front matter, + * a prompt, and optionally a transcript section (marked by "## Transcript"). + * + * Front matter is assumed to be between the first pair of '---' lines at the top. + * The content after front matter and before "## Transcript" is considered prompt, + * and any content after "## Transcript" is considered transcript. + * + * Any recognized YAML keys in the front matter are mapped into the metadata object. + * + * @param {string} fileContent - The content of the markdown file + * @returns {ParsedPromptFile} An object containing frontMatter, prompt, transcript, and metadata + */ +function parsePromptFile(fileContent: string) { + let frontMatter = '' + let prompt = '' + let transcript = '' + let metadata: EpisodeMetadata = { + showLink: '', + channel: '', + channelURL: '', + title: '', + description: '', + publishDate: '', + coverImage: '' + } + + const lines = fileContent.split('\n') + let readingFrontMatter = false + let frontMatterDone = false + let readingTranscript = false + + for (const line of lines) { + if (!frontMatterDone && line.trim() === '---') { + // Toggle front matter reading on/off + readingFrontMatter = !readingFrontMatter + + frontMatter += `${line}\n` + if (!readingFrontMatter) { + frontMatterDone = true + } + continue + } + + if (!frontMatterDone && readingFrontMatter) { + // Inside front matter + frontMatter += `${line}\n` + + // Capture known metadata from lines like: key: "value" + const match = line.match(/^(\w+):\s*"?([^"]+)"?/) + if (match) { + const key = match[1] + const value = match[2] + if (key === 'showLink') metadata.showLink = value + if (key === 'channel') metadata.channel = value + if (key === 'channelURL') metadata.channelURL = value + if (key === 'title') metadata.title = value + if (key === 'description') metadata.description = value + if (key === 'publishDate') metadata.publishDate = value + if (key === 'coverImage') metadata.coverImage = value + } + continue + } + + if (line.trim().toLowerCase().startsWith('## transcript')) { + readingTranscript = true + transcript += `${line}\n` + continue + } + + if (readingTranscript) { + transcript += `${line}\n` + } else { + prompt += `${line}\n` + } + } + + return { frontMatter, prompt, transcript, metadata } +} + +/** + * Reads a prompt markdown file and runs Step 5 (LLM processing) directly, + * bypassing the earlier steps of front matter generation, audio download, and transcription. + * + * The markdown file is expected to contain optional front matter delimited by '---' lines, + * followed by prompt text, and optionally a "## Transcript" section. + * + * This function extracts that content and calls {@link runLLM} with the user-specified LLM service. + * + * @param {string} filePath - The path to the .md file containing front matter, prompt, and optional transcript + * @param {ProcessingOptions} options - Configuration options (including any LLM model flags) + * @param {LLMServices} llmServices - The chosen LLM service (e.g., 'chatgpt', 'claude', etc.) + * @returns {Promise} A promise that resolves when the LLM processing completes + */ +export async function runLLMFromPromptFile( + filePath: string, + options: ProcessingOptions, + llmServices: LLMServices, +): Promise { + try { + const fileContent = await readFile(filePath, 'utf8') + const { frontMatter, prompt, transcript, metadata } = parsePromptFile(fileContent) + + // Derive a base "finalPath" from the file path, removing the .md extension if present + const finalPath = filePath.replace(/\.[^.]+$/, '') + + // Execute Step 5 + await runLLM( + options, + finalPath, + frontMatter, + prompt, + transcript, + metadata, + llmServices + ) + } catch (error) { + err(`Error in runLLMFromPromptFile: ${(error as Error).message}`) + throw error + } } \ No newline at end of file diff --git a/src/server/db.ts b/src/server/db.ts index e4a3094..e4d32ea 100644 --- a/src/server/db.ts +++ b/src/server/db.ts @@ -52,7 +52,7 @@ export function insertShowNote( transcript: string, llmOutput: string ): void { - l.wait('\n Inserting show note into the database...') + l.dim('\n Inserting show note into the database...') db.prepare(` INSERT INTO show_notes ( showLink, @@ -81,5 +81,5 @@ export function insertShowNote( transcript, llmOutput ) - l.wait('\n - Show note inserted successfully.\n') + l.dim(' - Show note inserted successfully.\n') } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 953e835..871cc7d 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,7 +7,7 @@ import { handleProcessRequest } from './routes/process' import { getShowNotes } from './routes/show-notes' import { getShowNote } from './routes/show-note' import { l } from '../../src/utils/logging' -import { envVarsServerMap } from '../utils/globals/llms' +import { envVarsServerMap } from '../utils/llm-utils' import type { RequestBody } from '../utils/types/process' // Set server port from environment variable or default to 3000 diff --git a/src/server/utils/validation.ts b/src/server/utils/validation.ts index ffe60c7..372fa43 100644 --- a/src/server/utils/validation.ts +++ b/src/server/utils/validation.ts @@ -1,7 +1,7 @@ -import { LLM_OPTIONS } from '../../utils/globals/llms' -import { TRANSCRIPT_OPTIONS } from '../../utils/globals/transcription' +import { LLM_OPTIONS } from '../../utils/llm-utils' +import { TRANSCRIPT_OPTIONS } from '../../utils/transcription-utils' import type { ProcessingOptions, ValidAction } from '../../utils/types/process' import type { TranscriptServices } from '../../utils/types/transcription' diff --git a/src/transcription/assembly.ts b/src/transcription/assembly.ts index 430efdf..243a5d5 100644 --- a/src/transcription/assembly.ts +++ b/src/transcription/assembly.ts @@ -10,9 +10,8 @@ import { readFile } from 'node:fs/promises' import { env } from 'node:process' -import { l, err, logTranscriptionCost } from '../utils/logging' -import { formatAssemblyTranscript } from './format-transcript' -import { ASSEMBLY_MODELS } from '../utils/globals/transcription' +import { l, err } from '../utils/logging' +import { ASSEMBLY_MODELS, logTranscriptionCost, formatAssemblyTranscript } from '../utils/transcription-utils' import type { ProcessingOptions } from '../utils/types/process' import type { AssemblyAITranscriptionOptions, @@ -38,9 +37,9 @@ export async function callAssembly( finalPath: string, model: string = 'NANO' ) { - l.wait('\n callAssembly called with arguments:\n') - l.wait(` - finalPath: ${finalPath}`) - l.wait(` - model: ${model}`) + l.dim('\n callAssembly called with arguments:') + l.dim(` - finalPath: ${finalPath}`) + l.dim(` - model: ${model}`) if (!env['ASSEMBLY_API_KEY']) { throw new Error('ASSEMBLY_API_KEY environment variable is not set. Please set it to your AssemblyAI API key.') @@ -64,7 +63,7 @@ export async function callAssembly( }) // Step 1: Uploading the audio file to AssemblyAI - l.wait('\n Uploading audio file to AssemblyAI...') + l.dim('\n Uploading audio file to AssemblyAI...') const fileBuffer = await readFile(audioFilePath) const uploadResponse = await fetch(`${BASE_URL}/upload`, { @@ -86,7 +85,7 @@ export async function callAssembly( if (!upload_url) { throw new Error('Upload URL not returned by AssemblyAI.') } - l.wait(' - Audio file uploaded successfully.') + l.dim(' - Audio file uploaded successfully.') // Step 2: Requesting the transcription const transcriptionOptions: AssemblyAITranscriptionOptions = { diff --git a/src/transcription/deepgram.ts b/src/transcription/deepgram.ts index 0c10c34..a1fc60a 100644 --- a/src/transcription/deepgram.ts +++ b/src/transcription/deepgram.ts @@ -10,9 +10,8 @@ import { readFile } from 'node:fs/promises' import { env } from 'node:process' -import { l, err, logTranscriptionCost } from '../utils/logging' -import { formatDeepgramTranscript } from './format-transcript' -import { DEEPGRAM_MODELS } from '../utils/globals/transcription' +import { l, err } from '../utils/logging' +import { DEEPGRAM_MODELS, logTranscriptionCost, formatDeepgramTranscript } from '../utils/transcription-utils' import type { ProcessingOptions } from '../utils/types/process' import type { DeepgramResponse, DeepgramModelType } from '../utils/types/transcription' @@ -29,9 +28,9 @@ export async function callDeepgram( finalPath: string, model: string = 'NOVA_2' ) { - l.wait('\n callDeepgram called with arguments:\n') - l.wait(` - finalPath: ${finalPath}`) - l.wait(` - model: ${model}`) + l.dim('\n callDeepgram called with arguments:') + l.dim(` - finalPath: ${finalPath}`) + l.dim(` - model: ${model}`) if (!env['DEEPGRAM_API_KEY']) { throw new Error('DEEPGRAM_API_KEY environment variable is not set. Please set it to your Deepgram API key.') diff --git a/src/transcription/format-transcript.ts b/src/transcription/format-transcript.ts deleted file mode 100644 index 6f928bd..0000000 --- a/src/transcription/format-transcript.ts +++ /dev/null @@ -1,210 +0,0 @@ -// src/utils/format-transcript.ts - -// This file contains utility functions to format transcripts from different transcription services into -// a uniform plain text format with timestamps. It includes: -// - formatDeepgramTranscript: Formats transcripts returned by Deepgram -// - formatAssemblyTranscript: Formats transcripts returned by AssemblyAI -// - formatWhisperTranscript: Converts LRC files to plain text with timestamps - -import type { - AssemblyAIPollingResponse, - AssemblyAIUtterance, - AssemblyAIWord -} from '../utils/types/transcription' - -/** - * Formats the Deepgram transcript by adding timestamps and newlines based on conditions. - * Rules: - * - Add a timestamp if it's the first word, every 30th word, or the start of a sentence (capitalized word). - * - Insert a newline if the word ends a sentence (ends in punctuation), every 30th word, or it's the last word. - * - * @param words - The array of word objects from Deepgram (each contains a 'word' and 'start' time) - * @returns A formatted transcript string with timestamps and newlines - */ -export function formatDeepgramTranscript(words: Array<{ word: string; start: number }>): string { - return words.reduce((acc, { word, start }, i, arr) => { - const addTimestamp = (i % 30 === 0 || /^[A-Z]/.test(word)) - let timestamp = '' - if (addTimestamp) { - const minutes = Math.floor(start / 60).toString().padStart(2, '0') - const seconds = Math.floor(start % 60).toString().padStart(2, '0') - timestamp = `[${minutes}:${seconds}] ` - } - - const endOfSentence = /[.!?]$/.test(word) - const endOfBlock = (i % 30 === 29 || i === arr.length - 1) - const newline = (endOfSentence || endOfBlock) ? '\n' : '' - - return `${acc}${timestamp}${word} ${newline}` - }, '') -} - -/** - * Formats the AssemblyAI transcript into text with timestamps and optional speaker labels. - * Logic: - * - If transcript.utterances are present, format each utterance line with optional speaker labels and timestamps. - * - If only transcript.words are available, group them into lines ~80 chars, prepend each line with a timestamp. - * - If no structured data is available, use the raw transcript text or 'No transcription available.' as fallback. - * - * @param transcript - The polling response from AssemblyAI after transcription completes - * @param speakerLabels - Whether to include speaker labels in the output - * @returns The fully formatted transcript as a string - * @throws If words are expected but not found (no content to format) - */ -export function formatAssemblyTranscript(transcript: AssemblyAIPollingResponse, speakerLabels: boolean): string { - // Helper inline formatting function for timestamps (AssemblyAI returns ms) - const inlineFormatTime = (timestamp: number): string => { - const totalSeconds = Math.floor(timestamp / 1000) - const minutes = Math.floor(totalSeconds / 60).toString().padStart(2, '0') - const seconds = (totalSeconds % 60).toString().padStart(2, '0') - return `${minutes}:${seconds}` - } - - let txtContent = '' - - if (transcript.utterances && transcript.utterances.length > 0) { - // If utterances are available, format each line with optional speaker labels and timestamps - txtContent = transcript.utterances.map((utt: AssemblyAIUtterance) => - `${speakerLabels ? `Speaker ${utt.speaker} ` : ''}(${inlineFormatTime(utt.start)}): ${utt.text}` - ).join('\n') - } else if (transcript.words && transcript.words.length > 0) { - // If only words are available, we must form lines with timestamps every ~80 chars - const firstWord = transcript.words[0] - if (!firstWord) { - throw new Error('No words found in transcript') - } - - let currentLine = '' - let currentTimestamp = inlineFormatTime(firstWord.start) - - transcript.words.forEach((word: AssemblyAIWord) => { - if (currentLine.length + word.text.length > 80) { - // Start a new line if the current line exceeds ~80 characters - txtContent += `[${currentTimestamp}] ${currentLine.trim()}\n` - currentLine = '' - currentTimestamp = inlineFormatTime(word.start) - } - currentLine += `${word.text} ` - }) - - // Add any remaining text as a final line - if (currentLine.length > 0) { - txtContent += `[${currentTimestamp}] ${currentLine.trim()}\n` - } - } else { - // If no utterances or words, fallback to transcript.text or a default message - txtContent = transcript.text || 'No transcription available.' - } - - return txtContent -} - -/** - * Converts LRC content (common lyrics file format) to plain text with timestamps. - * - Strips out lines that contain certain metadata (like [by:whisper.cpp]). - * - Converts original timestamps [MM:SS.xx] to a simplified [MM:SS] format. - * - Properly extracts all timestamps in each line, then merges them into - * chunks of up to 15 words, adopting the newest timestamp as soon - * as it appears. - * - * @param lrcContent - The content of the LRC file as a string - * @returns The converted text content with simple timestamps - */ -export function formatWhisperTranscript(lrcContent: string): string { - // 1) Remove lines like `[by:whisper.cpp]`, convert "[MM:SS.xx]" to "[MM:SS]" - const rawLines = lrcContent - .split('\n') - .filter(line => !line.startsWith('[by:whisper.cpp]')) - .map(line => - line.replace( - /\[(\d{1,3}):(\d{2})(\.\d+)?\]/g, - (_, minutes, seconds) => `[${minutes}:${seconds}]` - ) - ) - - // We define a Segment with timestamp: string | undefined - type Segment = { - timestamp: string | undefined - words: string[] - } - - /** - * Given a line (which may contain multiple [MM:SS] tags), - * extract those timestamps + the words in between. - */ - function parseLineIntoSegments(line: string): Segment[] { - const segments: Segment[] = [] - const pattern = /\[(\d{1,3}:\d{2})\]/g - - let lastIndex = 0 - let match: RegExpExecArray | null - let currentTimestamp: string | undefined = undefined - - while ((match = pattern.exec(line)) !== null) { - // Text before this timestamp - const textBeforeThisTimestamp = line.slice(lastIndex, match.index).trim() - if (textBeforeThisTimestamp) { - segments.push({ - timestamp: currentTimestamp, - words: textBeforeThisTimestamp.split(/\s+/).filter(Boolean), - }) - } - // Update timestamp to the newly found one - currentTimestamp = match[1] - lastIndex = pattern.lastIndex - } - - // After the last timestamp, grab any trailing text - const trailing = line.slice(lastIndex).trim() - if (trailing) { - segments.push({ - timestamp: currentTimestamp, - words: trailing.split(/\s+/).filter(Boolean), - }) - } - - // If line had no timestamps, the entire line is one segment with `timestamp: undefined`. - return segments - } - - // 2) Flatten all lines into an array of typed segments - const allSegments: Segment[] = rawLines.flatMap(line => parseLineIntoSegments(line)) - - // 3) Accumulate words into lines up to 15 words each. - // Whenever we see a new timestamp, we finalize the previous chunk - // and start a new chunk with that timestamp. - const finalLines: string[] = [] - let currentTimestamp: string | undefined = undefined - let currentWords: string[] = [] - - function finalizeChunk() { - if (currentWords.length > 0) { - // If we have never encountered a timestamp, default to "00:00" - const tsToUse = currentTimestamp ?? '00:00' - finalLines.push(`[${tsToUse}] ${currentWords.join(' ')}`) - currentWords = [] - } - } - - for (const segment of allSegments) { - // If this segment has a new timestamp, finalize the old chunk and start new - if (segment.timestamp !== undefined) { - finalizeChunk() - currentTimestamp = segment.timestamp - } - - // Accumulate words from this segment, chunking at 15 - for (const word of segment.words) { - currentWords.push(word) - if (currentWords.length === 15) { - finalizeChunk() - } - } - } - - // 4) Finalize any leftover words - finalizeChunk() - - // 5) Return as simple text - return finalLines.join('\n') -} \ No newline at end of file diff --git a/src/transcription/whisper.ts b/src/transcription/whisper.ts index 07f9e72..1368540 100644 --- a/src/transcription/whisper.ts +++ b/src/transcription/whisper.ts @@ -6,10 +6,8 @@ */ import { readFile, unlink } from 'node:fs/promises' -// import { formatWhisperTranscript } from './format-transcript' -import { checkWhisperDirAndModel } from '../utils/validate-option' -import { WHISPER_MODELS } from '../utils/globals/transcription' -import { execPromise } from '../utils/globals/process' +import { WHISPER_MODELS, checkWhisperDirAndModel } from '../utils/transcription-utils' +import { execPromise } from '../utils/validate-option' import { l, err } from '../utils/logging' import type { ProcessingOptions } from '../utils/types/process' import type { WhisperModelType } from '../utils/types/transcription' @@ -24,8 +22,8 @@ export async function callWhisper( options: ProcessingOptions, finalPath: string ) { - l.wait('\n callWhisper called with arguments:\n') - l.wait(` - finalPath: ${finalPath}`) + l.opts('\n callWhisper called with arguments:') + l.opts(` - finalPath: ${finalPath}`) try { // Determine which model was requested (default to "base" if `--whisper` is passed with no model) @@ -40,15 +38,15 @@ export async function callWhisper( throw new Error(`Unknown model type: ${whisperModel}`) } - l.wait(`\n Whisper model information:\n\n - whisperModel: ${whisperModel}`) + l.dim(`\n Whisper model information:\n\n - whisperModel: ${whisperModel}`) const modelGGMLName = WHISPER_MODELS[whisperModel as WhisperModelType] - l.wait(` - modelGGMLName: ${modelGGMLName}`) + l.dim(` - modelGGMLName: ${modelGGMLName}`) await checkWhisperDirAndModel(whisperModel, modelGGMLName) // Run whisper.cpp on the WAV file - l.wait(` Invoking whisper.cpp on file:\n - ${finalPath}.wav`) + l.dim(` Invoking whisper.cpp on file:\n - ${finalPath}.wav`) try { await execPromise( `./whisper.cpp/build/bin/whisper-cli --no-gpu ` + @@ -62,17 +60,13 @@ export async function callWhisper( throw whisperError } - // Convert .lrc -> .txt - l.wait(`\n Transcript LRC file successfully created, reading file for txt conversion:\n - ${finalPath}.lrc\n`) + l.dim(`\n Transcript LRC file successfully created, reading file for txt conversion:\n - ${finalPath}.lrc\n`) const lrcContent = await readFile(`${finalPath}.lrc`, 'utf8') - l.dim(lrcContent) + // l.dim(lrcContent) const txtContent = `${lrcContent}` - // const txtContent = formatWhisperTranscript(lrcContent) - l.dim(txtContent) await unlink(`${finalPath}.lrc`) // Return the transcript text - l.wait('\n Returning transcript text from callWhisper...') return txtContent } catch (error) { err('Error in callWhisper:', (error as Error).message) diff --git a/src/utils/channel-utils.ts b/src/utils/channel-utils.ts new file mode 100644 index 0000000..ec0e0ee --- /dev/null +++ b/src/utils/channel-utils.ts @@ -0,0 +1,65 @@ +// src/utils/channel-utils.ts + +import { l, err } from '../utils/logging' + +import type { ProcessingOptions } from './types/process' + +/** + * Validates channel processing options for consistency and correct values. + * Logs the current channel processing action based on provided options. + * + * @param options - Configuration options to validate + * @throws Will exit the process if validation fails + */ +export function validateChannelOptions(options: ProcessingOptions): void { + if (options.last !== undefined) { + if (!Number.isInteger(options.last) || options.last < 1) { + err('Error: The --last option must be a positive integer.') + process.exit(1) + } + if (options.skip !== undefined || options.order !== undefined) { + err('Error: The --last option cannot be used with --skip or --order.') + process.exit(1) + } + } + + if (options.skip !== undefined && (!Number.isInteger(options.skip) || options.skip < 0)) { + err('Error: The --skip option must be a non-negative integer.') + process.exit(1) + } + + if (options.order !== undefined && !['newest', 'oldest'].includes(options.order)) { + err("Error: The --order option must be either 'newest' or 'oldest'.") + process.exit(1) + } + + if (options.last) { + l.dim(`\nProcessing the last ${options.last} videos`) + } else if (options.skip) { + l.dim(`\nSkipping first ${options.skip || 0} videos`) + } +} + +/** + * Logs the processing status and video counts for channel downloads. + * + * @param total - Total number of videos found. + * @param processing - Number of videos to process. + * @param options - Configuration options. + */ +export function logChannelProcessingStatus( + total: number, + processing: number, + options: ProcessingOptions +): void { + if (options.last) { + l.dim(`\n - Found ${total} videos in the channel.`) + l.dim(` - Processing the last ${processing} videos.`) + } else if (options.skip) { + l.dim(`\n - Found ${total} videos in the channel.`) + l.dim(` - Processing ${processing} videos after skipping ${options.skip || 0}.\n`) + } else { + l.dim(`\n - Found ${total} videos in the channel.`) + l.dim(` - Processing all ${processing} videos.\n`) + } +} \ No newline at end of file diff --git a/src/utils/globals/process.ts b/src/utils/globals/process.ts deleted file mode 100644 index 27165a9..0000000 --- a/src/utils/globals/process.ts +++ /dev/null @@ -1,119 +0,0 @@ -// src/utils/globals.ts - -/** - * @file Defines constants, model mappings, and utility functions used throughout the application. - * @packageDocumentation - */ - -import { exec, execFile } from 'node:child_process' -import { promisify } from 'node:util' -import { processVideo } from '../../process-commands/video' -import { processPlaylist } from '../../process-commands/playlist' -import { processChannel } from '../../process-commands/channel' -import { processURLs } from '../../process-commands/urls' -import { processFile } from '../../process-commands/file' -import { processRSS } from '../../process-commands/rss' - -import type { ValidAction, HandlerFunction } from '../types/process' - -export const execPromise = promisify(exec) -export const execFilePromise = promisify(execFile) - -import { XMLParser } from 'fast-xml-parser' - -/** - * Configure XML parser for RSS feed processing. - * Handles attributes without prefixes and allows boolean values. - * - */ -export const parser = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '', - allowBooleanAttributes: true, -}) - -/* ------------------------------------------------------------------ - * Prompt & Action Choices - * ------------------------------------------------------------------ */ - -// Map each action to its corresponding handler function -export const PROCESS_HANDLERS: Record = { - video: processVideo, - playlist: processPlaylist, - channel: processChannel, - urls: processURLs, - file: processFile, - rss: processRSS, -} - -/** - * Provides user-friendly prompt choices for content generation or summary tasks. - * - */ -export const PROMPT_CHOICES: Array<{ name: string; value: string }> = [ - { name: 'Titles', value: 'titles' }, - { name: 'Summary', value: 'summary' }, - { name: 'Short Summary', value: 'shortSummary' }, - { name: 'Long Summary', value: 'longSummary' }, - { name: 'Bullet Point Summary', value: 'bulletPoints' }, - { name: 'Short Chapters', value: 'shortChapters' }, - { name: 'Medium Chapters', value: 'mediumChapters' }, - { name: 'Long Chapters', value: 'longChapters' }, - { name: 'Key Takeaways', value: 'takeaways' }, - { name: 'Questions', value: 'questions' }, - { name: 'FAQ', value: 'faq' }, - { name: 'Blog', value: 'blog' }, - { name: 'Rap Song', value: 'rapSong' }, - { name: 'Rock Song', value: 'rockSong' }, - { name: 'Country Song', value: 'countrySong' }, -] - -/** - * Available action options for content processing with additional metadata. - * - */ -export const ACTION_OPTIONS: Array<{ - name: string - description: string - message: string - validate: (input: string) => boolean | string -}> = [ - { - name: 'video', - description: 'Single YouTube Video', - message: 'Enter the YouTube video URL:', - validate: (input: string) => (input ? true : 'Please enter a valid URL.'), - }, - { - name: 'playlist', - description: 'YouTube Playlist', - message: 'Enter the YouTube playlist URL:', - validate: (input: string) => (input ? true : 'Please enter a valid URL.'), - }, - { - name: 'channel', - description: 'YouTube Channel', - message: 'Enter the YouTube channel URL:', - validate: (input: string) => (input ? true : 'Please enter a valid URL.'), - }, - { - name: 'urls', - description: 'List of URLs from File', - message: 'Enter the file path containing URLs:', - validate: (input: string) => - (input ? true : 'Please enter a valid file path.'), - }, - { - name: 'file', - description: 'Local Audio/Video File', - message: 'Enter the local audio/video file path:', - validate: (input: string) => - (input ? true : 'Please enter a valid file path.'), - }, - { - name: 'rss', - description: 'Podcast RSS Feed', - message: 'Enter the podcast RSS feed URL:', - validate: (input: string) => (input ? true : 'Please enter a valid URL.'), - }, -] \ No newline at end of file diff --git a/src/utils/globals/transcription.ts b/src/utils/globals/transcription.ts deleted file mode 100644 index 68b62aa..0000000 --- a/src/utils/globals/transcription.ts +++ /dev/null @@ -1,114 +0,0 @@ -// src/utils/transcription-globals.ts - -/** - * @file Defines Deepgram and Assembly transcription model configurations, - * including name, modelId, and cost per minute. - */ - -import type { WhisperModelType, TranscriptServiceConfig } from '../types/transcription' -import type { DeepgramModelType, AssemblyModelType } from '../types/transcription' - -/* ------------------------------------------------------------------ - * Transcription Services & Models - * ------------------------------------------------------------------ */ - -/** - * Available transcription services and their configuration. - * - */ -export const TRANSCRIPT_SERVICES: Record = { - WHISPER: { name: 'Whisper.cpp', value: 'whisper', isWhisper: true }, - DEEPGRAM: { name: 'Deepgram', value: 'deepgram' }, - ASSEMBLY: { name: 'AssemblyAI', value: 'assembly' }, -} as const - -/** - * Array of valid transcription service values. - * - */ -export const TRANSCRIPT_OPTIONS: string[] = Object.values(TRANSCRIPT_SERVICES).map( - (service) => service.value -) - -/** - * Whisper-only transcription services (subset of TRANSCRIPT_SERVICES). - * - */ -export const WHISPER_SERVICES: string[] = Object.values(TRANSCRIPT_SERVICES) - .filter( - ( - service - ): service is TranscriptServiceConfig & { - isWhisper: true - } => service.isWhisper === true - ) - .map((service) => service.value) - -/** - * Mapping of Whisper model types to their corresponding binary filenames for whisper.cpp. - * @type {Record} - */ -export const WHISPER_MODELS: Record = { - tiny: 'ggml-tiny.bin', - 'tiny.en': 'ggml-tiny.en.bin', - base: 'ggml-base.bin', - 'base.en': 'ggml-base.en.bin', - small: 'ggml-small.bin', - 'small.en': 'ggml-small.en.bin', - medium: 'ggml-medium.bin', - 'medium.en': 'ggml-medium.en.bin', - 'large-v1': 'ggml-large-v1.bin', - 'large-v2': 'ggml-large-v2.bin', - 'large-v3-turbo': 'ggml-large-v3-turbo.bin', - turbo: 'ggml-large-v3-turbo.bin', -} - -/** - * Deepgram models with their per-minute cost. - */ -export const DEEPGRAM_MODELS: Record = { - NOVA_2: { - name: 'Nova-2', - modelId: 'nova-2', - costPerMinute: 0.0043 - }, - NOVA: { - name: 'Nova', - modelId: 'nova', - costPerMinute: 0.0043 - }, - ENHANCED: { - name: 'Enhanced', - modelId: 'enhanced', - costPerMinute: 0.0145 - }, - BASE: { - name: 'Base', - modelId: 'base', - costPerMinute: 0.0125 - } -} - -/** - * AssemblyAI models with their per-minute cost. - */ -export const ASSEMBLY_MODELS: Record = { - BEST: { - name: 'Best', - modelId: 'best', - costPerMinute: 0.0062 - }, - NANO: { - name: 'Nano', - modelId: 'nano', - costPerMinute: 0.0020 - } -} \ No newline at end of file diff --git a/src/utils/globals/llms.ts b/src/utils/llm-utils.ts similarity index 55% rename from src/utils/globals/llms.ts rename to src/utils/llm-utils.ts index ac16620..19d6e1a 100644 --- a/src/utils/globals/llms.ts +++ b/src/utils/llm-utils.ts @@ -1,16 +1,21 @@ // src/utils/globals/llms.ts -import { callOllama } from '../../llms/ollama' -import { callChatGPT } from '../../llms/chatgpt' -import { callClaude } from '../../llms/claude' -import { callGemini } from '../../llms/gemini' -import { callCohere } from '../../llms/cohere' -import { callMistral } from '../../llms/mistral' -import { callDeepSeek } from '../../llms/deepseek' -import { callGrok } from '../../llms/grok' -import { callFireworks } from '../../llms/fireworks' -import { callTogether } from '../../llms/together' -import { callGroq } from '../../llms/groq' +import { spawn } from 'node:child_process' +import { readFile } from 'node:fs/promises' +import { callOllama } from '../llms/ollama' +import { callChatGPT } from '../llms/chatgpt' +import { callClaude } from '../llms/claude' +import { callGemini } from '../llms/gemini' +import { callCohere } from '../llms/cohere' +import { callMistral } from '../llms/mistral' +import { callDeepSeek } from '../llms/deepseek' +import { callGrok } from '../llms/grok' +import { callFireworks } from '../llms/fireworks' +import { callTogether } from '../llms/together' +import { callGroq } from '../llms/groq' + +import chalk from 'chalk' +import { l, err } from './logging' import type { ModelConfig, @@ -25,10 +30,189 @@ import type { TogetherModelType, FireworksModelType, GroqModelType, -} from '../types/llms' -import type { RequestBody } from '../types/process' -import type { LLMServiceConfig, LLMServices } from '../types/llms' -import type { ModelConfigValue } from '../types/llms' + LLMServiceConfig, + LLMServices, + ModelConfigValue, + OllamaTagsResponse +} from './types/llms' +import type { LogLLMCost } from './types/logging' +import type { RequestBody, ProcessingOptions } from './types/process' + +/** + * Finds the model configuration based on the model key + * @param modelKey - The key/name of the model (e.g., 'LLAMA_3_2_3B') + * @returns The model configuration if found, undefined otherwise + */ +function findModelConfig(modelKey: string) { + // First try to find the model directly in our combined models + const model = ALL_MODELS[modelKey] + if (model) return model + + // If not found by key, try matching by model ID as a fallback + return Object.values(ALL_MODELS).find(model => + model.modelId.toLowerCase() === modelKey.toLowerCase() + ) +} + +/** + * Formats a cost value to a standardized string representation + * @param cost - The cost value to format + * @returns Formatted cost string + */ +function formatCost(cost: number | undefined): string { + if (cost === undefined) return 'N/A' + if (cost === 0) return '$0.0000' + return `$${cost.toFixed(4)}` +} + +/** + * Logs API call results in a standardized format across different LLM providers. + * Includes token usage and cost calculations. + * @param info - Object containing model info, stop reason, and token usage + */ +export function logLLMCost(info: LogLLMCost): void { + const { modelName, stopReason, tokenUsage } = info + + // Get model display name if available, otherwise use the provided name + const modelConfig = findModelConfig(modelName) + const displayName = modelConfig?.name ?? modelName + + // Log stop/finish reason and model + l.dim(` - ${stopReason ? `${stopReason} Reason` : 'Status'}: ${stopReason}\n - Model: ${displayName}`) + + // Format token usage string based on available data + const tokenLines = [] + if (tokenUsage.input) tokenLines.push(`${tokenUsage.input} input tokens`) + if (tokenUsage.output) tokenLines.push(`${tokenUsage.output} output tokens`) + if (tokenUsage.total) tokenLines.push(`${tokenUsage.total} total tokens`) + + // Log token usage if any data is available + if (tokenLines.length > 0) { + l.dim(` - Token Usage:\n - ${tokenLines.join('\n - ')}`) + } + + // Calculate and log costs + let inputCost: number | undefined + let outputCost: number | undefined + let totalCost: number | undefined + + // Check if model config is found + if (!modelConfig) { + console.warn(`Warning: Could not find cost configuration for model: ${modelName}`) + } else if (modelConfig.inputCostPer1M === 0 && modelConfig.outputCostPer1M === 0) { + // If both costs per million are zero, return all zeros + inputCost = 0 + outputCost = 0 + totalCost = 0 + } else { + // Calculate costs if token usage is available + if (tokenUsage.input) { + const rawInputCost = (tokenUsage.input / 1_000_000) * modelConfig.inputCostPer1M + inputCost = Math.abs(rawInputCost) < 0.00001 ? 0 : rawInputCost + } + + if (tokenUsage.output) { + outputCost = (tokenUsage.output / 1_000_000) * modelConfig.outputCostPer1M + } + + // Calculate total cost only if both input and output costs are available + if (inputCost !== undefined && outputCost !== undefined) { + totalCost = inputCost + outputCost + } + } + + const costLines = [] + + if (inputCost !== undefined) { + costLines.push(`Input cost: ${formatCost(inputCost)}`) + } + if (outputCost !== undefined) { + costLines.push(`Output cost: ${formatCost(outputCost)}`) + } + if (totalCost !== undefined) { + costLines.push(`Total cost: ${chalk.bold(formatCost(totalCost))}`) + } + + // Log costs if any calculations were successful + if (costLines.length > 0) { + l.dim(` - Cost Breakdown:\n - ${costLines.join('\n - ')}`) + } +} + +/** + * Minimal token counting utility. Splits on whitespace to get an approximate token count. + * For more accurate results with ChatGPT, a library like 'tiktoken' can be integrated. + * + * @param text - The text for which we need an approximate token count + * @returns Approximate token count + */ +function approximateTokens(text: string): number { + const words = text.trim().split(/\s+/) + // This is a naive approximation of tokens + return Math.max(1, words.length) +} + +/** + * estimateLLMCost() + * ----------------- + * Estimates the cost for an LLM-based model by: + * 1. Reading a combined prompt + transcript file + * 2. Approximating the token usage + * 3. Looking up cost info from the LLM model config + * 4. Logging the estimated cost to the console + * + * @param {ProcessingOptions} options - The command-line options (must include `llmCost` file path) + * @param {LLMServices} llmService - The selected LLM service (e.g., 'chatgpt', 'ollama', 'claude', etc.) + * @returns {Promise} A promise that resolves when cost estimation is complete + */ +export async function estimateLLMCost( + options: ProcessingOptions, + llmService: LLMServices +): Promise { + const filePath = options.llmCost + if (!filePath) { + throw new Error('No file path provided to estimate LLM cost.') + } + + l.dim(`\nEstimating LLM cost for '${llmService}' with file: ${filePath}`) + + try { + // Read content from file + const content = await readFile(filePath, 'utf8') + const tokenCount = approximateTokens(content) + + /** + * Determine if the user provided a specific model string (e.g. "--chatgpt GPT_4o"), + * otherwise fallback to a default model if only "--chatgpt" was used. + */ + let userModel = typeof options[llmService] === 'string' + ? options[llmService] as string + : undefined + + // Provide default fallback for ChatGPT if no string model was given + if (llmService === 'chatgpt' && (userModel === undefined || userModel === 'true')) { + userModel = 'GPT_4o_MINI' + } + + // If still nothing is set, use the service name as a last resort + const modelName = userModel || llmService + + // Log cost using the same function that logs LLM usage after real calls + logLLMCost({ + modelName, + stopReason: 'n/a', + tokenUsage: { + input: tokenCount, + output: 4000, + total: tokenCount + } + }) + + } catch (error) { + err(`Error estimating LLM cost: ${(error as Error).message}`) + throw error + } +} /* ------------------------------------------------------------------ * LLM Services & Model Configurations @@ -515,4 +699,124 @@ export const ALL_MODELS: { [key: string]: ModelConfigValue } = { ...FIREWORKS_MODELS, ...TOGETHER_MODELS, ...GROQ_MODELS, +} + +/** + * checkOllamaServerAndModel() + * --------------------- + * Checks if the Ollama server is running, attempts to start it if not, + * and ensures the specified model is available (pulling if needed). + * + * @param {string} ollamaHost - The Ollama host + * @param {string} ollamaPort - The Ollama port + * @param {string} ollamaModelName - The Ollama model name (e.g. 'qwen2.5:0.5b') + * @returns {Promise} + */ +export async function checkOllamaServerAndModel( + ollamaHost: string, + ollamaPort: string, + ollamaModelName: string +): Promise { + // Helper to check if the Ollama server responds + async function checkServer(): Promise { + try { + const serverResponse = await fetch(`http://${ollamaHost}:${ollamaPort}`) + return serverResponse.ok + } catch (error) { + return false + } + } + + l.dim(`[checkOllamaServerAndModel] Checking server: http://${ollamaHost}:${ollamaPort}`) + + // 1) Confirm the server is running + if (await checkServer()) { + l.dim('\n Ollama server is already running...') + } else { + // If the Docker-based environment uses 'ollama' as hostname but it's not up, that's likely an error + if (ollamaHost === 'ollama') { + throw new Error('Ollama server is not running. Please ensure the Ollama server is running and accessible.') + } else { + // Attempt to spawn an Ollama server locally + l.dim('\n Ollama server is not running. Attempting to start it locally...') + const ollamaProcess = spawn('ollama', ['serve'], { + detached: true, + stdio: 'ignore', + }) + ollamaProcess.unref() + + // Wait up to ~30 seconds for the server to respond + let attempts = 0 + while (attempts < 30) { + if (await checkServer()) { + l.dim(' - Ollama server is now ready.\n') + break + } + await new Promise((resolve) => setTimeout(resolve, 1000)) + attempts++ + } + if (attempts === 30) { + throw new Error('Ollama server failed to become ready in time.') + } + } + } + + // 2) Confirm the model is available; if not, pull it + l.dim(` Checking if model is available: ${ollamaModelName}`) + try { + const tagsResponse = await fetch(`http://${ollamaHost}:${ollamaPort}/api/tags`) + if (!tagsResponse.ok) { + throw new Error(`HTTP error! status: ${tagsResponse.status}`) + } + + const tagsData = (await tagsResponse.json()) as OllamaTagsResponse + const isModelAvailable = tagsData.models.some((m) => m.name === ollamaModelName) + l.dim(`[checkOllamaServerAndModel] isModelAvailable=${isModelAvailable}`) + + if (!isModelAvailable) { + l.dim(`\n Model ${ollamaModelName} is NOT available; pulling now...`) + const pullResponse = await fetch(`http://${ollamaHost}:${ollamaPort}/api/pull`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: ollamaModelName }), + }) + if (!pullResponse.ok) { + throw new Error(`Failed to initiate pull for model ${ollamaModelName}`) + } + if (!pullResponse.body) { + throw new Error('Response body is null while pulling model.') + } + + const reader = pullResponse.body.getReader() + const decoder = new TextDecoder() + + // Stream the JSON lines from the server + while (true) { + const { done, value } = await reader.read() + if (done) break + + const chunk = decoder.decode(value) + const lines = chunk.split('\n') + for (const line of lines) { + if (line.trim() === '') continue + + // Each line should be a JSON object from the Ollama server + try { + const parsedLine = JSON.parse(line) + if (parsedLine.status === 'success') { + l.dim(` - Model ${ollamaModelName} pulled successfully.\n`) + break + } + } catch (parseError) { + err(`Error parsing JSON while pulling model: ${parseError}`) + } + } + } + } else { + l.dim(`\n Model ${ollamaModelName} is already available.\n`) + } + } catch (error) { + err(`Error checking/pulling model: ${(error as Error).message}`) + throw error + } } \ No newline at end of file diff --git a/src/utils/logging.ts b/src/utils/logging.ts index 728fcc1..b17fdba 100644 --- a/src/utils/logging.ts +++ b/src/utils/logging.ts @@ -1,12 +1,9 @@ // src/utils/logging.ts -import { execPromise } from './globals/process' -import { ALL_MODELS } from './globals/llms' import chalk from 'chalk' -import type { ProcessingOptions, SeparatorParams } from './types/process' -import type { TranscriptionCostInfo } from './types/transcription' -import type { LogLLMCost, ChainableLogger } from './types/logging' +import type { ChainableLogger } from './types/logging' +import type { SeparatorParams } from './types/process' /** * Logs the first step of a top-level function call with its relevant options or parameters. @@ -15,12 +12,13 @@ import type { LogLLMCost, ChainableLogger } from './types/logging' * @param details - An object containing relevant parameters to log */ export function logInitialFunctionCall(functionName: string, details: Record): void { - l.info(`${functionName} called with the following arguments:`) + l.opts(`${functionName} called with the following arguments:\n`) for (const [key, value] of Object.entries(details)) { if (typeof value === 'object' && value !== null) { - l.opts(` - ${key}: ${JSON.stringify(value, null, 2)}`) + l.opts(`${key}:\n`) + l.opts(`${JSON.stringify(value, null, 2)}`) } else { - l.opts(` - ${key}: ${value}`) + l.opts(`${key}: ${value}`) } } l.opts('') @@ -41,19 +39,19 @@ export function logSeparator(params: SeparatorParams): void { case 'channel': case 'playlist': case 'urls': - l.opts(`\n================================================================================================`) + l.final(`\n================================================================================================`) if (params.type === 'urls') { - l.opts(` Processing URL ${params.index + 1}/${params.total}: ${params.descriptor}`) + l.final(` Processing URL ${params.index + 1}/${params.total}: ${params.descriptor}`) } else { - l.opts(` Processing video ${params.index + 1}/${params.total}: ${params.descriptor}`) + l.final(` Processing video ${params.index + 1}/${params.total}: ${params.descriptor}`) } - l.opts(`================================================================================================\n`) + l.final(`================================================================================================\n`) break case 'rss': - l.opts(`\n========================================================================================`) - l.opts(` Item ${params.index + 1}/${params.total} processing: ${params.descriptor}`) - l.opts(`========================================================================================\n`) + l.final(`\n========================================================================================`) + l.final(` Item ${params.index + 1}/${params.total} processing: ${params.descriptor}`) + l.final(`========================================================================================\n`) break case 'completion': @@ -64,130 +62,6 @@ export function logSeparator(params: SeparatorParams): void { } } -/** - * Finds the model configuration based on the model key - * @param modelKey - The key/name of the model (e.g., 'LLAMA_3_2_3B') - * @returns The model configuration if found, undefined otherwise - */ -function findModelConfig(modelKey: string) { - // First try to find the model directly in our combined models - const model = ALL_MODELS[modelKey] - if (model) return model - - // If not found by key, try matching by model ID as a fallback - return Object.values(ALL_MODELS).find(model => - model.modelId.toLowerCase() === modelKey.toLowerCase() - ) -} - -/** - * Formats a cost value to a standardized string representation - * @param cost - The cost value to format - * @returns Formatted cost string - */ -function formatCost(cost: number | undefined): string { - if (cost === undefined) return 'N/A' - if (cost === 0) return '$0.0000' - return `$${cost.toFixed(4)}` -} - -/** - * Logs API call results in a standardized format across different LLM providers. - * Includes token usage and cost calculations. - * @param info - Object containing model info, stop reason, and token usage - */ -export function logLLMCost(info: LogLLMCost): void { - const { modelName, stopReason, tokenUsage } = info - - // Get model display name if available, otherwise use the provided name - const modelConfig = findModelConfig(modelName) - const displayName = modelConfig?.name ?? modelName - - // Log stop/finish reason and model - l.wait(` - ${stopReason ? `${stopReason} Reason` : 'Status'}: ${stopReason}\n - Model: ${displayName}`) - - // Format token usage string based on available data - const tokenLines = [] - if (tokenUsage.input) tokenLines.push(`${tokenUsage.input} input tokens`) - if (tokenUsage.output) tokenLines.push(`${tokenUsage.output} output tokens`) - if (tokenUsage.total) tokenLines.push(`${tokenUsage.total} total tokens`) - - // Log token usage if any data is available - if (tokenLines.length > 0) { - l.wait(` - Token Usage:\n - ${tokenLines.join('\n - ')}`) - } - - // Calculate and log costs - let inputCost: number | undefined - let outputCost: number | undefined - let totalCost: number | undefined - - // Check if model config is found - if (!modelConfig) { - console.warn(`Warning: Could not find cost configuration for model: ${modelName}`) - } else if (modelConfig.inputCostPer1M === 0 && modelConfig.outputCostPer1M === 0) { - // If both costs per million are zero, return all zeros - inputCost = 0 - outputCost = 0 - totalCost = 0 - } else { - // Calculate costs if token usage is available - if (tokenUsage.input) { - const rawInputCost = (tokenUsage.input / 1_000_000) * modelConfig.inputCostPer1M - inputCost = Math.abs(rawInputCost) < 0.00001 ? 0 : rawInputCost - } - - if (tokenUsage.output) { - outputCost = (tokenUsage.output / 1_000_000) * modelConfig.outputCostPer1M - } - - // Calculate total cost only if both input and output costs are available - if (inputCost !== undefined && outputCost !== undefined) { - totalCost = inputCost + outputCost - } - } - - const costLines = [] - - if (inputCost !== undefined) { - costLines.push(`Input cost: ${formatCost(inputCost)}`) - } - if (outputCost !== undefined) { - costLines.push(`Output cost: ${formatCost(outputCost)}`) - } - if (totalCost !== undefined) { - costLines.push(`Total cost: ${chalk.bold(formatCost(totalCost))}`) - } - - // Log costs if any calculations were successful - if (costLines.length > 0) { - l.wait(` - Cost Breakdown:\n - ${costLines.join('\n - ')}`) - } -} - -/** - * Asynchronously logs the estimated transcription cost based on audio duration and per-minute cost. - * Internally calculates the audio file duration using ffprobe. - * @param info - Object containing the model name, cost per minute, and path to the audio file. - * @throws {Error} If ffprobe fails or returns invalid data. - */ -export async function logTranscriptionCost(info: TranscriptionCostInfo): Promise { - const cmd = `ffprobe -v error -show_entries format=duration -of csv=p=0 "${info.filePath}"` - const { stdout } = await execPromise(cmd) - const seconds = parseFloat(stdout.trim()) - if (isNaN(seconds)) { - throw new Error(`Could not parse audio duration for file: ${info.filePath}`) - } - const minutes = seconds / 60 - const cost = info.costPerMinute * minutes - - l.wait( - ` - Estimated Transcription Cost for ${info.modelName}:\n` + - ` - Audio Length: ${minutes.toFixed(2)} minutes\n` + - ` - Cost: $${cost.toFixed(4)}` - ) -} - /** * Creates a chainable logger function that maintains both function call and method syntax. * @@ -238,52 +112,4 @@ function createChainableErrorLogger(): ChainableLogger { // Create and export the chainable loggers export const l = createChainableLogger() -export const err = createChainableErrorLogger() - -/** - * Logs the processing status and item counts for RSS feeds. - * - * @param total - Total number of RSS items found. - * @param processing - Number of RSS items to process. - * @param options - Configuration options. - */ -export function logRSSProcessingStatus( - total: number, - processing: number, - options: ProcessingOptions -): void { - if (options.item && options.item.length > 0) { - l.wait(`\n - Found ${total} items in the RSS feed.`) - l.wait(` - Processing ${processing} specified items.`) - } else if (options.last) { - l.wait(`\n - Found ${total} items in the RSS feed.`) - l.wait(` - Processing the last ${options.last} items.`) - } else { - l.wait(`\n - Found ${total} item(s) in the RSS feed.`) - l.wait(` - Processing ${processing} item(s) after skipping ${options.skip || 0}.\n`) - } -} - -/** - * Logs the processing status and video counts for channel downloads. - * - * @param total - Total number of videos found. - * @param processing - Number of videos to process. - * @param options - Configuration options. - */ -export function logChannelProcessingStatus( - total: number, - processing: number, - options: ProcessingOptions -): void { - if (options.last) { - l.wait(`\n - Found ${total} videos in the channel.`) - l.wait(` - Processing the last ${processing} videos.`) - } else if (options.skip) { - l.wait(`\n - Found ${total} videos in the channel.`) - l.wait(` - Processing ${processing} videos after skipping ${options.skip || 0}.\n`) - } else { - l.wait(`\n - Found ${total} videos in the channel.`) - l.wait(` - Processing all ${processing} videos.\n`) - } -} \ No newline at end of file +export const err = createChainableErrorLogger() \ No newline at end of file diff --git a/src/utils/retry.ts b/src/utils/retry.ts index 3897f00..eefc5ef 100644 --- a/src/utils/retry.ts +++ b/src/utils/retry.ts @@ -1,7 +1,7 @@ // src/utils/retry.ts import { l, err } from './logging' -import { execFilePromise } from './globals/process' +import { execFilePromise } from './validate-option' /** * Executes a command with retry logic to recover from transient failures. @@ -35,7 +35,7 @@ export async function executeWithRetry( // Exponential backoff: Wait before trying again const delayMs = 1000 * 2 ** (attempt - 1) // 1s, 2s, 4s, ... - l.wait( + l.dim( `Retry ${attempt} of ${retries} failed. Waiting ${delayMs} ms before next attempt...` ) await new Promise((resolve) => setTimeout(resolve, delayMs)) @@ -61,9 +61,9 @@ export async function retryLLMCall( while (attempt < maxRetries) { try { attempt++ - l.wait(` Attempt ${attempt} - Processing LLM call...\n`) + l.dim(` Attempt ${attempt} - Processing LLM call...\n`) await fn() - l.wait(`\n LLM call completed successfully on attempt ${attempt}.`) + l.dim(`\n LLM call completed successfully on attempt ${attempt}.`) return } catch (error) { err(` Attempt ${attempt} failed: ${(error as Error).message}`) @@ -71,7 +71,7 @@ export async function retryLLMCall( err(` Max retries (${maxRetries}) reached. Aborting LLM processing.`) throw error } - l.wait(` Retrying in ${delayBetweenRetries / 1000} seconds...`) + l.dim(` Retrying in ${delayBetweenRetries / 1000} seconds...`) await new Promise((resolve) => setTimeout(resolve, delayBetweenRetries)) } } @@ -95,9 +95,8 @@ export async function retryTranscriptionCall( while (attempt < maxRetries) { try { attempt++ - l.wait(` Attempt ${attempt} - Processing Transcription call...\n`) const transcript = await fn() - l.wait(`\n Transcription call completed successfully on attempt ${attempt}.`) + l.dim(` Transcription call completed successfully on attempt ${attempt}.`) return transcript } catch (error) { err(` Attempt ${attempt} failed: ${(error as Error).message}`) @@ -105,7 +104,7 @@ export async function retryTranscriptionCall( err(` Max retries (${maxRetries}) reached. Aborting transcription.`) throw error } - l.wait(` Retrying in ${delayBetweenRetries / 1000} seconds...`) + l.dim(` Retrying in ${delayBetweenRetries / 1000} seconds...`) await new Promise((resolve) => setTimeout(resolve, delayBetweenRetries)) } } @@ -132,9 +131,9 @@ export async function retryRSSFetch( while (attempt < maxRetries) { try { attempt++ - l.wait(` Attempt ${attempt} - Fetching RSS...\n`) + l.dim(` Attempt ${attempt} - Fetching RSS...\n`) const response = await fn() - l.wait(`\n RSS fetch succeeded on attempt ${attempt}.`) + l.dim(`\n RSS fetch succeeded on attempt ${attempt}.`) return response } catch (error) { err(` Attempt ${attempt} failed: ${(error as Error).message}`) @@ -142,7 +141,7 @@ export async function retryRSSFetch( err(` Max retries (${maxRetries}) reached. Aborting RSS fetch.`) throw error } - l.wait(` Retrying in ${delayBetweenRetries / 1000} seconds...`) + l.dim(` Retrying in ${delayBetweenRetries / 1000} seconds...`) await new Promise((resolve) => setTimeout(resolve, delayBetweenRetries)) } } diff --git a/src/utils/rss-utils.ts b/src/utils/rss-utils.ts new file mode 100644 index 0000000..eb9a2b0 --- /dev/null +++ b/src/utils/rss-utils.ts @@ -0,0 +1,212 @@ +// src/utils/rss-utils.ts + +import { exec, execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { l, err } from '../utils/logging' + +import type { TranscriptServices } from './types/transcription' +import type { LLMServices } from './types/llms' +import type { ProcessingOptions, RSSItem, HandlerFunction } from './types/process' + +export const execPromise = promisify(exec) +export const execFilePromise = promisify(execFile) + +/** + * Validates RSS flags (e.g., --last, --skip, --order, --date, --lastDays) without requiring feed data. + * + * @param options - The command-line options provided by the user + * @throws Exits the process if any flag is invalid + */ +export function validateRSSOptions(options: ProcessingOptions): void { + if (options.last !== undefined) { + if (!Number.isInteger(options.last) || options.last < 1) { + err('Error: The --last option must be a positive integer.') + process.exit(1) + } + if (options.skip !== undefined || options.order !== undefined) { + err('Error: The --last option cannot be used with --skip or --order.') + process.exit(1) + } + } + + if (options.skip !== undefined && (!Number.isInteger(options.skip) || options.skip < 0)) { + err('Error: The --skip option must be a non-negative integer.') + process.exit(1) + } + + if (options.order !== undefined && !['newest', 'oldest'].includes(options.order)) { + err("Error: The --order option must be either 'newest' or 'oldest'.") + process.exit(1) + } + + if (options.lastDays !== undefined) { + if (!Number.isInteger(options.lastDays) || options.lastDays < 1) { + err('Error: The --lastDays option must be a positive integer.') + process.exit(1) + } + if ( + options.last !== undefined || + options.skip !== undefined || + options.order !== undefined || + (options.date && options.date.length > 0) + ) { + err('Error: The --lastDays option cannot be used with --last, --skip, --order, or --date.') + process.exit(1) + } + } + + if (options.date && options.date.length > 0) { + const dateRegex = /^\d{4}-\d{2}-\d{2}$/ + for (const d of options.date) { + if (!dateRegex.test(d)) { + err(`Error: Invalid date format "${d}". Please use YYYY-MM-DD format.`) + process.exit(1) + } + } + + if ( + options.last !== undefined || + options.skip !== undefined || + options.order !== undefined + ) { + err('Error: The --date option cannot be used with --last, --skip, or --order.') + process.exit(1) + } + } +} + +/** + * Filters RSS feed items based on user-supplied options (e.g., item URLs, date ranges, etc.). + * + * @param options - Configuration options to filter the feed items + * @param feedItemsArray - Parsed array of RSS feed items (raw JSON from XML parser) + * @param channelTitle - Title of the RSS channel (optional) + * @param channelLink - URL to the RSS channel (optional) + * @param channelImage - A fallback channel image URL (optional) + * @returns Filtered RSS items based on the provided options + */ +export async function filterRSSItems( + options: ProcessingOptions, + feedItemsArray?: any, + channelTitle?: string, + channelLink?: string, + channelImage?: string +): Promise { + const defaultDate = new Date().toISOString().substring(0, 10) + const unfilteredItems: RSSItem[] = (feedItemsArray || []) + .filter((item: any) => { + if (!item.enclosure || !item.enclosure.type) return false + const audioVideoTypes = ['audio/', 'video/'] + return audioVideoTypes.some((type) => item.enclosure.type.startsWith(type)) + }) + .map((item: any) => { + let publishDate: string + try { + const date = item.pubDate ? new Date(item.pubDate) : new Date() + publishDate = date.toISOString().substring(0, 10) + } catch { + publishDate = defaultDate + } + + return { + showLink: item.enclosure?.url || '', + channel: channelTitle || '', + channelURL: channelLink || '', + title: item.title || '', + description: '', + publishDate, + coverImage: item['itunes:image']?.href || channelImage || '', + } + }) + + let itemsToProcess: RSSItem[] = [] + + if (options.item && options.item.length > 0) { + itemsToProcess = unfilteredItems.filter((it) => + options.item!.includes(it.showLink) + ) + } else if (options.lastDays !== undefined) { + const now = new Date() + const cutoff = new Date(now.getTime() - options.lastDays * 24 * 60 * 60 * 1000) + + itemsToProcess = unfilteredItems.filter((it) => { + const itDate = new Date(it.publishDate) + return itDate >= cutoff + }) + } else if (options.date && options.date.length > 0) { + const selectedDates = new Set(options.date) + itemsToProcess = unfilteredItems.filter((it) => + selectedDates.has(it.publishDate) + ) + } else if (options.last) { + itemsToProcess = unfilteredItems.slice(0, options.last) + } else { + const sortedItems = + options.order === 'oldest' + ? unfilteredItems.slice().reverse() + : unfilteredItems + itemsToProcess = sortedItems.slice(options.skip || 0) + } + + return itemsToProcess +} + +/** + * A helper function that validates RSS action input and processes it if valid. + * Separately validates flags with {@link validateRSSOptions} and leaves feed-item filtering to {@link filterRSSItems}. + * + * @param options - The ProcessingOptions containing RSS feed details + * @param handler - The function to handle each RSS feed + * @param llmServices - The optional LLM service for processing + * @param transcriptServices - The chosen transcription service + * @throws An error if no valid RSS URLs are provided + * @returns A promise that resolves when all RSS feeds have been processed + */ +export async function validateRSSAction( + options: ProcessingOptions, + handler: HandlerFunction, + llmServices?: LLMServices, + transcriptServices?: TranscriptServices +): Promise { + if (options.item && !Array.isArray(options.item)) { + options.item = [options.item] + } + if (typeof options.rss === 'string') { + options.rss = [options.rss] + } + + validateRSSOptions(options) + + const rssUrls = options.rss + if (!rssUrls || rssUrls.length === 0) { + throw new Error(`No valid RSS URLs provided for processing`) + } + + for (const rssUrl of rssUrls) { + await handler(options, rssUrl, llmServices, transcriptServices) + } +} + +/** + * Logs the processing status and item counts for RSS feeds. + * + * @param total - Total number of RSS items found. + * @param processing - Number of RSS items to process. + * @param options - Configuration options. + */ +export function logRSSProcessingStatus( + total: number, + processing: number, + options: ProcessingOptions +): void { + if (options.item && options.item.length > 0) { + l.dim(`\n - Found ${total} items in the RSS feed.`) + l.dim(` - Processing ${processing} specified items.`) + } else if (options.last) { + l.dim(`\n - Found ${total} items in the RSS feed.`) + l.dim(` - Processing the last ${options.last} items.`) + } else { + l.dim(`\n - Found ${total} item(s) in the RSS feed.`) + l.dim(` - Processing ${processing} item(s) after skipping ${options.skip || 0}.\n`) + } +} \ No newline at end of file diff --git a/src/utils/transcription-utils.ts b/src/utils/transcription-utils.ts new file mode 100644 index 0000000..cb8051f --- /dev/null +++ b/src/utils/transcription-utils.ts @@ -0,0 +1,477 @@ +// src/utils/transcription-globals.ts + +/** + * @file Defines Deepgram and Assembly transcription model configurations, + * including name, modelId, and cost per minute. Also provides + * Whisper model mappings for whisper.cpp usage. + */ + +import { existsSync } from 'node:fs' +import { execPromise } from './validate-option' +import { l, err } from './logging' +import type { ProcessingOptions } from './types/process' +import type { TranscriptServices, TranscriptionCostInfo, WhisperModelType, TranscriptServiceConfig, DeepgramModelType, AssemblyModelType } from './types/transcription' + +import type { + AssemblyAIPollingResponse, + AssemblyAIUtterance, + AssemblyAIWord +} from '../utils/types/transcription' + +/** + * Formats the Deepgram transcript by adding timestamps and newlines based on conditions. + * Rules: + * - Add a timestamp if it's the first word, every 30th word, or the start of a sentence (capitalized word). + * - Insert a newline if the word ends a sentence (ends in punctuation), every 30th word, or it's the last word. + * + * @param words - The array of word objects from Deepgram (each contains a 'word' and 'start' time) + * @returns A formatted transcript string with timestamps and newlines + */ +export function formatDeepgramTranscript(words: Array<{ word: string; start: number }>): string { + return words.reduce((acc, { word, start }, i, arr) => { + const addTimestamp = (i % 30 === 0 || /^[A-Z]/.test(word)) + let timestamp = '' + if (addTimestamp) { + const minutes = Math.floor(start / 60).toString().padStart(2, '0') + const seconds = Math.floor(start % 60).toString().padStart(2, '0') + timestamp = `[${minutes}:${seconds}] ` + } + + const endOfSentence = /[.!?]$/.test(word) + const endOfBlock = (i % 30 === 29 || i === arr.length - 1) + const newline = (endOfSentence || endOfBlock) ? '\n' : '' + + return `${acc}${timestamp}${word} ${newline}` + }, '') +} + +/** + * Formats the AssemblyAI transcript into text with timestamps and optional speaker labels. + * Logic: + * - If transcript.utterances are present, format each utterance line with optional speaker labels and timestamps. + * - If only transcript.words are available, group them into lines ~80 chars, prepend each line with a timestamp. + * - If no structured data is available, use the raw transcript text or 'No transcription available.' as fallback. + * + * @param transcript - The polling response from AssemblyAI after transcription completes + * @param speakerLabels - Whether to include speaker labels in the output + * @returns The fully formatted transcript as a string + * @throws If words are expected but not found (no content to format) + */ +export function formatAssemblyTranscript(transcript: AssemblyAIPollingResponse, speakerLabels: boolean): string { + // Helper inline formatting function for timestamps (AssemblyAI returns ms) + const inlineFormatTime = (timestamp: number): string => { + const totalSeconds = Math.floor(timestamp / 1000) + const minutes = Math.floor(totalSeconds / 60).toString().padStart(2, '0') + const seconds = (totalSeconds % 60).toString().padStart(2, '0') + return `${minutes}:${seconds}` + } + + let txtContent = '' + + if (transcript.utterances && transcript.utterances.length > 0) { + // If utterances are available, format each line with optional speaker labels and timestamps + txtContent = transcript.utterances.map((utt: AssemblyAIUtterance) => + `${speakerLabels ? `Speaker ${utt.speaker} ` : ''}(${inlineFormatTime(utt.start)}): ${utt.text}` + ).join('\n') + } else if (transcript.words && transcript.words.length > 0) { + // If only words are available, we must form lines with timestamps every ~80 chars + const firstWord = transcript.words[0] + if (!firstWord) { + throw new Error('No words found in transcript') + } + + let currentLine = '' + let currentTimestamp = inlineFormatTime(firstWord.start) + + transcript.words.forEach((word: AssemblyAIWord) => { + if (currentLine.length + word.text.length > 80) { + // Start a new line if the current line exceeds ~80 characters + txtContent += `[${currentTimestamp}] ${currentLine.trim()}\n` + currentLine = '' + currentTimestamp = inlineFormatTime(word.start) + } + currentLine += `${word.text} ` + }) + + // Add any remaining text as a final line + if (currentLine.length > 0) { + txtContent += `[${currentTimestamp}] ${currentLine.trim()}\n` + } + } else { + // If no utterances or words, fallback to transcript.text or a default message + txtContent = transcript.text || 'No transcription available.' + } + + return txtContent +} + +/** + * Converts LRC content (common lyrics file format) to plain text with timestamps. + * - Strips out lines that contain certain metadata (like [by:whisper.cpp]). + * - Converts original timestamps [MM:SS.xx] to a simplified [MM:SS] format. + * - Properly extracts all timestamps in each line, then merges them into + * chunks of up to 15 words, adopting the newest timestamp as soon + * as it appears. + * + * @param lrcContent - The content of the LRC file as a string + * @returns The converted text content with simple timestamps + */ +export function formatWhisperTranscript(lrcContent: string): string { + // 1) Remove lines like `[by:whisper.cpp]`, convert "[MM:SS.xx]" to "[MM:SS]" + const rawLines = lrcContent + .split('\n') + .filter(line => !line.startsWith('[by:whisper.cpp]')) + .map(line => + line.replace( + /\[(\d{1,3}):(\d{2})(\.\d+)?\]/g, + (_, minutes, seconds) => `[${minutes}:${seconds}]` + ) + ) + + // We define a Segment with timestamp: string | undefined + type Segment = { + timestamp: string | undefined + words: string[] + } + + /** + * Given a line (which may contain multiple [MM:SS] tags), + * extract those timestamps + the words in between. + */ + function parseLineIntoSegments(line: string): Segment[] { + const segments: Segment[] = [] + const pattern = /\[(\d{1,3}:\d{2})\]/g + + let lastIndex = 0 + let match: RegExpExecArray | null + let currentTimestamp: string | undefined = undefined + + while ((match = pattern.exec(line)) !== null) { + // Text before this timestamp + const textBeforeThisTimestamp = line.slice(lastIndex, match.index).trim() + if (textBeforeThisTimestamp) { + segments.push({ + timestamp: currentTimestamp, + words: textBeforeThisTimestamp.split(/\s+/).filter(Boolean), + }) + } + // Update timestamp to the newly found one + currentTimestamp = match[1] + lastIndex = pattern.lastIndex + } + + // After the last timestamp, grab any trailing text + const trailing = line.slice(lastIndex).trim() + if (trailing) { + segments.push({ + timestamp: currentTimestamp, + words: trailing.split(/\s+/).filter(Boolean), + }) + } + + // If line had no timestamps, the entire line is one segment with `timestamp: undefined`. + return segments + } + + // 2) Flatten all lines into an array of typed segments + const allSegments: Segment[] = rawLines.flatMap(line => parseLineIntoSegments(line)) + + // 3) Accumulate words into lines up to 15 words each. + // Whenever we see a new timestamp, we finalize the previous chunk + // and start a new chunk with that timestamp. + const finalLines: string[] = [] + let currentTimestamp: string | undefined = undefined + let currentWords: string[] = [] + + function finalizeChunk() { + if (currentWords.length > 0) { + // If we have never encountered a timestamp, default to "00:00" + const tsToUse = currentTimestamp ?? '00:00' + finalLines.push(`[${tsToUse}] ${currentWords.join(' ')}`) + currentWords = [] + } + } + + for (const segment of allSegments) { + // If this segment has a new timestamp, finalize the old chunk and start new + if (segment.timestamp !== undefined) { + finalizeChunk() + currentTimestamp = segment.timestamp + } + + // Accumulate words from this segment, chunking at 15 + for (const word of segment.words) { + currentWords.push(word) + if (currentWords.length === 15) { + finalizeChunk() + } + } + } + + // 4) Finalize any leftover words + finalizeChunk() + + // 5) Return as simple text + return finalLines.join('\n') +} + +/** + * Asynchronously logs the estimated transcription cost based on audio duration and per-minute cost. + * Internally calculates the audio file duration using ffprobe. + * @param info - Object containing the model name, cost per minute, and path to the audio file. + * @throws {Error} If ffprobe fails or returns invalid data. + */ +export async function logTranscriptionCost(info: TranscriptionCostInfo): Promise { + const cmd = `ffprobe -v error -show_entries format=duration -of csv=p=0 "${info.filePath}"` + const { stdout } = await execPromise(cmd) + const seconds = parseFloat(stdout.trim()) + if (isNaN(seconds)) { + throw new Error(`Could not parse audio duration for file: ${info.filePath}`) + } + const minutes = seconds / 60 + const cost = info.costPerMinute * minutes + + l.dim( + ` - Estimated Transcription Cost for ${info.modelName}:\n` + + ` - Audio Length: ${minutes.toFixed(2)} minutes\n` + + ` - Cost: $${cost.toFixed(4)}` + ) +} + +/** + * Estimates transcription cost for the provided file and chosen transcription service. + * + * @param {ProcessingOptions} options - The command-line options (must include `transcriptCost` file path). + * @param {TranscriptServices} transcriptServices - The selected transcription service (e.g., "deepgram", "assembly", "whisper"). + * @returns {Promise} A promise that resolves when cost estimation is complete. + */ +export async function estimateTranscriptCost( + options: ProcessingOptions, + transcriptServices: TranscriptServices +): Promise { + const filePath = options.transcriptCost + if (!filePath) { + throw new Error('No file path provided to estimate transcription cost.') + } + + switch (transcriptServices) { + case 'deepgram': { + const deepgramModel = typeof options.deepgram === 'string' ? options.deepgram : 'NOVA_2' + const modelInfo = DEEPGRAM_MODELS[deepgramModel as DeepgramModelType] || DEEPGRAM_MODELS.NOVA_2 + await logTranscriptionCost({ + modelName: modelInfo.name, + costPerMinute: modelInfo.costPerMinute, + filePath + }) + break + } + case 'assembly': { + const assemblyModel = typeof options.assembly === 'string' ? options.assembly : 'NANO' + const modelInfo = ASSEMBLY_MODELS[assemblyModel as AssemblyModelType] || ASSEMBLY_MODELS.NANO + await logTranscriptionCost({ + modelName: modelInfo.name, + costPerMinute: modelInfo.costPerMinute, + filePath + }) + break + } + case 'whisper': { + // Currently, no official cost data for Whisper.cpp + l.wait('\nNo cost data available for Whisper.\n') + break + } + default: { + throw new Error(`Unsupported transcription service for cost estimation: ${transcriptServices}`) + } + } +} + +/* ------------------------------------------------------------------ + * Transcription Services & Models + * ------------------------------------------------------------------ */ + +/** + * Available transcription services and their configuration. + */ +export const TRANSCRIPT_SERVICES: Record = { + WHISPER: { name: 'Whisper.cpp', value: 'whisper', isWhisper: true }, + DEEPGRAM: { name: 'Deepgram', value: 'deepgram' }, + ASSEMBLY: { name: 'AssemblyAI', value: 'assembly' }, +} as const + +/** + * Array of valid transcription service values. + */ +export const TRANSCRIPT_OPTIONS: string[] = Object.values(TRANSCRIPT_SERVICES) + .map((service) => service.value) + +/** + * Whisper-only transcription services (subset of TRANSCRIPT_SERVICES). + */ +export const WHISPER_SERVICES: string[] = Object.values(TRANSCRIPT_SERVICES) + .filter( + ( + service + ): service is TranscriptServiceConfig & { + isWhisper: true + } => service.isWhisper === true + ) + .map((service) => service.value) + +/** + * Mapping of Whisper model flags (`--whisper=`) to the actual + * ggml binary filenames for whisper.cpp. + */ +export const WHISPER_MODELS: Record = { + // Tiny models + tiny: 'ggml-tiny.bin', + 'tiny.en': 'ggml-tiny.en.bin', + + // Base models + base: 'ggml-base.bin', + 'base.en': 'ggml-base.en.bin', + + // Small/Medium + small: 'ggml-small.bin', + 'small.en': 'ggml-small.en.bin', + medium: 'ggml-medium.bin', + 'medium.en': 'ggml-medium.en.bin', + + // Large variations + 'large-v1': 'ggml-large-v1.bin', + 'large-v2': 'ggml-large-v2.bin', + + // Add or rename as needed: + 'large-v3-turbo': 'ggml-large-v3-turbo.bin', + // Provide an alias if you like shorter flags: + turbo: 'ggml-large-v3-turbo.bin', +} + +/** + * Deepgram models with their per-minute cost. + */ +export const DEEPGRAM_MODELS: Record< + DeepgramModelType, + { name: string; modelId: string; costPerMinute: number } +> = { + NOVA_2: { + name: 'Nova-2', + modelId: 'nova-2', + costPerMinute: 0.0043, + }, + NOVA: { + name: 'Nova', + modelId: 'nova', + costPerMinute: 0.0043, + }, + ENHANCED: { + name: 'Enhanced', + modelId: 'enhanced', + costPerMinute: 0.0145, + }, + BASE: { + name: 'Base', + modelId: 'base', + costPerMinute: 0.0125, + }, +} + +/** + * AssemblyAI models with their per-minute cost. + */ +export const ASSEMBLY_MODELS: Record< + AssemblyModelType, + { name: string; modelId: string; costPerMinute: number } +> = { + BEST: { + name: 'Best', + modelId: 'best', + costPerMinute: 0.0062, + }, + NANO: { + name: 'Nano', + modelId: 'nano', + costPerMinute: 0.002, + }, +} + + +/** + * Checks if whisper.cpp directory exists and, if missing, clones and compiles it. + * Also checks if the chosen model file is present and, if missing, downloads it. + * @param whisperModel - The requested Whisper model name (e.g. "turbo" or "large-v3-turbo") + * @param modelGGMLName - The corresponding GGML model filename (e.g. "ggml-large-v3-turbo.bin") + */ +export async function checkWhisperDirAndModel( + whisperModel: string, + modelGGMLName: string +): Promise { + // OPTIONAL: If you want to handle "turbo" as an alias for "large-v3-turbo" + // so the user can do --whisper=turbo but the script sees "large-v3-turbo". + if (whisperModel === 'turbo') { + whisperModel = 'large-v3-turbo' + } + + // Double-check that the requested model is actually in WHISPER_MODELS, + // to avoid passing an unrecognized name to download-ggml-model.sh + if (!Object.prototype.hasOwnProperty.call(WHISPER_MODELS, whisperModel)) { + throw new Error( + `Unknown Whisper model "${whisperModel}". ` + + `Please use one of: ${Object.keys(WHISPER_MODELS).join(', ')}` + ) + } + + // 1. Ensure whisper.cpp is cloned and built + if (!existsSync('./whisper.cpp')) { + l.dim(`\n No whisper.cpp repo found, cloning and compiling...\n`) + try { + await execPromise( + 'git clone https://github.com/ggerganov/whisper.cpp.git ' + + '&& cmake -B whisper.cpp/build -S whisper.cpp ' + + '&& cmake --build whisper.cpp/build --config Release' + ) + l.dim(`\n - whisper.cpp clone and compilation complete.\n`) + } catch (cloneError) { + err(`Error cloning/building whisper.cpp: ${(cloneError as Error).message}`) + throw cloneError + } + } else { + l.dim(`\n Whisper.cpp repo is already available at:\n - ./whisper.cpp\n`) + } + + // Also check for whisper-cli binary, just in case + const whisperCliPath = './whisper.cpp/build/bin/whisper-cli' + if (!existsSync(whisperCliPath)) { + l.dim(`\n No whisper-cli binary found, rebuilding...\n`) + try { + await execPromise( + 'cmake -B whisper.cpp/build -S whisper.cpp ' + + '&& cmake --build whisper.cpp/build --config Release' + ) + l.dim(`\n - whisper.cpp build completed.\n`) + } catch (buildError) { + err(`Error (re)building whisper.cpp: ${(buildError as Error).message}`) + throw buildError + } + } else { + l.dim(` Found whisper-cli at:\n - ${whisperCliPath}\n`) + } + + // 2. Make sure the chosen model file is present + const modelPath = `./whisper.cpp/models/${modelGGMLName}` + if (!existsSync(modelPath)) { + l.dim(`\n Model not found locally, attempting download...\n - ${whisperModel}\n`) + try { + await execPromise(`bash ./whisper.cpp/models/download-ggml-model.sh ${whisperModel}`) + l.dim(' - Model download completed.\n') + } catch (modelError) { + err(`Error downloading model: ${(modelError as Error).message}`) + throw modelError + } + } else { + l.dim( + ` Model "${whisperModel}" is already available at:\n` + + ` - ${modelPath}\n` + ) + } +} \ No newline at end of file diff --git a/src/utils/types/process.ts b/src/utils/types/process.ts index 76ca742..7df01eb 100644 --- a/src/utils/types/process.ts +++ b/src/utils/types/process.ts @@ -66,13 +66,13 @@ export type SeparatorParams = * @property {string} [coverImage] */ export interface EpisodeMetadata { - showLink?: string - channel?: string - channelURL?: string - title?: string - description?: string - publishDate?: string - coverImage?: string + showLink?: string | undefined + channel?: string | undefined + channelURL?: string | undefined + title?: string | undefined + description?: string | undefined + publishDate?: string | undefined + coverImage?: string | undefined } /** @@ -148,6 +148,15 @@ export type ProcessingOptions = { /** Flag to use speaker labels in AssemblyAI transcription. */ speakerLabels?: boolean + /** File path for estimating transcription cost. */ + transcriptCost?: string + + /** File path for estimating LLM cost. */ + llmCost?: string + + /** Flag to run LLM on the processed transcript. */ + runLLM?: string + /** ChatGPT model to use (e.g., 'GPT_4o_MINI'). */ chatgpt?: string @@ -175,6 +184,12 @@ export type ProcessingOptions = { /** Gemini model to use (e.g., 'GEMINI_1_5_FLASH'). */ gemini?: string + /** DeepSeek model to use (e.g., ''). */ + deepseek?: string + + /** Grok model to use (e.g., ''). */ + grok?: string + /** Array of prompt sections to include (e.g., ['titles', 'summary']). */ prompt?: string[] diff --git a/src/utils/validate-option.ts b/src/utils/validate-option.ts index f54e898..02adf34 100644 --- a/src/utils/validate-option.ts +++ b/src/utils/validate-option.ts @@ -11,17 +11,124 @@ */ import { unlink, writeFile } from 'node:fs/promises' +import { exec, execFile } from 'node:child_process' +import { promisify } from 'node:util' import { exit } from 'node:process' -import { spawn } from 'node:child_process' -import { existsSync } from 'node:fs' +import { processVideo } from '../process-commands/video' +import { processPlaylist } from '../process-commands/playlist' +import { processChannel } from '../process-commands/channel' +import { processURLs } from '../process-commands/urls' +import { processFile } from '../process-commands/file' +import { processRSS } from '../process-commands/rss' import { l, err } from '../utils/logging' -import { execPromise, execFilePromise, PROCESS_HANDLERS, ACTION_OPTIONS } from './globals/process' -import { LLM_OPTIONS } from './globals/llms' -import { TRANSCRIPT_OPTIONS } from './globals/transcription' +import { LLM_OPTIONS } from './llm-utils' +import { TRANSCRIPT_OPTIONS } from './transcription-utils' +import { validateRSSAction } from './rss-utils' -import type { ProcessingOptions, ValidAction, HandlerFunction, VideoMetadata, VideoInfo, RSSItem } from './types/process' import type { TranscriptServices } from './types/transcription' -import type { LLMServices, OllamaTagsResponse } from './types/llms' +import type { LLMServices } from './types/llms' +import type { ProcessingOptions, VideoMetadata, VideoInfo, RSSItem, ValidAction, HandlerFunction } from './types/process' + +export const execPromise = promisify(exec) +export const execFilePromise = promisify(execFile) + +import { XMLParser } from 'fast-xml-parser' + +/** + * Configure XML parser for RSS feed processing. + * Handles attributes without prefixes and allows boolean values. + * + */ +export const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '', + allowBooleanAttributes: true, +}) + +/** + * Map each action to its corresponding handler function + * + */ +export const PROCESS_HANDLERS: Record = { + video: processVideo, + playlist: processPlaylist, + channel: processChannel, + urls: processURLs, + file: processFile, + rss: processRSS, +} + +/** + * Provides user-friendly prompt choices for content generation or summary tasks. + * + */ +export const PROMPT_CHOICES: Array<{ name: string; value: string }> = [ + { name: 'Titles', value: 'titles' }, + { name: 'Summary', value: 'summary' }, + { name: 'Short Summary', value: 'shortSummary' }, + { name: 'Long Summary', value: 'longSummary' }, + { name: 'Bullet Point Summary', value: 'bulletPoints' }, + { name: 'Short Chapters', value: 'shortChapters' }, + { name: 'Medium Chapters', value: 'mediumChapters' }, + { name: 'Long Chapters', value: 'longChapters' }, + { name: 'Key Takeaways', value: 'takeaways' }, + { name: 'Questions', value: 'questions' }, + { name: 'FAQ', value: 'faq' }, + { name: 'Blog', value: 'blog' }, + { name: 'Rap Song', value: 'rapSong' }, + { name: 'Rock Song', value: 'rockSong' }, + { name: 'Country Song', value: 'countrySong' }, +] + +/** + * Available action options for content processing with additional metadata. + * + */ +export const ACTION_OPTIONS: Array<{ + name: string + description: string + message: string + validate: (input: string) => boolean | string +}> = [ + { + name: 'video', + description: 'Single YouTube Video', + message: 'Enter the YouTube video URL:', + validate: (input: string) => (input ? true : 'Please enter a valid URL.'), + }, + { + name: 'playlist', + description: 'YouTube Playlist', + message: 'Enter the YouTube playlist URL:', + validate: (input: string) => (input ? true : 'Please enter a valid URL.'), + }, + { + name: 'channel', + description: 'YouTube Channel', + message: 'Enter the YouTube channel URL:', + validate: (input: string) => (input ? true : 'Please enter a valid URL.'), + }, + { + name: 'urls', + description: 'List of URLs from File', + message: 'Enter the file path containing URLs:', + validate: (input: string) => + (input ? true : 'Please enter a valid file path.'), + }, + { + name: 'file', + description: 'Local Audio/Video File', + message: 'Enter the local audio/video file path:', + validate: (input: string) => + (input ? true : 'Please enter a valid file path.'), + }, + { + name: 'rss', + description: 'Podcast RSS Feed', + message: 'Enter the podcast RSS feed URL:', + validate: (input: string) => (input ? true : 'Please enter a valid URL.'), + }, +] /** * Validates CLI options by ensuring that only one of each set of conflicting options is provided, @@ -83,182 +190,6 @@ export function validateCLIOptions(options: ProcessingOptions): { return { action, llmServices, transcriptServices } } -/** - * Validates RSS flags (e.g., --last, --skip, --order, --date, --lastDays) without requiring feed data. - * - * @param options - The command-line options provided by the user - * @throws Exits the process if any flag is invalid - */ -export function validateRSSOptions(options: ProcessingOptions): void { - if (options.last !== undefined) { - if (!Number.isInteger(options.last) || options.last < 1) { - err('Error: The --last option must be a positive integer.') - process.exit(1) - } - if (options.skip !== undefined || options.order !== undefined) { - err('Error: The --last option cannot be used with --skip or --order.') - process.exit(1) - } - } - - if (options.skip !== undefined && (!Number.isInteger(options.skip) || options.skip < 0)) { - err('Error: The --skip option must be a non-negative integer.') - process.exit(1) - } - - if (options.order !== undefined && !['newest', 'oldest'].includes(options.order)) { - err("Error: The --order option must be either 'newest' or 'oldest'.") - process.exit(1) - } - - if (options.lastDays !== undefined) { - if (!Number.isInteger(options.lastDays) || options.lastDays < 1) { - err('Error: The --lastDays option must be a positive integer.') - process.exit(1) - } - if ( - options.last !== undefined || - options.skip !== undefined || - options.order !== undefined || - (options.date && options.date.length > 0) - ) { - err('Error: The --lastDays option cannot be used with --last, --skip, --order, or --date.') - process.exit(1) - } - } - - if (options.date && options.date.length > 0) { - const dateRegex = /^\d{4}-\d{2}-\d{2}$/ - for (const d of options.date) { - if (!dateRegex.test(d)) { - err(`Error: Invalid date format "${d}". Please use YYYY-MM-DD format.`) - process.exit(1) - } - } - - if ( - options.last !== undefined || - options.skip !== undefined || - options.order !== undefined - ) { - err('Error: The --date option cannot be used with --last, --skip, or --order.') - process.exit(1) - } - } -} - -/** - * Filters RSS feed items based on user-supplied options (e.g., item URLs, date ranges, etc.). - * - * @param options - Configuration options to filter the feed items - * @param feedItemsArray - Parsed array of RSS feed items (raw JSON from XML parser) - * @param channelTitle - Title of the RSS channel (optional) - * @param channelLink - URL to the RSS channel (optional) - * @param channelImage - A fallback channel image URL (optional) - * @returns Filtered RSS items based on the provided options - */ -export async function filterRSSItems( - options: ProcessingOptions, - feedItemsArray?: any, - channelTitle?: string, - channelLink?: string, - channelImage?: string -): Promise { - const defaultDate = new Date().toISOString().substring(0, 10) - const unfilteredItems: RSSItem[] = (feedItemsArray || []) - .filter((item: any) => { - if (!item.enclosure || !item.enclosure.type) return false - const audioVideoTypes = ['audio/', 'video/'] - return audioVideoTypes.some((type) => item.enclosure.type.startsWith(type)) - }) - .map((item: any) => { - let publishDate: string - try { - const date = item.pubDate ? new Date(item.pubDate) : new Date() - publishDate = date.toISOString().substring(0, 10) - } catch { - publishDate = defaultDate - } - - return { - showLink: item.enclosure?.url || '', - channel: channelTitle || '', - channelURL: channelLink || '', - title: item.title || '', - description: '', - publishDate, - coverImage: item['itunes:image']?.href || channelImage || '', - } - }) - - let itemsToProcess: RSSItem[] = [] - - if (options.item && options.item.length > 0) { - itemsToProcess = unfilteredItems.filter((it) => - options.item!.includes(it.showLink) - ) - } else if (options.lastDays !== undefined) { - const now = new Date() - const cutoff = new Date(now.getTime() - options.lastDays * 24 * 60 * 60 * 1000) - - itemsToProcess = unfilteredItems.filter((it) => { - const itDate = new Date(it.publishDate) - return itDate >= cutoff - }) - } else if (options.date && options.date.length > 0) { - const selectedDates = new Set(options.date) - itemsToProcess = unfilteredItems.filter((it) => - selectedDates.has(it.publishDate) - ) - } else if (options.last) { - itemsToProcess = unfilteredItems.slice(0, options.last) - } else { - const sortedItems = - options.order === 'oldest' - ? unfilteredItems.slice().reverse() - : unfilteredItems - itemsToProcess = sortedItems.slice(options.skip || 0) - } - - return itemsToProcess -} - -/** - * A helper function that validates RSS action input and processes it if valid. - * Separately validates flags with {@link validateRSSOptions} and leaves feed-item filtering to {@link filterRSSItems}. - * - * @param options - The ProcessingOptions containing RSS feed details - * @param handler - The function to handle each RSS feed - * @param llmServices - The optional LLM service for processing - * @param transcriptServices - The chosen transcription service - * @throws An error if no valid RSS URLs are provided - * @returns A promise that resolves when all RSS feeds have been processed - */ -export async function validateRSSAction( - options: ProcessingOptions, - handler: HandlerFunction, - llmServices?: LLMServices, - transcriptServices?: TranscriptServices -): Promise { - if (options.item && !Array.isArray(options.item)) { - options.item = [options.item] - } - if (typeof options.rss === 'string') { - options.rss = [options.rss] - } - - validateRSSOptions(options) - - const rssUrls = options.rss - if (!rssUrls || rssUrls.length === 0) { - throw new Error(`No valid RSS URLs provided for processing`) - } - - for (const rssUrl of rssUrls) { - await handler(options, rssUrl, llmServices, transcriptServices) - } -} - /** * Combines the validation logic for action, LLM, and transcription selection from the CLI options, * returning an object containing the validated action, chosen LLM services, and chosen transcription services. @@ -365,42 +296,6 @@ export function validateOption( return selectedOptions[0] as string | undefined } -/** - * Validates channel processing options for consistency and correct values. - * Logs the current channel processing action based on provided options. - * - * @param options - Configuration options to validate - * @throws Will exit the process if validation fails - */ -export function validateChannelOptions(options: ProcessingOptions): void { - if (options.last !== undefined) { - if (!Number.isInteger(options.last) || options.last < 1) { - err('Error: The --last option must be a positive integer.') - process.exit(1) - } - if (options.skip !== undefined || options.order !== undefined) { - err('Error: The --last option cannot be used with --skip or --order.') - process.exit(1) - } - } - - if (options.skip !== undefined && (!Number.isInteger(options.skip) || options.skip < 0)) { - err('Error: The --skip option must be a non-negative integer.') - process.exit(1) - } - - if (options.order !== undefined && !['newest', 'oldest'].includes(options.order)) { - err("Error: The --order option must be either 'newest' or 'oldest'.") - process.exit(1) - } - - if (options.last) { - l.wait(`\nProcessing the last ${options.last} videos`) - } else if (options.skip) { - l.wait(`\nSkipping first ${options.skip || 0} videos`) - } -} - /** * Removes temporary files generated during content processing. * Attempts to delete files with specific extensions and logs the results. @@ -432,18 +327,17 @@ export async function saveAudio(id: string, ensureFolders?: boolean) { if (ensureFolders) { // If "ensureFolders" is set, skip deleting files // (this can serve as a placeholder for ensuring directories) - l.info('\nSkipping cleanup to preserve or ensure metadata directories.\n') + l.dim('\nSkipping cleanup to preserve or ensure metadata directories.\n') return } - l.step('\nStep 6 - Cleaning Up Extra Files\n') const extensions = ['.wav'] - l.wait(`\n Temporary files deleted:`) + l.dim(` Temporary files deleted:`) for (const ext of extensions) { try { await unlink(`${id}${ext}`) - l.wait(` - ${id}${ext}`) + l.dim(` - ${id}${ext}`) } catch (error) { if (error instanceof Error && (error as Error).message !== 'ENOENT') { err(`Error deleting file ${id}${ext}: ${(error as Error).message}`) @@ -452,166 +346,6 @@ export async function saveAudio(id: string, ensureFolders?: boolean) { } } -/** - * Checks if whisper.cpp directory exists and, if missing, clones and compiles it. - * Also checks if the chosen model file is present and, if missing, downloads it. - * @param {string} whisperModel - The requested Whisper model name - * @param {string} modelGGMLName - The corresponding GGML model filename - * @returns {Promise} - */ -export async function checkWhisperDirAndModel( - whisperModel: string, - modelGGMLName: string -): Promise { - // Check if whisper.cpp directory is present - if (!existsSync('./whisper.cpp')) { - l.wait(`\n No whisper.cpp repo found, cloning and compiling...\n`) - try { - await execPromise('git clone https://github.com/ggerganov/whisper.cpp.git && cmake -B whisper.cpp/build -S whisper.cpp && cmake --build whisper.cpp/build --config Release') - l.wait(`\n - whisper.cpp clone and compilation complete.\n`) - } catch (cloneError) { - err(`Error cloning/building whisper.cpp: ${(cloneError as Error).message}`) - throw cloneError - } - } else { - l.wait(`\n Whisper.cpp repo is already available at:\n - ./whisper.cpp\n`) - } - - // Check if the chosen model file is present - if (!existsSync(`./whisper.cpp/models/${modelGGMLName}`)) { - l.wait(`\n Model not found, downloading...\n - ${whisperModel}\n`) - try { - await execPromise(`bash ./whisper.cpp/models/download-ggml-model.sh ${whisperModel}`) - l.wait(' - Model download completed, running transcription...\n') - } catch (modelError) { - err(`Error downloading model: ${(modelError as Error).message}`) - throw modelError - } - } else { - l.wait(` Model ${whisperModel} is already available at\n - ./whisper.cpp/models/${modelGGMLName}\n`) - } -} - -/** - * checkOllamaServerAndModel() - * --------------------- - * Checks if the Ollama server is running, attempts to start it if not, - * and ensures the specified model is available (pulling if needed). - * - * @param {string} ollamaHost - The Ollama host - * @param {string} ollamaPort - The Ollama port - * @param {string} ollamaModelName - The Ollama model name (e.g. 'qwen2.5:0.5b') - * @returns {Promise} - */ -export async function checkOllamaServerAndModel( - ollamaHost: string, - ollamaPort: string, - ollamaModelName: string -): Promise { - // Helper to check if the Ollama server responds - async function checkServer(): Promise { - try { - const serverResponse = await fetch(`http://${ollamaHost}:${ollamaPort}`) - return serverResponse.ok - } catch (error) { - return false - } - } - - l.info(`[checkOllamaServerAndModel] Checking server: http://${ollamaHost}:${ollamaPort}`) - - // 1) Confirm the server is running - if (await checkServer()) { - l.wait('\n Ollama server is already running...') - } else { - // If the Docker-based environment uses 'ollama' as hostname but it's not up, that's likely an error - if (ollamaHost === 'ollama') { - throw new Error('Ollama server is not running. Please ensure the Ollama server is running and accessible.') - } else { - // Attempt to spawn an Ollama server locally - l.wait('\n Ollama server is not running. Attempting to start it locally...') - const ollamaProcess = spawn('ollama', ['serve'], { - detached: true, - stdio: 'ignore', - }) - ollamaProcess.unref() - - // Wait up to ~30 seconds for the server to respond - let attempts = 0 - while (attempts < 30) { - if (await checkServer()) { - l.wait(' - Ollama server is now ready.\n') - break - } - await new Promise((resolve) => setTimeout(resolve, 1000)) - attempts++ - } - if (attempts === 30) { - throw new Error('Ollama server failed to become ready in time.') - } - } - } - - // 2) Confirm the model is available; if not, pull it - l.wait(` Checking if model is available: ${ollamaModelName}`) - try { - const tagsResponse = await fetch(`http://${ollamaHost}:${ollamaPort}/api/tags`) - if (!tagsResponse.ok) { - throw new Error(`HTTP error! status: ${tagsResponse.status}`) - } - - const tagsData = (await tagsResponse.json()) as OllamaTagsResponse - const isModelAvailable = tagsData.models.some((m) => m.name === ollamaModelName) - l.info(`[checkOllamaServerAndModel] isModelAvailable=${isModelAvailable}`) - - if (!isModelAvailable) { - l.wait(`\n Model ${ollamaModelName} is NOT available; pulling now...`) - const pullResponse = await fetch(`http://${ollamaHost}:${ollamaPort}/api/pull`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ name: ollamaModelName }), - }) - if (!pullResponse.ok) { - throw new Error(`Failed to initiate pull for model ${ollamaModelName}`) - } - if (!pullResponse.body) { - throw new Error('Response body is null while pulling model.') - } - - const reader = pullResponse.body.getReader() - const decoder = new TextDecoder() - - // Stream the JSON lines from the server - while (true) { - const { done, value } = await reader.read() - if (done) break - - const chunk = decoder.decode(value) - const lines = chunk.split('\n') - for (const line of lines) { - if (line.trim() === '') continue - - // Each line should be a JSON object from the Ollama server - try { - const parsedLine = JSON.parse(line) - if (parsedLine.status === 'success') { - l.wait(` - Model ${ollamaModelName} pulled successfully.\n`) - break - } - } catch (parseError) { - err(`Error parsing JSON while pulling model: ${parseError}`) - } - } - } - } else { - l.wait(`\n Model ${ollamaModelName} is already available.\n`) - } - } catch (error) { - err(`Error checking/pulling model: ${(error as Error).message}`) - throw error - } -} - /** * Sanitizes a title string for use in filenames by: * - Removing special characters except spaces and hyphens @@ -694,7 +428,7 @@ export async function saveInfo( const sanitizedTitle = sanitizeTitle(title || '') const jsonFilePath = `content/${sanitizedTitle}_info.json` await writeFile(jsonFilePath, jsonContent) - l.wait(`RSS feed information saved to: ${jsonFilePath}`) + l.dim(`RSS feed information saved to: ${jsonFilePath}`) return } diff --git a/test/local.test.ts b/test/local.test.ts index 9d6f753..4f4a39b 100644 --- a/test/local.test.ts +++ b/test/local.test.ts @@ -9,7 +9,7 @@ import { join } from 'node:path' const commands = [ { // Process single local audio file. - cmd: 'npm run as -- --file "content/audio.mp3" --whisper tiny', + cmd: 'npm run as -- --file "content/audio.mp3" --whisper base', expectedFile: 'audio-prompt.md', newName: '01-file-default.md' }, @@ -53,7 +53,7 @@ const commands = [ }, { // Process multiple YouTube videos from URLs listed in a file. - cmd: 'npm run as -- --urls "content/example-urls.md"', + cmd: 'npm run as -- --urls "content/example-urls.md" --whisper tiny', expectedFiles: [ { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: '09-urls-default.md' }, { file: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: '10-urls-default.md' }