Merge pull request #119 from RipeSeed/feature/pinecone-index-name-configuration

Add Pinecone Index Name to Configuration

Author: riazosama

src/apis/admin/configuration.ts

@@ -7,6 +7,7 @@ interface ConfigurationData {
   xAccessKey?: string
   xBaseUrl?: string
   pineconeApiKey?: string
+  pineconeIndexName?: string
   calendlyMeetingLink?: string
 }
 // add or update credentials
@@ -23,7 +24,10 @@ export const addUpdateConfiguration = async (data: ConfigurationData) => {
           accessKey: data.xAccessKey,
           baseUrl: data.xBaseUrl
         } : undefined,
-        pinecone: data.pineconeApiKey ? { apiKey: data.pineconeApiKey } : undefined
+        pinecone: data.pineconeApiKey ? { 
+          apiKey: data.pineconeApiKey,
+          indexName: data.pineconeIndexName 
+        } : undefined
       },
       calendlyMeetingLink: data.calendlyMeetingLink
     })

src/app/(admin)/dashboard/configuration/_types/schema.ts

@@ -8,6 +8,7 @@ export const UpdateSchema = z
     xAccessKey: z.string().optional(),
     xBaseUrl: z.string().url('Please enter a valid X API base URL (e.g., https://api.x.com)').optional(),
     pineconeApiKey: z.string().optional(),
+    pineconeIndexName: z.string().optional(),
     calendlyMeetingLink: z.string().url('Please enter a valid Calendly meeting URL (e.g., https://calendly.com/your-name/meeting)').optional(),
   })
   .superRefine((data, ctx) => {
@@ -29,6 +30,14 @@ export const UpdateSchema = z
         path: data.xAccessKey ? ['xBaseUrl'] : ['xAccessKey'],
       })
     }
+    
+    if (data.pineconeApiKey && !data.pineconeIndexName) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: 'Pinecone Index Name is required when API Key is provided',
+        path: ['pineconeIndexName'],
+      })
+    }
   })
 
 export type TUpdateSchema = z.infer<typeof UpdateSchema>

src/app/(admin)/dashboard/configuration/page.tsx

@@ -6,8 +6,12 @@ import { zodResolver } from '@hookform/resolvers/zod'
 import { useMutation, useQuery } from '@tanstack/react-query'
 import { useForm } from 'react-hook-form'
 
-import { addUpdateConfiguration, getConfiguration } from '@/apis/admin/configuration'
+import {
+  addUpdateConfiguration,
+  getConfiguration,
+} from '@/apis/admin/configuration'
 import { Button } from '@/components/ui/button'
+import { Input } from '@/components/ui/input'
 import { Label } from '@/components/ui/label'
 import { useToast } from '@/hooks/use-toast'
 import { InputWithToggle } from './_components/InputWithToggle'
@@ -54,6 +58,7 @@ export default function Configuration() {
           xAccessKey: creds.providers?.x?.accessKey || '',
           xBaseUrl: creds.providers?.x?.baseUrl || '',
           pineconeApiKey: creds.providers?.pinecone?.apiKey || '',
+          pineconeIndexName: creds.providers?.pinecone?.indexName || '',
           calendlyMeetingLink: creds.calendlyMeetingLink || '',
         })
       }
@@ -62,7 +67,7 @@ export default function Configuration() {
     // Disable automatic refetching
     refetchOnMount: false,
     refetchOnWindowFocus: false,
-    refetchOnReconnect: false
+    refetchOnReconnect: false,
   })
 
   const handleClick = (data: TUpdateSchema) => {
@@ -243,6 +248,34 @@ export default function Configuration() {
                       error={errors.pineconeApiKey}
                     />
                   </div>
+
+                  <div className='flex w-full flex-col space-y-2'>
+                    <Label className='flex items-center gap-2 text-dashboardText'>
+                      <span>Pinecone Index Name</span>
+                      <span className='relative -top-1.5 mt-1'>
+                        <Image
+                          src='/assets/knowledgebase/required.svg'
+                          alt='required'
+                          width={4}
+                          height={4}
+                        />
+                      </span>
+                    </Label>
+                    <Input
+                      {...register('pineconeIndexName')}
+                      placeholder='your-pinecone-index'
+                      className='h-10 bg-white'
+                    />
+                    <p className='mt-1 text-xs text-muted-foreground'>
+                      You can find your index name in the Pinecone dashboard
+                      under &apos;Indexes&apos;
+                    </p>
+                    {errors.pineconeIndexName && (
+                      <p className='text-xs text-red-500'>
+                        {errors.pineconeIndexName.message}
+                      </p>
+                    )}
+                  </div>
                 </div>
               </div>
 

src/app/api/(admin)/config/route.ts

@@ -8,6 +8,8 @@ export const POST = async (request: NextRequest) => {
     const { providers, calendlyMeetingLink } = await request.json()
     await connectDB()
 
+    console.log('providers', providers)
+
     await APICredentials.deleteMany({})
 
     const updatedCredentials = await APICredentials.create({

src/app/api/(admin)/knowledgebase/file/route.ts

@@ -1,17 +1,13 @@
 import { NextRequest, NextResponse } from 'next/server'
 import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf'
 import { OpenAIEmbeddings } from '@langchain/openai'
-import { Index, Pinecone as PineconeClient } from '@pinecone-database/pinecone'
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
 import { v4 as uuid } from 'uuid'
 
 import { connectDB } from '@/models'
 import FileModel from '@/models/knowledgeBase/File.model'
+import { getPineconeIndex } from '@/services/pinecone/client'
 
-const pinecone = new PineconeClient({
-  apiKey: process.env.PINECONE_API_KEY!,
-})
-const pineconeIndex: Index = pinecone.Index(process.env.PINECONE_INDEX!)
 export const POST = async (request: NextRequest, response: NextResponse) => {
   try {
     const form = await request.formData()
@@ -85,6 +81,9 @@ const uploadChunksToPineCone = async (
   fileId: string,
 ) => {
   try {
+    // Get the Pinecone index from our centralized service
+    const pineconeIndex = await getPineconeIndex()
+    
     let batchSize = 100
     let batch = []
     for (let i = 0; i < chunks.length; i++) {

src/models/credentials/APICredentials.model.ts

@@ -32,6 +32,10 @@ const APICredentialsSchema = new mongoose.Schema(
         apiKey: {
           type: String,
           required: false
+        },
+        indexName: {
+          type: String,
+          required: false
         }
       }
     },

src/services/chat/chat.ts

@@ -6,7 +6,7 @@ import { PineconeStore } from '@langchain/pinecone'
 import type { Document as LangchainDoc } from 'langchain/document'
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
 
-import { pineconeIndex } from './config'
+import { getPineconeIndex } from './config'
 
 export const splitText = async (file: File, id: string) => {
   try {
@@ -25,6 +25,9 @@ export const createEmbeddings = async (
   embeddings: OpenAIEmbeddings,
 ) => {
   try {
+    // Get the Pinecone index
+    const pineconeIndex = await getPineconeIndex()
+    
     const vectorStore = await PineconeStore.fromDocuments(docs, embeddings, {
       pineconeIndex,
     })

src/services/chat/config.ts

@@ -1,11 +1,6 @@
 import 'server-only'
 
-import { Pinecone } from '@pinecone-database/pinecone'
+import { getPineconeIndex } from '../pinecone/client'
 
-const pinecone = new Pinecone({
-  apiKey: process.env.PINECONE_API_KEY!,
-})
-
-const pineconeIndex = pinecone.Index(process.env.PINECONE_INDEX!)
-
-export { pineconeIndex }
+// Export the getPineconeIndex function directly
+export { getPineconeIndex }

src/services/chat/conversation.ts

@@ -7,7 +7,7 @@ import 'server-only'
 import { OpenAI } from 'openai'
 
 import { connectDB } from '@/models'
-import { pineconeIndex } from './config'
+import { getPineconeIndex } from './config'
 import APICredentials from '@/models/credentials/APICredentials.model'
 
 export interface Context {
@@ -178,6 +178,7 @@ export function converse(
 
         let serializedDocs = ''
         if (idArray[0] !== null) {
+          const pineconeIndex = await getPineconeIndex()
           const docs = await pineconeIndex.query({
             vector,
             topK: 2,

src/services/pinecone/client.ts

@@ -0,0 +1,76 @@
+import { Index, Pinecone as PineconeClient } from '@pinecone-database/pinecone'
+
+import { connectDB } from '@/models'
+import APICredentials from '@/models/credentials/APICredentials.model'
+
+// Cache for the Pinecone client and index
+let pineconeClientInstance: PineconeClient | null = null
+let pineconeIndexInstance: Index | null = null
+let indexName: string = 'default-index'
+
+/**
+ * Initialize the Pinecone client with credentials from the database
+ * @returns The Pinecone client instance
+ */
+export const getPineconeClient = async (): Promise<PineconeClient> => {
+  // Return cached instance if available
+  if (pineconeClientInstance) {
+    return pineconeClientInstance
+  }
+
+  try {
+    await connectDB()
+    const credentials = await APICredentials.findOne()
+
+    if (!credentials || !credentials.providers.pinecone?.apiKey) {
+      throw new Error('Pinecone API key not found in database')
+    }
+
+    // Create a new Pinecone client
+    pineconeClientInstance = new PineconeClient({
+      apiKey: credentials.providers.pinecone.apiKey,
+    })
+    indexName =
+      credentials.providers.pinecone.indexName ||
+      process.env.PINECONE_INDEX ||
+      'default-index'
+
+    return pineconeClientInstance
+  } catch (error) {
+    console.error('Error initializing Pinecone client:', error)
+    throw error
+  }
+}
+
+/**
+ * Get the Pinecone index
+ * @param indexNameParam Optional index name to override the default
+ * @returns The Pinecone index instance
+ */
+export const getPineconeIndex = async (
+  indexNameParam?: string,
+): Promise<Index> => {
+  // If index name is provided and different from cached one, reset the index instance
+  if (indexNameParam && indexNameParam !== indexName) {
+    pineconeIndexInstance = null
+    indexName = indexNameParam
+  }
+
+  // Return cached instance if available
+  if (pineconeIndexInstance) {
+    return pineconeIndexInstance
+  }
+
+  try {
+    // Get the Pinecone client
+    const pineconeClient = await getPineconeClient()
+
+    // Create a new Pinecone index
+    pineconeIndexInstance = pineconeClient.Index(indexName)
+
+    return pineconeIndexInstance
+  } catch (error) {
+    console.error('Error getting Pinecone index:', error)
+    throw error
+  }
+}