Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@
"@opentelemetry/sdk-trace-base": "^1.30.1",
"@opentelemetry/semantic-conventions": "^1.29.0",
"@perplexity-ai/perplexity_ai": "^0.10.0",
"@qdrant/js-client-rest": "^1.16.2",
"@sinclair/typebox": "^0.34.41",
"@slack/bolt": "^4.4.0",
"@slack/web-api": "^7.9.3",
Expand Down
27 changes: 27 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

49 changes: 43 additions & 6 deletions src/swe/vector/core/autoDetect.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { AlloyDBNestedConfig, ChromaNestedConfig, DiscoveryEngineConfig, GoogleCloudConfig, VectorStoreConfig } from './config';
import type { AlloyDBNestedConfig, ChromaNestedConfig, DiscoveryEngineConfig, GoogleCloudConfig, QdrantNestedConfig, VectorStoreConfig } from './config';

export type VectorBackend = 'alloydb' | 'discovery-engine' | 'chroma';
export type VectorBackend = 'alloydb' | 'discovery-engine' | 'chroma' | 'qdrant';

export interface BackendDetection {
backend: VectorBackend | null;
Expand All @@ -18,7 +18,28 @@ export interface BackendDetection {
* 4. null - No backend detected
*/
export function detectBackend(): BackendDetection {
// 1. Check for ChromaDB (local-first)
// 1. Check for Qdrant
const qdrantUrl = process.env.QDRANT_URL;
if (qdrantUrl) {
const qdrant: QdrantNestedConfig = {
url: qdrantUrl,
apiKey: process.env.QDRANT_API_KEY,
};

return {
backend: 'qdrant',
reason: 'Qdrant detected via QDRANT_URL',
config: {
qdrant,
embedding: {
provider: 'ollama',
model: process.env.OLLAMA_EMBEDDING_MODEL || 'manutic/nomic-embed-code',
},
},
};
}

// 2. Check for ChromaDB (local-first)
const chromaUrl = process.env.CHROMA_URL;
if (chromaUrl) {
const chroma: ChromaNestedConfig = {
Expand All @@ -41,7 +62,7 @@ export function detectBackend(): BackendDetection {
};
}

// 2. Check for AlloyDB/Postgres
// 3. Check for AlloyDB/Postgres
const pgHost = process.env.ALLOYDB_HOST || process.env.PGHOST;
if (pgHost) {
const alloydb: AlloyDBNestedConfig = {
Expand All @@ -59,7 +80,7 @@ export function detectBackend(): BackendDetection {
};
}

// 3. Check for Discovery Engine
// 4. Check for Discovery Engine
const gcpProject = process.env.GCLOUD_PROJECT;
if (gcpProject) {
const googleCloud: GoogleCloudConfig = {
Expand All @@ -80,7 +101,7 @@ export function detectBackend(): BackendDetection {
};
}

// 4. No backend detected
// 5. No backend detected
return {
backend: null,
reason: 'No backend detected',
Expand All @@ -96,6 +117,7 @@ export function requireBackend(): BackendDetection {
if (!detection.backend) {
throw new Error(
'No vector backend detected. Set one of:\n' +
' - QDRANT_URL (for Qdrant + Ollama, local development)\n' +
' - CHROMA_URL (for ChromaDB + Ollama, local development)\n' +
' - ALLOYDB_HOST or PGHOST (for AlloyDB/Postgres)\n' +
' - GCLOUD_PROJECT (for Discovery Engine)',
Expand All @@ -109,6 +131,19 @@ export function requireBackend(): BackendDetection {
*/
export function buildBackendConfig(backend: VectorBackend): Partial<VectorStoreConfig> {
switch (backend) {
case 'qdrant': {
const qdrant: QdrantNestedConfig = {
url: process.env.QDRANT_URL || 'http://localhost:6333',
apiKey: process.env.QDRANT_API_KEY,
};
return {
qdrant,
embedding: {
provider: 'ollama',
model: process.env.OLLAMA_EMBEDDING_MODEL || 'manutic/nomic-embed-code',
},
};
}
case 'chroma': {
const chroma: ChromaNestedConfig = {
url: process.env.CHROMA_URL || 'http://localhost:8000',
Expand Down Expand Up @@ -146,5 +181,7 @@ export function buildBackendConfig(backend: VectorBackend): Partial<VectorStoreC
};
return { googleCloud, discoveryEngine };
}
default:
throw new Error(`Unknown backend: ${backend}`);
}
}
30 changes: 30 additions & 0 deletions src/swe/vector/core/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,20 @@ export interface ChromaNestedConfig {
textWeight?: number;
}

/**
* Qdrant configuration (for vector storage)
*/
export interface QdrantNestedConfig {
/** Qdrant server URL (default: http://localhost:6333 or QDRANT_URL env var) */
url?: string;
/** API key for Qdrant Cloud or authenticated servers (optional) */
apiKey?: string;
/** Collection name prefix (default: 'code_chunks') */
collectionPrefix?: string;
/** Distance function: 'Cosine' | 'Euclid' | 'Dot' (default: 'Cosine') */
distanceFunction?: 'Cosine' | 'Euclid' | 'Dot';
}

/**
* Embedding configuration
*/
Expand Down Expand Up @@ -177,6 +191,10 @@ export interface VectorStoreConfig {
/** ChromaDB settings (for local vector storage) */
chroma?: ChromaNestedConfig;

// === Qdrant Configuration ===
/** Qdrant settings (for vector search) */
qdrant?: QdrantNestedConfig;

// === Embedding Configuration ===
/** Embedding settings */
embedding?: EmbeddingConfig;
Expand Down Expand Up @@ -700,6 +718,18 @@ export async function createVectorOrchestrator(repoRoot: string): Promise<IVecto
if (!config.indexed) return null;

// Determine backend based on config
if (config.qdrant?.url) {
// Use Qdrant - dynamic import to avoid loading when not needed
const { buildQdrantConfig, QdrantOrchestrator } = await import('../qdrant/index.js');
return new QdrantOrchestrator(repoRoot, config);
}

if (config.chroma?.url) {
// Use ChromaDB - dynamic import to avoid loading when not needed
const { ChromaOrchestrator } = await import('../chroma/index.js');
return new ChromaOrchestrator(repoRoot, config);
}

if (config.alloydb?.host || config.alloydb?.instance) {
// Use AlloyDB - dynamic import to avoid loading when not needed
const { buildAlloyDBConfig, AlloyDBOrchestrator } = await import('../alloydb/index.js');
Expand Down
3 changes: 3 additions & 0 deletions src/swe/vector/qdrant/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export { QdrantAdapter } from './qdrantAdapter';
export { QdrantOrchestrator } from './qdrantOrchestrator';
export { buildQdrantConfig, getCollectionNameForRepo, sanitizeRepoName, type QdrantConfig } from './qdrantConfig';
166 changes: 166 additions & 0 deletions src/swe/vector/qdrant/qdrantAdapter.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
import { QdrantClient } from '@qdrant/js-client-rest';
import { v5 as uuidv5 } from 'uuid';
import { logger } from '#o11y/logger';
import type { VectorStoreConfig } from '../core/config';
import type { EmbeddedChunk, IVectorStore, SearchResult } from '../core/interfaces';
import type { QdrantConfig } from './qdrantConfig';
import { getCollectionNameForRepo } from './qdrantConfig';

export class QdrantAdapter implements IVectorStore {
private client: QdrantClient;
private config: VectorStoreConfig;
private qdrantConfig: QdrantConfig;
private collectionName: string;
private initialized = false;

constructor(repoIdentifier: string, qdrantConfig: QdrantConfig) {
this.qdrantConfig = qdrantConfig;
this.collectionName = getCollectionNameForRepo(repoIdentifier, qdrantConfig.collectionPrefix);
this.config = { chunking: { dualEmbedding: false, contextualChunking: false } };
this.client = new QdrantClient({ url: qdrantConfig.url, apiKey: qdrantConfig.apiKey });
}

async initialize(config: VectorStoreConfig): Promise<void> {
this.config = config;
logger.info({ collectionName: this.collectionName }, 'Initializing Qdrant adapter');

try {
const { collections } = await this.client.getCollections();
if (!collections.some((c) => c.name === this.collectionName)) {
await this.client.createCollection(this.collectionName, {
vectors: {
size: this.qdrantConfig.embeddingDimension,
distance: this.qdrantConfig.distanceFunction || 'Cosine',
},
});

for (const field of ['config_name', 'filename', 'language']) {
await this.client.createPayloadIndex(this.collectionName, { field_name: field, field_schema: 'keyword' });
}
logger.info({ collectionName: this.collectionName }, 'Created collection');
}

this.initialized = true;
} catch (error) {
logger.error({ error, collectionName: this.collectionName }, 'Failed to initialize');
throw error;
}
}

private generatePointId(chunk: EmbeddedChunk): string {
const key = `${chunk.filePath}:${chunk.chunk.sourceLocation.startLine}:${chunk.chunk.sourceLocation.endLine}`;
return uuidv5(key, uuidv5.DNS);
}

private configFilter(configName: string) {
return { must: [{ key: 'config_name', match: { value: configName } }] } as any;
}

async indexChunks(chunks: EmbeddedChunk[]): Promise<void> {
if (!chunks.length) return;
if (!this.initialized) throw new Error('Not initialized');

const configName = this.config.name || 'default';
logger.info({ chunkCount: chunks.length }, 'Indexing');

for (let i = 0; i < chunks.length; i += 100) {
const points = chunks.slice(i, i + 100).map((chunk) => ({
id: this.generatePointId(chunk),
vector: chunk.embedding,
payload: {
config_name: configName,
filename: chunk.filePath,
line_from: chunk.chunk.sourceLocation.startLine,
line_to: chunk.chunk.sourceLocation.endLine,
original_text: chunk.chunk.content,
contextualized_text: 'contextualizedContent' in chunk.chunk ? chunk.chunk.contextualizedContent : chunk.chunk.content,
language: chunk.language,
chunk_type: chunk.chunk.chunkType,
function_name: chunk.chunk.metadata?.functionName || '',
class_name: chunk.chunk.metadata?.className || '',
natural_language_description: chunk.naturalLanguageDescription || '',
},
}));

await this.client.upsert(this.collectionName, { wait: true, points });
}
}

async deleteByFilePath(filePath: string): Promise<number> {
if (!this.initialized) throw new Error('Not initialized');

const configName = this.config.name || 'default';
const filter = {
must: [
{ key: 'filename', match: { value: filePath } },
{ key: 'config_name', match: { value: configName } },
],
} as any;

const { count } = await this.client.count(this.collectionName, { filter, exact: true });
if (count > 0) {
await this.client.delete(this.collectionName, { wait: true, filter });
logger.info({ filePath, deletedCount: count }, 'Deleted');
}
return count;
}

async search(query: string, queryEmbedding: number[], maxResults: number, config: VectorStoreConfig): Promise<SearchResult[]> {
if (!this.initialized) throw new Error('Not initialized');

const results = await this.client.query(this.collectionName, {
query: queryEmbedding,
limit: maxResults,
filter: this.configFilter(config.name || 'default'),
with_payload: true,
with_vector: false,
});

return results.points.map((r) => ({
id: String(r.id),
score: r.score,
document: {
filePath: String(r.payload?.filename || ''),
functionName: r.payload?.function_name ? String(r.payload.function_name) : undefined,
className: r.payload?.class_name ? String(r.payload.class_name) : undefined,
startLine: Number(r.payload?.line_from) || 0,
endLine: Number(r.payload?.line_to) || 0,
language: String(r.payload?.language || 'unknown'),
originalCode: String(r.payload?.original_text || ''),
naturalLanguageDescription: r.payload?.natural_language_description ? String(r.payload.natural_language_description) : undefined,
},
metadata: { chunkType: r.payload?.chunk_type },
}));
}

async purge(): Promise<void> {
if (!this.initialized) throw new Error('Not initialized');
await this.client.delete(this.collectionName, { wait: true, filter: this.configFilter(this.config.name || 'default') });
}

async getStats(): Promise<{ totalDocuments: number; totalChunks: number; storageSize?: number }> {
if (!this.initialized) throw new Error('Not initialized');

const [info, { count }] = await Promise.all([
this.client.getCollection(this.collectionName),
this.client.count(this.collectionName, { filter: this.configFilter(this.config.name || 'default'), exact: true }),
]);

return { totalDocuments: count, totalChunks: count, storageSize: info.points_count ?? undefined };
}

async isAvailable(): Promise<boolean> {
try {
await this.client.getCollections();
return true;
} catch {
return false;
}
}

async deleteCollection(): Promise<void> {
await this.client.deleteCollection(this.collectionName);
this.initialized = false;
logger.info({ collectionName: this.collectionName }, 'Collection deleted');
}
}
Loading