Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

V3 graph implmentations #1593

Merged
merged 25 commits into from
Nov 15, 2024
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion js/sdk/__tests__/r2rV2ClientIntegrationSuperUser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ let newCollectionId: string;
* X createGraph
* X enrichGraph
* X getEntities
* X getTriples
* X getRelationships
* X getCommunities
* X getTunedPrompt
* X deduplicateEntities
Expand Down
2 changes: 1 addition & 1 deletion js/sdk/__tests__/r2rV2ClientIntegrationUser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ const baseUrl = "http://localhost:7272";
* X createGraph
* X enrichGraph
* X getEntities
* X getTriples
* X getRelationships
* X getCommunities
* X getTunedPrompt
* X deduplicateEntities
Expand Down
4 changes: 2 additions & 2 deletions js/sdk/src/models.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,13 @@ export enum KGRunType {
}

export interface KGCreationSettings {
kg_triples_extraction_prompt?: string;
kg_relationships_extraction_prompt?: string;
kg_entity_description_prompt?: string;
force_kg_creation?: boolean;
entity_types?: string[];
relation_types?: string[];
extractions_merge_count?: number;
max_knowledge_triples?: number;
max_knowledge_relationships?: number;
max_description_input_length?: number;
generation_config?: GenerationConfig;
}
Expand Down
16 changes: 8 additions & 8 deletions js/sdk/src/r2rClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1685,21 +1685,21 @@ export class r2rClient extends BaseClient {
}

/**
* Retrieve triples from the knowledge graph.
* Retrieve relationships from the knowledge graph.
* @returns A promise that resolves to the response from the server.
* @param collection_id The ID of the collection to retrieve entities for.
* @param offset The offset for pagination.
* @param limit The limit for pagination.
* @param entity_level The level of entity to filter by.
* @param triple_ids Triple IDs to filter by.
* @param relationship_ids Relationship IDs to filter by.
*/
@feature("getTriples")
async getTriples(
@feature("getRelationships")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's keep the SDK methods here to getTriples, as not to break any current implementation. We can also add an @deprecated warning to this such that it warns users to switch to the v3 SDK methods

async getRelationships(
collection_id?: string,
offset?: number,
limit?: number,
entity_level?: string,
triple_ids?: string[],
relationship_ids?: string[],
): Promise<any> {
this._ensureAuthenticated();

Expand All @@ -1716,11 +1716,11 @@ export class r2rClient extends BaseClient {
if (entity_level !== undefined) {
params.entity_level = entity_level;
}
if (triple_ids !== undefined) {
params.entity_ids = triple_ids;
if (relationship_ids !== undefined) {
params.entity_ids = relationship_ids;
}

return this._makeRequest("GET", `triples`, { params });
return this._makeRequest("GET", `relationships`, { params });
}

/**
Expand Down
18 changes: 9 additions & 9 deletions py/cli/commands/v2/kg.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ async def get_entities(
@click.option(
"--collection-id",
required=True,
help="Collection ID to retrieve triples from.",
help="Collection ID to retrieve relationships from.",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar to the SDK, let's leave the CLI as is. I have a deprecation wrapper that we can add to these methods, and then we will point users to the new command.

)
@click.option(
"--offset",
Expand All @@ -244,31 +244,31 @@ async def get_entities(
help="Limit for pagination.",
)
@click.option(
"--triple-ids",
"--relationship-ids",
multiple=True,
help="Triple IDs to filter by.",
help="Relationship IDs to filter by.",
)
@click.option(
"--entity-names",
multiple=True,
help="Entity names to filter by.",
)
@pass_context
async def get_triples(
ctx, collection_id, offset, limit, triple_ids, entity_names
async def get_relationships(
ctx, collection_id, offset, limit, relationship_ids, entity_names
):
"""
Retrieve triples from the knowledge graph.
Retrieve relationships from the knowledge graph.
"""
client: R2RAsyncClient = ctx.obj

with timer():
response = await client.get_triples(
response = await client.get_relationships(
collection_id,
offset,
limit,
list(entity_names),
list(triple_ids),
list(relationship_ids),
)

click.echo(json.dumps(response, indent=2))
Expand All @@ -291,7 +291,7 @@ async def delete_graph_for_collection(ctx, collection_id, cascade):
"""
Delete the graph for a given collection.

NOTE: Setting the cascade flag to true will delete entities and triples for documents that are shared across multiple collections. Do not set this flag unless you are absolutely sure that you want to delete the entities and triples for all documents in the collection.
NOTE: Setting the cascade flag to true will delete entities and relationships for documents that are shared across multiple collections. Do not set this flag unless you are absolutely sure that you want to delete the entities and relationships for all documents in the collection.
"""
client: R2RAsyncClient = ctx.obj

Expand Down
4 changes: 2 additions & 2 deletions py/core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
# KG abstractions
"Entity",
"KGExtraction",
"Triple",
"Relationship",
# LLM abstractions
"GenerationConfig",
"LLMChatCompletion",
Expand Down Expand Up @@ -213,7 +213,7 @@
## PIPES
"SearchPipe",
"EmbeddingPipe",
"KGTriplesExtractionPipe",
"KGRelationshipsExtractionPipe",
"ParsingPipe",
"QueryTransformPipe",
"SearchRAGPipe",
Expand Down
11 changes: 6 additions & 5 deletions py/core/base/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,12 @@
# KG abstractions
"Entity",
"KGExtraction",
"Triple",
"Relationship",
"Community",
"CommunityInfo",
"KGCreationSettings",
"KGEnrichmentSettings",
"KGRunType",
# LLM abstractions
"GenerationConfig",
"LLMChatCompletion",
Expand All @@ -47,10 +52,6 @@
"VectorSearchResult",
"SearchSettings",
"HybridSearchSettings",
# KG abstractions
"KGCreationSettings",
"KGEnrichmentSettings",
"KGRunType",
# User abstractions
"Token",
"TokenData",
Expand Down
9 changes: 5 additions & 4 deletions py/core/base/abstractions/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@
from shared.abstractions.graph import (
Community,
CommunityInfo,
CommunityReport,
Community,
Entity,
EntityLevel,
EntityType,
Graph,
KGExtraction,
RelationshipType,
Triple,
Relationship,
)
from shared.abstractions.ingestion import (
ChunkEnrichmentSettings,
Expand Down Expand Up @@ -109,9 +110,9 @@
"EntityType",
"RelationshipType",
"Community",
"CommunityReport",
"CommunityInfo",
"KGExtraction",
"Triple",
"Relationship",
"EntityLevel",
# Index abstractions
"IndexConfig",
Expand Down
44 changes: 36 additions & 8 deletions py/core/base/api/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,36 @@
WrappedUpdateResponse,
)
from shared.api.models.kg.responses import (
KGCreationEstimationResponse,
KGCreationResponse,
KGDeduplicationEstimationResponse,
KGEnrichmentEstimationResponse,
KGEnrichmentResponse,
KGEntityDeduplicationResponse,
WrappedKGCommunitiesResponse,
WrappedKGCreationResponse,
WrappedKGEnrichmentResponse,
WrappedKGEntitiesResponse,
WrappedKGEntityDeduplicationResponse,
WrappedKGTriplesResponse,
WrappedKGRelationshipsResponse,
WrappedKGTunePromptResponse,
)


from shared.api.models.kg.responses_v3 import (
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's fine to get ride of the validation on the v2 endpoints—it's there for legacy, and should work, but, no need to have two versions of this for legacy code.

WrappedKGEntitiesResponse as WrappedKGEntitiesResponseV3,
WrappedKGRelationshipsResponse as WrappedKGRelationshipsResponseV3,
WrappedKGCommunitiesResponse as WrappedKGCommunitiesResponseV3,
WrappedKGCreationResponse as WrappedKGCreationResponseV3,
WrappedKGEnrichmentResponse as WrappedKGEnrichmentResponseV3,
WrappedKGTunePromptResponse as WrappedKGTunePromptResponseV3,
WrappedKGEntityDeduplicationResponse as WrappedKGEntityDeduplicationResponseV3,
WrappedKGDeletionResponse as WrappedKGDeletionResponseV3,
KGCreationResponse as KGCreationResponseV3,
KGEnrichmentResponse as KGEnrichmentResponseV3,
KGEntityDeduplicationResponse as KGEntityDeduplicationResponseV3,
KGTunePromptResponse as KGTunePromptResponseV3,
KGDeletionResponse as KGDeletionResponseV3,
)


from shared.api.models.management.responses import (
AnalyticsResponse,
AppSettingsResponse,
Expand Down Expand Up @@ -96,17 +112,29 @@
"WrappedMetadataUpdateResponse",
"WrappedListVectorIndicesResponse",
"UpdateResponse",
# Knowledge Graph Responses
# Knowledge Graph Responses for V2
# will be removed eventually
"KGCreationResponse",
"WrappedKGCreationResponse",
"KGEnrichmentResponse",
"WrappedKGEnrichmentResponse",
"KGEntityDeduplicationResponse",
"WrappedKGEntityDeduplicationResponse",
"WrappedKGTunePromptResponse",
"KGCreationEstimationResponse",
"KGDeduplicationEstimationResponse",
"KGEnrichmentEstimationResponse",
# Knowledge Graph Responses for V3
"WrappedKGEntitiesResponseV3",
"WrappedKGRelationshipsResponseV3",
"WrappedKGCommunitiesResponseV3",
"WrappedKGCreationResponseV3",
"WrappedKGEnrichmentResponseV3",
"WrappedKGTunePromptResponseV3",
"WrappedKGEntityDeduplicationResponseV3",
"KGCreationResponseV3",
"KGEnrichmentResponseV3",
"KGEntityDeduplicationResponseV3",
"KGTunePromptResponseV3",
"WrappedKGDeletionResponseV3",
"KGDeletionResponseV3",
# Management Responses
"PromptResponse",
"ServerStats",
Expand Down
4 changes: 2 additions & 2 deletions py/core/base/providers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
DatabaseProvider,
DocumentHandler,
FileHandler,
KGHandler,
GraphHandler,
LoggingHandler,
PostgresConfigurationSettings,
PromptHandler,
Expand Down Expand Up @@ -48,7 +48,7 @@
"UserHandler",
"LoggingHandler",
"VectorHandler",
"KGHandler",
"GraphHandler",
"PromptHandler",
"FileHandler",
"DatabaseConfig",
Expand Down
Loading
Loading