diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..5008a95 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,102 @@ +**@accordproject/concerto-graph** • **Docs** + +*** + +--- +title: Concerto Graph +description: Store Concerto Concepts in a Graph DB +tags: + - Concerto + - Neo4J + - Graph +--- + +# Concerto Graph + +This project uses a [Concerto model](https://concerto.accordproject.org) to define the nodes and edges in a Neo4J graph database and uses the model to validate the properties on the nodes. + +![demo](demo.png) +[Demo](src/demo/index.ts) + +In a few lines of code you can define a Concerto data model validated graph and perform a vector similarity search over +nodes with text content. + +Concerto model (snippet): + +``` +concept Movie extends GraphNode { + @vector_index("summary", 1536, "COSINE") + o Double[] embedding optional + @embedding + o String summary optional + @label("IN_GENRE") + --> Genre[] genres optional +} +``` + +TypeScript code: + +```typescript + await graphModel.mergeNode(transaction, `${NS}.Movie`, {identifier: 'Brazil', summary: 'The film centres on Sam Lowry, a low-ranking bureaucrat trying to find a woman who appears in his dreams while he is working in a mind-numbing job and living in a small apartment, set in a dystopian world in which there is an over-reliance on poorly maintained (and rather whimsical) machines'} ); + + await graphModel.mergeNode(transaction, `${NS}.Genre`, {identifier: 'Comedy'} ); + + await graphModel.mergeRelationship(transaction, `${NS}.Movie`, 'Brazil', `${NS}.Genre`, 'Comedy', 'genres' ); + + await graphModel.mergeNode(transaction, `${NS}.Director`, {identifier: 'Terry Gilliam'} ); + await graphModel.mergeRelationship(transaction, `${NS}.Director`, 'Terry Gilliam', `${NS}.Movie`, 'Brazil', 'directed' ); + + await graphModel.mergeNode(transaction, `${NS}.Actor`, {identifier: 'Jonathan Pryce'} ); + await graphModel.mergeRelationship(transaction, `${NS}.Actor`, 'Jonathan Pryce', `${NS}.Movie`, 'Brazil', 'actedIn' ); + + const search = 'Working in a boring job and looking for love.'; + const results = await graphModel.similarityQuery(`${NS}.Movie`, 'embedding', search, 3); +``` + +Runtime result: + +```json +[ + { + identifier: 'Brazil', + content: 'The film centres on Sam Lowry, a low-ranking bureaucrat trying to find a woman who appears in his dreams while he is working in a mind-numbing job and living in a small apartment, set in a dystopian world in which there is an over-reliance on poorly maintained (and rather whimsical) machines', + score: 0.901830792427063 + } +] +``` + +## Environment Variables + +### GraphDB + +- NEO4J_URL: the NEO4J URL. E.g. `neo4j+s://.databases.neo4j.io` if you are using AuraDB. +- NEO4J_PASS: your neo4j password. +- NEO4J_USER: defaults to `neo4j` + +### Text Embeddings +- OPENAI_API_KEY: the OpenAI API key. If not set embeddings are not computed and written to the agreement graph and similarity search, natural language to Cypher generation ("chat with data") is not possible. + +## API Index + +### Classes + +- [GraphModel](classes/GraphModel.md) + +### Type Aliases + +- [Context](type-aliases/Context.md) +- [GraphModelOptions](type-aliases/GraphModelOptions.md) +- [GraphNodeProperties](type-aliases/GraphNodeProperties.md) +- [SimilarityResult](type-aliases/SimilarityResult.md) + +### Variables + +- [ROOT\_MODEL](variables/ROOT_MODEL.md) +- [ROOT\_NAMESPACE](variables/ROOT_NAMESPACE.md) + +### Functions + +- [getObjectChecksum](functions/getObjectChecksum.md) +- [getOpenAiEmbedding](functions/getOpenAiEmbedding.md) +- [getTextChecksum](functions/getTextChecksum.md) +- [textToCypher](functions/textToCypher.md) diff --git a/docs/classes/GraphModel.md b/docs/classes/GraphModel.md new file mode 100644 index 0000000..32da27c --- /dev/null +++ b/docs/classes/GraphModel.md @@ -0,0 +1,655 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / GraphModel + +# Class: GraphModel + +Provides typed-access to Neo4J graph database +with the nodes and relationships for the graph defined +using a Concerto model. + +## Constructors + +### new GraphModel() + +> **new GraphModel**(`graphModels`, `options`): [`GraphModel`](GraphModel.md) + +Creates a new instance of GraphModel + +#### Parameters + +• **graphModels**: `string`[] + +an array of strings in Concerto CTO format + +• **options**: [`GraphModelOptions`](../type-aliases/GraphModelOptions.md) + +the options used to configure the instance + +#### Returns + +[`GraphModel`](GraphModel.md) + +#### Source + +[graphmodel.ts:252](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L252) + +## Properties + +### defaultNamespace + +> **defaultNamespace**: `undefined` \| `string` + +#### Source + +[graphmodel.ts:245](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L245) + +*** + +### driver + +> **driver**: `undefined` \| `Driver` = `undefined` + +#### Source + +[graphmodel.ts:243](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L243) + +*** + +### modelManager + +> **modelManager**: `ModelManager` + +#### Source + +[graphmodel.ts:242](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L242) + +*** + +### options + +> **options**: [`GraphModelOptions`](../type-aliases/GraphModelOptions.md) + +#### Source + +[graphmodel.ts:244](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L244) + +## Methods + +### chatWithData() + +> **chatWithData**(`text`): `Promise`\<`object`[]\> + +Converts the incoming natural language query to Cypher and then +runs the Cypher query. + +#### Parameters + +• **text**: `string` + +the input text + +#### Returns + +`Promise`\<`object`[]\> + +the query results + +#### Source + +[graphmodel.ts:662](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L662) + +*** + +### closeSession() + +> **closeSession**(`context`): `Promise`\<`void`\> + +Closes a database context. + +#### Parameters + +• **context**: [`Context`](../type-aliases/Context.md) + +the database context + +#### Returns + +`Promise`\<`void`\> + +#### Source + +[graphmodel.ts:295](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L295) + +*** + +### connect() + +> **connect**(): `Promise`\<`void`\> + +Connects to Neo4J + +#### Returns + +`Promise`\<`void`\> + +#### Source + +[graphmodel.ts:266](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L266) + +*** + +### createConstraints() + +> **createConstraints**(): `Promise`\<`void`\> + +Create Neo4J constraints for the model + +#### Returns + +`Promise`\<`void`\> + +#### Source + +[graphmodel.ts:410](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L410) + +*** + +### createFullTextIndexes() + +> **createFullTextIndexes**(): `Promise`\<`void`\> + +#### Returns + +`Promise`\<`void`\> + +#### Source + +[graphmodel.ts:447](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L447) + +*** + +### createVectorIndexes() + +> **createVectorIndexes**(): `Promise`\<`void`\> + +Create vector indexes for the model + +#### Returns + +`Promise`\<`void`\> + +#### Source + +[graphmodel.ts:427](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L427) + +*** + +### deleteGraph() + +> **deleteGraph**(): `Promise`\<`void`\> + +Delete all nodes/edges in the graph + +#### Returns + +`Promise`\<`void`\> + +#### Source + +[graphmodel.ts:469](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L469) + +*** + +### dropIndexes() + +> **dropIndexes**(): `Promise`\<`void`\> + +Drop all Neo4J indexes for the model. + +#### Returns + +`Promise`\<`void`\> + +#### Source + +[graphmodel.ts:382](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L382) + +*** + +### fullTextQuery() + +> **fullTextQuery**(`typeName`, `searchText`, `count`): `Promise`\<`object`[]\> + +Uses the full text index for a type to perform a full text search + +#### Parameters + +• **typeName**: `string` + +the type to search + +• **searchText**: `string` + +the query text + +• **count**: `number` + +the number of items to return + +#### Returns + +`Promise`\<`object`[]\> + +the items + +#### Source + +[graphmodel.ts:693](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L693) + +*** + +### getFullTextIndex() + +> `private` **getFullTextIndex**(`decl`): `undefined` \| `FullTextIndex` + +#### Parameters + +• **decl**: `any` + +#### Returns + +`undefined` \| `FullTextIndex` + +#### Source + +[graphmodel.ts:323](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L323) + +*** + +### getFullTextIndexName() + +> `private` **getFullTextIndexName**(`decl`): `string` + +#### Parameters + +• **decl**: `any` + +#### Returns + +`string` + +#### Source + +[graphmodel.ts:375](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L375) + +*** + +### getFullyQualifiedType() + +> `private` **getFullyQualifiedType**(`type`): `string` + +#### Parameters + +• **type**: `string` + +#### Returns + +`string` + +#### Source + +[graphmodel.ts:299](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L299) + +*** + +### getGraphNodeDeclaration() + +> `private` **getGraphNodeDeclaration**(`typeName`): `ClassDeclaration` + +#### Parameters + +• **typeName**: `string` + +#### Returns + +`ClassDeclaration` + +#### Source + +[graphmodel.ts:306](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L306) + +*** + +### getGraphNodeDeclarations() + +> `private` **getGraphNodeDeclarations**(): `ClassDeclaration`[] + +#### Returns + +`ClassDeclaration`[] + +#### Source + +[graphmodel.ts:316](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L316) + +*** + +### getPropertyVectorIndex() + +> `private` **getPropertyVectorIndex**(`property`): `VectorIndex` + +#### Parameters + +• **property**: `any` + +#### Returns + +`VectorIndex` + +#### Source + +[graphmodel.ts:330](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L330) + +*** + +### getPropertyVectorIndexName() + +> `private` **getPropertyVectorIndexName**(`decl`, `vectorProperty`): `string` + +#### Parameters + +• **decl**: `any` + +• **vectorProperty**: `any` + +#### Returns + +`string` + +#### Source + +[graphmodel.ts:371](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L371) + +*** + +### mergeEmbeddingCacheNode() + +> `private` **mergeEmbeddingCacheNode**(`transaction`, `text`): `Promise`\<`EmbeddingCacheNode`\> + +We use the EmbeddingCacheNode GraphNode as a cache to ensure deterministic +embeddings for the same text, and to cut down on OpenAI API calls + +#### Parameters + +• **transaction**: `any` + +the transaction + +• **text**: `string` + +the text to cache + +#### Returns + +`Promise`\<`EmbeddingCacheNode`\> + +a promise to the EmbeddingCacheNode + +#### Source + +[graphmodel.ts:599](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L599) + +*** + +### mergeNode() + +> **mergeNode**(`transaction`, `typeName`, `properties`): `Promise`\<`QueryResult`\<`RecordShape`\>\> + +Merges Nodes into the graph. +Note that this merges nodes based on identifier. I.e. if a node with a given +identifier already exists then its properties are SET. If the node does not exist +then a new node is created. + +#### Parameters + +• **transaction**: `ManagedTransaction` + +the transaction + +• **typeName**: `string` + +the name of the type + +• **properties**: `PropertyBag` + +the properties for the node + +#### Returns + +`Promise`\<`QueryResult`\<`RecordShape`\>\> + +the graph node + +#### Source + +[graphmodel.ts:543](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L543) + +*** + +### mergeRelationship() + +> **mergeRelationship**(`transaction`, `sourceType`, `sourceIdentifier`, `targetType`, `targetIdentifier`, `sourcePropertyName`): `Promise`\<`QueryResult`\<`RecordShape`\>\> + +Merges a relationship into the graph + +#### Parameters + +• **transaction**: `ManagedTransaction` + +the transaction + +• **sourceType**: `string` + +the source node type of the relationship + +• **sourceIdentifier**: `string` + +the source identifier for the relationship + +• **targetType**: `string` + +the target node type + +• **targetIdentifier**: `string` + +the target identifier + +• **sourcePropertyName**: `string` + +the source property name + +#### Returns + +`Promise`\<`QueryResult`\<`RecordShape`\>\> + +the source node + +#### Source + +[graphmodel.ts:569](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L569) + +*** + +### openSession() + +> **openSession**(`database`): `Promise`\<[`Context`](../type-aliases/Context.md)\> + +Opens a new database session. Call 'closeSession' to +free resources. + +#### Parameters + +• **database**: `string`= `'neo4j'` + +the name of the database. Defaults to 'neo4j'. + +#### Returns + +`Promise`\<[`Context`](../type-aliases/Context.md)\> + +a promise to a Context for the database. + +#### Source + +[graphmodel.ts:283](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L283) + +*** + +### query() + +> **query**(`cypher`, `parameters`?, `tx`?): `Promise`\<`QueryResult`\<`RecordShape`\>\> + +Runs a Cypher query + +#### Parameters + +• **cypher**: `string` + +the Cypher query to execute + +• **parameters?**: `PropertyBag` + +any parameters for the query + +• **tx?**: `ManagedTransaction` + +the transaction + +#### Returns + +`Promise`\<`QueryResult`\<`RecordShape`\>\> + +the query results + +#### Source + +[graphmodel.ts:516](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L516) + +*** + +### similarityQuery() + +> **similarityQuery**(`typeName`, `propertyName`, `searchText`, `count`): `Promise`\<[`SimilarityResult`](../type-aliases/SimilarityResult.md)[]\> + +Searches for similar nodes, using a vector similarity search + +#### Parameters + +• **typeName**: `string` + +the name of the type + +• **propertyName**: `string` + +the property to search over + +• **searchText**: `string` + +the search text + +• **count**: `number` + +the number of items to return + +#### Returns + +`Promise`\<[`SimilarityResult`](../type-aliases/SimilarityResult.md)[]\> + +an array of similar nodes, up to the count limit + +#### Source + +[graphmodel.ts:628](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L628) + +*** + +### similarityQueryFromEmbedding() + +> **similarityQueryFromEmbedding**(`typeName`, `propertyName`, `embedding`, `count`): `Promise`\<[`SimilarityResult`](../type-aliases/SimilarityResult.md)[]\> + +Performs a similarity search on nodes with text content + +#### Parameters + +• **typeName**: `string` + +the name of the type E.g. 'Clause' + +• **propertyName**: `string` + +the name of the property to search. E.g. 'content'. + +• **embedding**: `number`[] + +the embeddings for the text to search for + +• **count**: `number` + +the number of similar nodes to return + +#### Returns + +`Promise`\<[`SimilarityResult`](../type-aliases/SimilarityResult.md)[]\> + +#### Source + +[graphmodel.ts:485](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L485) + +*** + +### textToCypher() + +> **textToCypher**(`text`): `Promise`\<`null` \| `string`\> + +Converts a natural language query string to a Cypher query + +#### Parameters + +• **text**: `string` + +the input text + +#### Returns + +`Promise`\<`null` \| `string`\> + +the Cypher query + +#### Source + +[graphmodel.ts:651](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L651) + +*** + +### validateAndTransformProperties() + +> `private` **validateAndTransformProperties**(`transaction`, `decl`, `properties`): `Promise`\<`PropertyBag`\> + +#### Parameters + +• **transaction**: `any` + +• **decl**: `ClassDeclaration` + +• **properties**: `PropertyBag` + +#### Returns + +`Promise`\<`PropertyBag`\> + +#### Source + +[graphmodel.ts:721](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L721) diff --git a/docs/functions/getObjectChecksum.md b/docs/functions/getObjectChecksum.md new file mode 100644 index 0000000..6831181 --- /dev/null +++ b/docs/functions/getObjectChecksum.md @@ -0,0 +1,27 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / getObjectChecksum + +# Function: getObjectChecksum() + +> **getObjectChecksum**(`obj`): `string` + +Computes a deterministic identifier for a set of properties. + +## Parameters + +• **obj**: `PropertyBag` + +the properties of an object + +## Returns + +`string` + +the identifier as a string + +## Source + +[graphmodel.ts:165](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L165) diff --git a/docs/functions/getOpenAiEmbedding.md b/docs/functions/getOpenAiEmbedding.md new file mode 100644 index 0000000..35a8aea --- /dev/null +++ b/docs/functions/getOpenAiEmbedding.md @@ -0,0 +1,28 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / getOpenAiEmbedding + +# Function: getOpenAiEmbedding() + +> **getOpenAiEmbedding**(`text`): `Promise`\<`number`[]\> + +Computes the vector embeddings for a text string. +Uses the Open AI `text-embedding-3-small` model. + +## Parameters + +• **text**: `string` + +the input text to compute embeddings for + +## Returns + +`Promise`\<`number`[]\> + +a promise to an array of numbers + +## Source + +[graphmodel.ts:53](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L53) diff --git a/docs/functions/getTextChecksum.md b/docs/functions/getTextChecksum.md new file mode 100644 index 0000000..fbdcaec --- /dev/null +++ b/docs/functions/getTextChecksum.md @@ -0,0 +1,27 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / getTextChecksum + +# Function: getTextChecksum() + +> **getTextChecksum**(`text`): `string` + +Computes a SHA256 checksum for input text + +## Parameters + +• **text**: `string` + +the input text + +## Returns + +`string` + +the checksum + +## Source + +[graphmodel.ts:178](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L178) diff --git a/docs/functions/textToCypher.md b/docs/functions/textToCypher.md new file mode 100644 index 0000000..e68eec0 --- /dev/null +++ b/docs/functions/textToCypher.md @@ -0,0 +1,35 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / textToCypher + +# Function: textToCypher() + +> **textToCypher**(`options`, `text`, `ctoModel`): `Promise`\<`string` \| `null`\> + +Converts a natural language query string to a Neo4J Cypher query. + +## Parameters + +• **options**: [`GraphModelOptions`](../type-aliases/GraphModelOptions.md) + +configure logger and embedding function + +• **text**: `string` + +the input text to convert to Cypher + +• **ctoModel**: `any` + +the text of all CTO models, used to configure Cypher generation + +## Returns + +`Promise`\<`string` \| `null`\> + +a promise to the Cypher query or null + +## Source + +[graphmodel.ts:70](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L70) diff --git a/docs/type-aliases/Context.md b/docs/type-aliases/Context.md new file mode 100644 index 0000000..58697bc --- /dev/null +++ b/docs/type-aliases/Context.md @@ -0,0 +1,21 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / Context + +# Type alias: Context + +> **Context**: `object` + +Runtime context + +## Type declaration + +### session + +> **session**: `Session` + +## Source + +[graphmodel.ts:25](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L25) diff --git a/docs/type-aliases/GraphModelOptions.md b/docs/type-aliases/GraphModelOptions.md new file mode 100644 index 0000000..bbb8a35 --- /dev/null +++ b/docs/type-aliases/GraphModelOptions.md @@ -0,0 +1,41 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / GraphModelOptions + +# Type alias: GraphModelOptions + +> **GraphModelOptions**: `object` + +Graph model options, used to configure Concerto Graph + +## Type declaration + +### NEO4J\_PASS? + +> `optional` **NEO4J\_PASS**: `string` + +### NEO4J\_URL? + +> `optional` **NEO4J\_URL**: `string` + +### NEO4J\_USER? + +> `optional` **NEO4J\_USER**: `string` + +### embeddingFunction? + +> `optional` **embeddingFunction**: `EmbeddingFunction` + +### logQueries? + +> `optional` **logQueries**: `boolean` + +### logger? + +> `optional` **logger**: `Logger` + +## Source + +[graphmodel.ts:208](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L208) diff --git a/docs/type-aliases/GraphNodeProperties.md b/docs/type-aliases/GraphNodeProperties.md new file mode 100644 index 0000000..9a169b7 --- /dev/null +++ b/docs/type-aliases/GraphNodeProperties.md @@ -0,0 +1,15 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / GraphNodeProperties + +# Type alias: GraphNodeProperties + +> **GraphNodeProperties**: `PropertyBag` + +The properties allowed on graph nodes + +## Source + +[graphmodel.ts:190](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L190) diff --git a/docs/type-aliases/SimilarityResult.md b/docs/type-aliases/SimilarityResult.md new file mode 100644 index 0000000..fc161d6 --- /dev/null +++ b/docs/type-aliases/SimilarityResult.md @@ -0,0 +1,29 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / SimilarityResult + +# Type alias: SimilarityResult + +> **SimilarityResult**: `object` + +Result of a vector similarity search + +## Type declaration + +### content + +> **content**: `string` + +### identifier + +> **identifier**: `string` + +### score + +> **score**: `number` + +## Source + +[graphmodel.ts:41](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L41) diff --git a/docs/variables/ROOT_MODEL.md b/docs/variables/ROOT_MODEL.md new file mode 100644 index 0000000..d54a7d7 --- /dev/null +++ b/docs/variables/ROOT_MODEL.md @@ -0,0 +1,15 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / ROOT\_MODEL + +# Variable: ROOT\_MODEL + +> `const` **ROOT\_MODEL**: "namespace org.accordproject.graph@1.0.0\nconcept GraphNode identified by identifier \{\n o String identifier\n\}\nconcept EmbeddingCacheNode extends GraphNode \{\n o Double\[\] embedding\n @vector\_index(\"embedding\", 1536, \"COSINE\")\n o String content \n\}\n" + +The concerto graph model, defines internal nodes + +## Source + +[graphmodel.ts:225](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L225) diff --git a/docs/variables/ROOT_NAMESPACE.md b/docs/variables/ROOT_NAMESPACE.md new file mode 100644 index 0000000..fbf0107 --- /dev/null +++ b/docs/variables/ROOT_NAMESPACE.md @@ -0,0 +1,15 @@ +[**@accordproject/concerto-graph**](../README.md) • **Docs** + +*** + +[@accordproject/concerto-graph](../README.md) / ROOT\_NAMESPACE + +# Variable: ROOT\_NAMESPACE + +> `const` **ROOT\_NAMESPACE**: `"org.accordproject.graph@1.0.0"` = `'org.accordproject.graph@1.0.0'` + +The concerto graph namespaces, used for internal nodes + +## Source + +[graphmodel.ts:220](https://github.com/accordproject/lab-concerto-graph/blob/7f2e9294ea86dce21442f2458a6ff685a4437085/src/graphmodel.ts#L220) diff --git a/package-lock.json b/package-lock.json index 93c5b20..cb192fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@accordproject/concerto-graph", - "version": "1.0.2", + "version": "1.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@accordproject/concerto-graph", - "version": "1.0.2", + "version": "1.1.0", "license": "Apache-2.0", "dependencies": { "@accordproject/concerto-core": "^3.16.8", @@ -36,6 +36,8 @@ "semver": "^7.5.4", "ts-jest": "^29.1.1", "ts-node": "^10.9.2", + "typedoc": "^0.25.13", + "typedoc-plugin-markdown": "^4.0.2", "typescript": "^5.3.3" } }, @@ -3232,6 +3234,12 @@ "node": ">=8" } }, + "node_modules/ansi-sequence-parser": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ansi-sequence-parser/-/ansi-sequence-parser-1.1.1.tgz", + "integrity": "sha512-vJXt3yiaUL4UU546s3rPXlsry/RnM730G1+HkpKE012AN0sx1eOrxSu95oKDIonskeLTijMgqWZ3uDEe3NFvyg==", + "dev": true + }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -6548,6 +6556,12 @@ "json5": "lib/cli.js" } }, + "node_modules/jsonc-parser": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.1.tgz", + "integrity": "sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA==", + "dev": true + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -6658,6 +6672,12 @@ "node": ">=10" } }, + "node_modules/lunr": { + "version": "2.3.9", + "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", + "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", + "dev": true + }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -6688,6 +6708,18 @@ "tmpl": "1.0.5" } }, + "node_modules/marked": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", + "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", + "dev": true, + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 12" + } + }, "node_modules/md5": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", @@ -7777,6 +7809,18 @@ "node": ">=8" } }, + "node_modules/shiki": { + "version": "0.14.7", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.7.tgz", + "integrity": "sha512-dNPAPrxSc87ua2sKJ3H5dQ/6ZaY8RNnaAqK+t0eG7p0Soi2ydiqbGOTaZCqaYvA/uZYfS1LJnemt3Q+mSfcPCg==", + "dev": true, + "dependencies": { + "ansi-sequence-parser": "^1.1.0", + "jsonc-parser": "^3.2.0", + "vscode-oniguruma": "^1.7.0", + "vscode-textmate": "^8.0.0" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -8326,6 +8370,60 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typedoc": { + "version": "0.25.13", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.25.13.tgz", + "integrity": "sha512-pQqiwiJ+Z4pigfOnnysObszLiU3mVLWAExSPf+Mu06G/qsc3wzbuM56SZQvONhHLncLUhYzOVkjFFpFfL5AzhQ==", + "dev": true, + "dependencies": { + "lunr": "^2.3.9", + "marked": "^4.3.0", + "minimatch": "^9.0.3", + "shiki": "^0.14.7" + }, + "bin": { + "typedoc": "bin/typedoc" + }, + "engines": { + "node": ">= 16" + }, + "peerDependencies": { + "typescript": "4.6.x || 4.7.x || 4.8.x || 4.9.x || 5.0.x || 5.1.x || 5.2.x || 5.3.x || 5.4.x" + } + }, + "node_modules/typedoc-plugin-markdown": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/typedoc-plugin-markdown/-/typedoc-plugin-markdown-4.0.2.tgz", + "integrity": "sha512-4MV3M+0lsmIaXuDBzeqLYemZqwTQDWQow+o8zdT9hC7KFu06GaFo2uUEbkjE6pgZA9hnkOTtzRVd0R9YJWcH8A==", + "dev": true, + "peerDependencies": { + "typedoc": "0.25.x" + } + }, + "node_modules/typedoc/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/typedoc/node_modules/minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/typescript": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", @@ -8481,6 +8579,18 @@ "node": ">=10.12.0" } }, + "node_modules/vscode-oniguruma": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", + "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==", + "dev": true + }, + "node_modules/vscode-textmate": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-8.0.0.tgz", + "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==", + "dev": true + }, "node_modules/walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", diff --git a/package.json b/package.json index 8f01495..ce49f29 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "prestart": "npm run build", "start": "node dist/src/demo/index.js", "test": "jest", - "coverage": "npx jest --coverage" + "coverage": "npx jest --coverage", + "docs": "typedoc" }, "devDependencies": { "@babel/core": "^7.23.6", @@ -34,6 +35,8 @@ "semver": "^7.5.4", "ts-jest": "^29.1.1", "ts-node": "^10.9.2", + "typedoc": "^0.25.13", + "typedoc-plugin-markdown": "^4.0.2", "typescript": "^5.3.3" }, "dependencies": { diff --git a/src/graphmodel.ts b/src/graphmodel.ts index 55cda20..ca38691 100644 --- a/src/graphmodel.ts +++ b/src/graphmodel.ts @@ -3,32 +3,53 @@ import neo4j, { DateTime, Driver, ManagedTransaction, Session } from 'neo4j-driv import * as crypto from 'crypto' import OpenAI from "openai"; +/** + * Definition of a vector (embeddings) index + */ type VectorIndex = { property: string; size: number; type: string; } +/** + * Definition of a full text index over some properties + */ type FullTextIndex = { properties: Array; } +/** + * Runtime context + */ export type Context = { session: Session; } +/** + * A Node type that is used to cache vector embeddings + */ type EmbeddingCacheNode = { $class: string; embedding: Array; content: string; } +/** + * Result of a vector similarity search + */ export type SimilarityResult = { identifier: string; content: string; score: number; } +/** + * Computes the vector embeddings for a text string. + * Uses the Open AI `text-embedding-3-small` model. + * @param text the input text to compute embeddings for + * @returns a promise to an array of numbers + */ export async function getOpenAiEmbedding(text: string): Promise> { const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); const response = await openai.embeddings.create({ @@ -39,6 +60,13 @@ export async function getOpenAiEmbedding(text: string): Promise> { return response.data[0].embedding; } +/** + * Converts a natural language query string to a Neo4J Cypher query. + * @param options configure logger and embedding function + * @param text the input text to convert to Cypher + * @param ctoModel the text of all CTO models, used to configure Cypher generation + * @returns a promise to the Cypher query or null + */ export async function textToCypher(options:GraphModelOptions, text: string, ctoModel): Promise { const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); @@ -129,7 +157,11 @@ Natural language query: """${text} return chatCompletion.choices[0].message.content; } - +/** + * Computes a deterministic identifier for a set of properties. + * @param obj the properties of an object + * @returns the identifier as a string + */ export function getObjectChecksum(obj: PropertyBag) { const deterministicReplacer = (_, v) => typeof v !== 'object' || v === null || Array.isArray(v) ? v : @@ -138,16 +170,41 @@ export function getObjectChecksum(obj: PropertyBag) { return crypto.createHash('sha256').update(JSON.stringify(obj, deterministicReplacer)).digest('hex') } +/** + * Computes a SHA256 checksum for input text + * @param text the input text + * @returns the checksum + */ export function getTextChecksum(text: string) { return crypto.createHash('sha256').update(text).digest('hex') } +/** + * A untyped set of properties + */ type PropertyBag = Record; + +/** + * The properties allowed on graph nodes + */ export type GraphNodeProperties = PropertyBag; + +/** + * Function signature for a function that can calculate + * vector embeddings for text + */ type EmbeddingFunction = (text: string) => Promise>; + +/** + * Function signature for a logger + */ type Logger = { log: (text: string) => void; } + +/** + * Graph model options, used to configure Concerto Graph + */ export type GraphModelOptions = { embeddingFunction?: EmbeddingFunction; logger?: Logger; @@ -157,7 +214,14 @@ export type GraphModelOptions = { logQueries?: boolean; } +/** + * The concerto graph namespaces, used for internal nodes + */ export const ROOT_NAMESPACE = 'org.accordproject.graph@1.0.0'; + +/** + * The concerto graph model, defines internal nodes + */ export const ROOT_MODEL = `namespace ${ROOT_NAMESPACE} concept GraphNode identified by identifier { o String identifier @@ -180,6 +244,11 @@ export class GraphModel { options: GraphModelOptions; defaultNamespace: string | undefined; + /** + * Creates a new instance of GraphModel + * @param graphModels an array of strings in Concerto CTO format + * @param options the options used to configure the instance + */ constructor(graphModels: Array, options: GraphModelOptions) { this.options = options; this.modelManager = new ModelManager({ strict: true, enableMapType: true }); @@ -191,6 +260,9 @@ export class GraphModel { this.modelManager.validateModelFiles(); } + /** + * Connects to Neo4J + */ async connect() { if (!this.driver) { this.driver = neo4j.driver(this.options.NEO4J_URL ?? 'bolt://localhost:7687', @@ -201,6 +273,13 @@ export class GraphModel { } } + /** + * Opens a new database session. Call 'closeSession' to + * free resources. + * + * @param database the name of the database. Defaults to 'neo4j'. + * @returns a promise to a Context for the database. + */ async openSession(database = 'neo4j'): Promise { if (this.driver) { const session = this.driver.session({ database }) @@ -209,10 +288,14 @@ export class GraphModel { throw new Error('No neo4j driver!'); } + /** + * Closes a database context. + * @param context the database context + */ async closeSession(context: Context) { context.session.close(); } - + private getFullyQualifiedType(type: string) { const typeNs = ModelUtil.getNamespace(type); const typeShortName = ModelUtil.getShortName(type); @@ -293,6 +376,9 @@ export class GraphModel { return `${decl.getName()}_fulltext`.toLowerCase(); } + /** + * Drop all Neo4J indexes for the model. + */ async dropIndexes() { this.options.logger?.log('Dropping indexes...'); const { session } = await this.openSession(); @@ -318,6 +404,9 @@ export class GraphModel { this.options.logger?.log('Drop indexes completed'); } + /** + * Create Neo4J constraints for the model + */ async createConstraints() { this.options.logger?.log('Creating constraints...'); const { session } = await this.openSession(); @@ -332,6 +421,9 @@ export class GraphModel { this.options.logger?.log('Create constraints completed'); } + /** + * Create vector indexes for the model + */ async createVectorIndexes() { this.options.logger?.log('Creating vector indexes...'); const { session } = await this.openSession(); @@ -371,6 +463,9 @@ export class GraphModel { this.options.logger?.log('Create full text indexes completed'); } + /** + * Delete all nodes/edges in the graph + */ async deleteGraph() { const { session } = await this.openSession(); await session.executeWrite(async tx => { @@ -384,10 +479,10 @@ export class GraphModel { * @param typeName the name of the type E.g. 'Clause' * @param propertyName the name of the property to search. E.g. 'content'. * @param count the number of similar nodes to return - * @param embeddings the embeddings for the text to search for + * @param embedding the embeddings for the text to search for * @returns */ - async similarityQueryFromEmbedding(typeName: string, propertyName: string, embedding, count: number): Promise> { + async similarityQueryFromEmbedding(typeName: string, propertyName: string, embedding:Array, count: number): Promise> { const decl = this.getGraphNodeDeclaration(typeName); const vectorProperty = decl.getProperty(propertyName); if (!vectorProperty) { @@ -411,6 +506,13 @@ export class GraphModel { }) : []; } + /** + * Runs a Cypher query + * @param cypher the Cypher query to execute + * @param parameters any parameters for the query + * @param tx the transaction + * @returns the query results + */ async query(cypher: string, parameters?: PropertyBag, tx?: ManagedTransaction) { if (this.options.logQueries) { this.options.logger?.log(cypher); @@ -429,13 +531,14 @@ export class GraphModel { } /** + * Merges Nodes into the graph. * Note that this merges nodes based on identifier. I.e. if a node with a given * identifier already exists then its properties are SET. If the node does not exist * then a new node is created. - * @param transaction - * @param typeName - * @param properties - * @returns + * @param transaction the transaction + * @param typeName the name of the type + * @param properties the properties for the node + * @returns the graph node */ async mergeNode(transaction: ManagedTransaction, typeName: string, properties: PropertyBag) { const decl = this.getGraphNodeDeclaration(typeName); @@ -453,6 +556,16 @@ export class GraphModel { return this.query(`MERGE (n:${decl.getName()}{identifier: $id}) ${set}`, { id, ...newProperties }, transaction); } + /** + * Merges a relationship into the graph + * @param transaction the transaction + * @param sourceType the source node type of the relationship + * @param sourceIdentifier the source identifier for the relationship + * @param targetType the target node type + * @param targetIdentifier the target identifier + * @param sourcePropertyName the source property name + * @returns the source node + */ async mergeRelationship(transaction: ManagedTransaction, sourceType: string, sourceIdentifier: string, targetType: string, targetIdentifier: string, sourcePropertyName: string) { const sourceDecl = this.getGraphNodeDeclaration(sourceType); @@ -479,9 +592,9 @@ export class GraphModel { /** * We use the EmbeddingCacheNode GraphNode as a cache to ensure deterministic * embeddings for the same text, and to cut down on OpenAI API calls - * @param transaction - * @param text - * @returns Promise + * @param transaction the transaction + * @param text the text to cache + * @returns a promise to the EmbeddingCacheNode */ private async mergeEmbeddingCacheNode(transaction, text: string): Promise { const embeddingId = getTextChecksum(text); @@ -504,6 +617,14 @@ export class GraphModel { } } + /** + * Searches for similar nodes, using a vector similarity search + * @param typeName the name of the type + * @param propertyName the property to search over + * @param searchText the search text + * @param count the number of items to return + * @returns an array of similar nodes, up to the count limit + */ async similarityQuery(typeName: string, propertyName: string, searchText: string, count: number): Promise> { const context = await this.openSession(); const transaction = await context.session.beginTransaction(); @@ -522,11 +643,22 @@ export class GraphModel { } } + /** + * Converts a natural language query string to a Cypher query + * @param text the input text + * @returns the Cypher query + */ async textToCypher(text: string): Promise { const ctoModels = this.modelManager.getModels().reduce((prev, cur) => prev += cur.content, ''); return textToCypher(this.options, text, ctoModels); } + /** + * Converts the incoming natural language query to Cypher and then + * runs the Cypher query. + * @param text the input text + * @returns the query results + */ async chatWithData(text: string) { const cypher = await this.textToCypher(text); if (cypher) { @@ -551,6 +683,13 @@ export class GraphModel { throw new Error(`Failed to convert to Cypher query ${text}`); } + /** + * Uses the full text index for a type to perform a full text search + * @param typeName the type to search + * @param searchText the query text + * @param count the number of items to return + * @returns the items + */ async fullTextQuery(typeName: string, searchText: string, count: number) { try { const graphNode = this.getGraphNodeDeclaration(typeName); diff --git a/typedoc.json b/typedoc.json new file mode 100644 index 0000000..f7bd0fc --- /dev/null +++ b/typedoc.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://typedoc.org/schema.json", + "plugin": ["typedoc-plugin-markdown"], + "out": "./docs", + "outputFileStrategy": "members", + "entryPoints": ["src/graphmodel.ts"], + "mergeReadme": true + } + \ No newline at end of file