Skip to content

Commit

Permalink
Add metadataFieldsToFilter property for MongoDB store
Browse files Browse the repository at this point in the history
Introduce a new property for the MongoDB vector store:
spring.ai.vectorstore.mongodb.metadata-fields-to-filter

This property accepts comma-separated values specifying which metadata
fields can be used for filtering when querying the vector store. It
ensures that metadata indexes are created if they don't already exist.

This addition enhances query performance and flexibility by allowing
users to define filterable fields in advance.

Co-authored-by: Eddú Meléndez <eddu.melendez@gmail.com>
  • Loading branch information
2 people authored and Mark Pollack committed Sep 17, 2024
1 parent 8b019c5 commit 8d31a57
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ spring.ai.vectorstore.mongodb.collection-name=vector_store
spring.ai.vectorstore.mongodb.initialize-schema=true
spring.ai.vectorstore.mongodb.path-name=embedding
spring.ai.vectorstore.mongodb.indexName=vector_index
spring.ai.vectorstore.mongodb.metadata-fields-to-filter=foo
----


Expand All @@ -131,6 +132,7 @@ spring.ai.vectorstore.mongodb.indexName=vector_index
|`spring.ai.vectorstore.mongodb.initialize-schema`| whether to initialize the backend schema for you | `false`
|`spring.ai.vectorstore.mongodb.path-name`| The name of the path to store the vectors. | `embedding`
|`spring.ai.vectorstore.mongodb.indexName`| The name of the index to store the vectors. | `vector_index`
|`spring.ai.vectorstore.mongodb.metadata-fields-to-filter` | comma separated values that specifies which metadata fields can be used for filtering when querying the vector store. Needed so that metadata indexes are created if they already don't exist | empty list
|===


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
* @author Eddú Meléndez
* @author Christian Tzolov
* @author Soby Chacko
* @author Ignacio López
* @since 1.0.0
*/
@AutoConfiguration
Expand Down Expand Up @@ -72,6 +73,9 @@ MongoDBAtlasVectorStore vectorStore(MongoTemplate mongoTemplate, EmbeddingModel
if (StringUtils.hasText(properties.getIndexName())) {
builder.withVectorIndexName(properties.getIndexName());
}
if (!properties.getMetadataFieldsToFilter().isEmpty()) {
builder.withMetadataFieldsToFilter(properties.getMetadataFieldsToFilter());
}
MongoDBAtlasVectorStore.MongoDBVectorStoreConfig config = builder.build();

return new MongoDBAtlasVectorStore(mongoTemplate, embeddingModel, config, properties.isInitializeSchema(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,12 @@
import org.springframework.ai.autoconfigure.vectorstore.CommonVectorStoreProperties;
import org.springframework.boot.context.properties.ConfigurationProperties;

import java.util.List;

/**
* @author Eddú Meléndez
* @author Christian Tzolov
* @author Ignacio López
* @since 1.0.0
*/
@ConfigurationProperties(MongoDBAtlasVectorStoreProperties.CONFIG_PREFIX)
Expand All @@ -43,6 +46,11 @@ public class MongoDBAtlasVectorStoreProperties extends CommonVectorStoreProperti
*/
private String indexName;

/**
* Name of the metadata fields to use as filters.
*/
private List<String> metadataFieldsToFilter = List.of();

public String getCollectionName() {
return this.collectionName;
}
Expand All @@ -67,4 +75,12 @@ public void setIndexName(String indexName) {
this.indexName = indexName;
}

public List<String> getMetadataFieldsToFilter() {
return this.metadataFieldsToFilter;
}

public void setMetadataFieldsToFilter(List<String> metadataFieldsToFilter) {
this.metadataFieldsToFilter = metadataFieldsToFilter;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.time.Duration;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import org.junit.jupiter.api.Disabled;
Expand All @@ -32,6 +33,8 @@
import org.springframework.ai.observation.conventions.VectorStoreProvider;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.filter.Filter;
import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder;
import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
Expand All @@ -40,6 +43,7 @@
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.junit.jupiter.Container;
Expand All @@ -51,6 +55,7 @@
* @author Eddú Meléndez
* @author Christian Tzolov
* @author Thomas Vitale
* @author Ignacio López
*/
@Testcontainers
@EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+")
Expand All @@ -70,7 +75,13 @@ class MongoDBAtlasVectorStoreAutoConfigurationIT {
new Document("Hello World Hello World Hello World Hello World Hello World Hello World Hello World"),
new Document(
"Great Depression Great Depression Great Depression Great Depression Great Depression Great Depression",
Collections.singletonMap("meta2", "meta2")));
Collections.singletonMap("meta2", "meta2")),
new Document(
"Testcontainers Testcontainers Testcontainers Testcontainers Testcontainers Testcontainers Testcontainers",
Collections.singletonMap("foo", "bar")),
new Document(
"Testcontainers Testcontainers Testcontainers Testcontainers Testcontainers Testcontainers Testcontainers",
Collections.singletonMap("foo", "baz")));

private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withUserConfiguration(Config.class)
Expand Down Expand Up @@ -123,6 +134,35 @@ public void addAndSearch() {

List<Document> results2 = vectorStore.similaritySearch(SearchRequest.query("Great").withTopK(1));
assertThat(results2).isEmpty();

context.getBean(MongoTemplate.class).dropCollection("test_collection");
});
}

@Test
public void addAndSearchWithFilters() {
contextRunner.withPropertyValues("spring.ai.vectorstore.mongodb.metadata-fields-to-filter=foo").run(context -> {

VectorStore vectorStore = context.getBean(VectorStore.class);
vectorStore.add(documents);

Thread.sleep(5000); // Await a second for the document to be indexed

List<Document> results = vectorStore.similaritySearch(SearchRequest.query("Testcontainers").withTopK(2));
assertThat(results).hasSize(2);
results.forEach(doc -> assertThat(doc.getContent().contains("Testcontainers")).isTrue());

FilterExpressionBuilder b = new FilterExpressionBuilder();
results = vectorStore.similaritySearch(
SearchRequest.query("Testcontainers").withTopK(2).withFilterExpression(b.eq("foo", "bar").build()));

assertThat(results).hasSize(1);
Document resultDoc = results.get(0);
assertThat(resultDoc.getId()).isEqualTo(documents.get(3).getId());
assertThat(resultDoc.getContent().contains("Testcontainers")).isTrue();
assertThat(resultDoc.getMetadata()).containsEntry("foo", "bar");

context.getBean(MongoTemplate.class).dropCollection("test_collection");
});
}

Expand Down

0 comments on commit 8d31a57

Please sign in to comment.