From 3d0455d6e76f8099895b4208c7a6fd7025e9d9e9 Mon Sep 17 00:00:00 2001 From: jonghoonpark Date: Thu, 19 Jun 2025 09:58:26 +0900 Subject: [PATCH] Make the meta field prefix configurable in Weaviate vector store properties Signed-off-by: jonghoonpark --- .../WeaviateVectorStoreAutoConfiguration.java | 1 + .../WeaviateVectorStoreProperties.java | 16 ++++ ...eaviateVectorStoreAutoConfigurationIT.java | 4 +- .../ROOT/pages/api/vectordbs/weaviate.adoc | 1 + .../WeaviateFilterExpressionConverter.java | 27 ++++++- .../weaviate/WeaviateVectorStore.java | 9 +-- .../weaviate/WeaviateVectorStoreOptions.java | 13 +++- .../WeaviateVectorStoreBuilderTests.java | 1 + .../weaviate/WeaviateVectorStoreIT.java | 73 +++++++++++++++++++ .../WeaviateVectorStoreOptionsTests.java | 16 ++++ 10 files changed, 152 insertions(+), 9 deletions(-) diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfiguration.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfiguration.java index 4e9c1eb156a..486da350c5c 100644 --- a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfiguration.java +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfiguration.java @@ -105,6 +105,7 @@ WeaviateVectorStoreOptions mappingPropertiesToOptions(WeaviateVectorStorePropert PropertyMapper mapper = PropertyMapper.get(); mapper.from(properties::getContentFieldName).whenHasText().to(weaviateVectorStoreOptions::setContentFieldName); mapper.from(properties::getObjectClass).whenHasText().to(weaviateVectorStoreOptions::setObjectClass); + mapper.from(properties::getMetaFieldPrefix).whenHasText().to(weaviateVectorStoreOptions::setMetaFieldPrefix); return weaviateVectorStoreOptions; } diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreProperties.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreProperties.java index cfa6fe4cb23..48eb859ab33 100644 --- a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreProperties.java +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/main/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreProperties.java @@ -44,6 +44,8 @@ public class WeaviateVectorStoreProperties { private String contentFieldName = "content"; + private String metaFieldPrefix = "meta_"; + private ConsistentLevel consistencyLevel = WeaviateVectorStore.ConsistentLevel.ONE; /** @@ -99,6 +101,20 @@ public void setContentFieldName(String contentFieldName) { this.contentFieldName = contentFieldName; } + /** + * @since 1.1.0 + */ + public String getMetaFieldPrefix() { + return metaFieldPrefix; + } + + /** + * @since 1.1.0 + */ + public void setMetaFieldPrefix(String metaFieldPrefix) { + this.metaFieldPrefix = metaFieldPrefix; + } + public ConsistentLevel getConsistencyLevel() { return this.consistencyLevel; } diff --git a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/test/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfigurationIT.java b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/test/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfigurationIT.java index fbcb7367046..bd80f8124c5 100644 --- a/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/test/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfigurationIT.java +++ b/auto-configurations/vector-stores/spring-ai-autoconfigure-vector-store-weaviate/src/test/java/org/springframework/ai/vectorstore/weaviate/autoconfigure/WeaviateVectorStoreAutoConfigurationIT.java @@ -180,7 +180,8 @@ public void autoConfigurationEnabledWhenTypeIsWeaviate() { public void testMappingPropertiesToOptions() { this.contextRunner .withPropertyValues("spring.ai.vectorstore.weaviate.object-class=CustomObjectClass", - "spring.ai.vectorstore.weaviate.content-field-name=customContentFieldName") + "spring.ai.vectorstore.weaviate.content-field-name=customContentFieldName", + "spring.ai.vectorstore.weaviate.meta-field-prefix=custom_") .run(context -> { WeaviateVectorStoreAutoConfiguration autoConfiguration = context .getBean(WeaviateVectorStoreAutoConfiguration.class); @@ -189,6 +190,7 @@ public void testMappingPropertiesToOptions() { assertThat(options.getObjectClass()).isEqualTo("CustomObjectClass"); assertThat(options.getContentFieldName()).isEqualTo("customContentFieldName"); + assertThat(options.getMetaFieldPrefix()).isEqualTo("custom_"); }); } diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/weaviate.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/weaviate.adoc index 9a058aa234d..b8fd536fb5b 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/weaviate.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/vectordbs/weaviate.adoc @@ -264,6 +264,7 @@ You can use the following properties in your Spring Boot configuration to custom |`spring.ai.vectorstore.weaviate.api-key`|The API key for authentication| |`spring.ai.vectorstore.weaviate.object-class`|The class name for storing documents. |SpringAiWeaviate |`spring.ai.vectorstore.weaviate.content-field-name`|The field name for content|content +|`spring.ai.vectorstore.weaviate.meta-field-prefix`|The field prefix for metadata|meta_ |`spring.ai.vectorstore.weaviate.consistency-level`|Desired tradeoff between consistency and speed|ConsistentLevel.ONE |`spring.ai.vectorstore.weaviate.filter-field`|Configures metadata fields that can be used in filters. Format: spring.ai.vectorstore.weaviate.filter-field.=| |=== diff --git a/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateFilterExpressionConverter.java b/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateFilterExpressionConverter.java index c3c71119349..3321cd179f2 100644 --- a/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateFilterExpressionConverter.java +++ b/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateFilterExpressionConverter.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 the original author or authors. + * Copyright 2023-2025 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,19 +35,42 @@ * (https://weaviate.io/developers/weaviate/api/graphql/filters) * * @author Christian Tzolov + * @author Jonghoon Park */ public class WeaviateFilterExpressionConverter extends AbstractFilterExpressionConverter { // https://weaviate.io/developers/weaviate/api/graphql/filters#special-cases private static final List SYSTEM_IDENTIFIERS = List.of("id", "_creationTimeUnix", "_lastUpdateTimeUnix"); + private static final String DEFAULT_META_FIELD_PREFIX = "meta_"; + private boolean mapIntegerToNumberValue = true; private List allowedIdentifierNames; + private final String metaFieldPrefix; + + /** + * Constructs a new instance of the {@code WeaviateFilterExpressionConverter} class. + * This constructor uses the default meta field prefix + * ({@link #DEFAULT_META_FIELD_PREFIX}). + * @param allowedIdentifierNames A {@code List} of allowed identifier names. + */ public WeaviateFilterExpressionConverter(List allowedIdentifierNames) { + this(allowedIdentifierNames, DEFAULT_META_FIELD_PREFIX); + } + + /** + * Constructs a new instance of the {@code WeaviateFilterExpressionConverter} class. + * @param allowedIdentifierNames A {@code List} of allowed identifier names. + * @param metaFieldPrefix the prefix for meta fields + * @since 1.1.0 + */ + public WeaviateFilterExpressionConverter(List allowedIdentifierNames, String metaFieldPrefix) { Assert.notNull(allowedIdentifierNames, "List can be empty but not null."); + Assert.notNull(metaFieldPrefix, "metaFieldPrefix can be empty but not null."); this.allowedIdentifierNames = allowedIdentifierNames; + this.metaFieldPrefix = metaFieldPrefix; } public void setAllowedIdentifierNames(List allowedIdentifierNames) { @@ -112,7 +135,7 @@ public String withMetaPrefix(String identifier) { } if (this.allowedIdentifierNames.contains(identifier)) { - return "meta_" + identifier; + return this.metaFieldPrefix + identifier; } throw new IllegalArgumentException("Not allowed filter identifier name: " + identifier diff --git a/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStore.java b/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStore.java index b3f30f29e00..28f29d18c1d 100644 --- a/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStore.java +++ b/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStore.java @@ -96,8 +96,6 @@ public class WeaviateVectorStore extends AbstractObservationVectorStore { private static final Logger logger = LoggerFactory.getLogger(WeaviateVectorStore.class); - private static final String METADATA_FIELD_PREFIX = "meta_"; - private static final String METADATA_FIELD_NAME = "metadata"; private static final String ADDITIONAL_FIELD_NAME = "_additional"; @@ -162,7 +160,8 @@ protected WeaviateVectorStore(Builder builder) { this.consistencyLevel = builder.consistencyLevel; this.filterMetadataFields = builder.filterMetadataFields; this.filterExpressionConverter = new WeaviateFilterExpressionConverter( - this.filterMetadataFields.stream().map(MetadataField::name).toList()); + this.filterMetadataFields.stream().map(MetadataField::name).toList(), + this.options.getMetaFieldPrefix()); this.weaviateSimilaritySearchFields = buildWeaviateSimilaritySearchFields(); } @@ -182,7 +181,7 @@ private Field[] buildWeaviateSimilaritySearchFields() { searchWeaviateFieldList.add(Field.builder().name(this.options.getContentFieldName()).build()); searchWeaviateFieldList.add(Field.builder().name(METADATA_FIELD_NAME).build()); searchWeaviateFieldList.addAll(this.filterMetadataFields.stream() - .map(mf -> Field.builder().name(METADATA_FIELD_PREFIX + mf.name()).build()) + .map(mf -> Field.builder().name(this.options.getMetaFieldPrefix() + mf.name()).build()) .toList()); searchWeaviateFieldList.add(Field.builder() .name(ADDITIONAL_FIELD_NAME) @@ -260,7 +259,7 @@ private WeaviateObject toWeaviateObject(Document document, List docume // expressions on them. for (MetadataField mf : this.filterMetadataFields) { if (document.getMetadata().containsKey(mf.name())) { - fields.put(METADATA_FIELD_PREFIX + mf.name(), document.getMetadata().get(mf.name())); + fields.put(this.options.getMetaFieldPrefix() + mf.name(), document.getMetadata().get(mf.name())); } } diff --git a/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptions.java b/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptions.java index 24f7badd061..3bff73e05b0 100644 --- a/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptions.java +++ b/vector-stores/spring-ai-weaviate-store/src/main/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptions.java @@ -22,7 +22,7 @@ * Provided Weaviate vector option configuration. * * @author Jonghoon Park - * @since 1.1.0. + * @since 1.1.0 */ public class WeaviateVectorStoreOptions { @@ -30,6 +30,8 @@ public class WeaviateVectorStoreOptions { private String contentFieldName = "content"; + private String metaFieldPrefix = "meta_"; + public String getObjectClass() { return objectClass; } @@ -48,4 +50,13 @@ public void setContentFieldName(String contentFieldName) { this.contentFieldName = contentFieldName; } + public String getMetaFieldPrefix() { + return metaFieldPrefix; + } + + public void setMetaFieldPrefix(String metaFieldPrefix) { + Assert.notNull(metaFieldPrefix, "metaFieldPrefix can be empty but not null"); + this.metaFieldPrefix = metaFieldPrefix; + } + } diff --git a/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreBuilderTests.java b/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreBuilderTests.java index 13ccd7e48bb..d1b2517dc01 100644 --- a/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreBuilderTests.java +++ b/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreBuilderTests.java @@ -60,6 +60,7 @@ void shouldBuildWithCustomConfiguration() { WeaviateVectorStoreOptions options = new WeaviateVectorStoreOptions(); options.setObjectClass("CustomObjectClass"); options.setContentFieldName("customContentFieldName"); + options.setMetaFieldPrefix("custom_"); WeaviateVectorStore vectorStore = WeaviateVectorStore.builder(weaviateClient, this.embeddingModel) .options(options) diff --git a/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreIT.java b/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreIT.java index ed439865d5a..0a268683cd0 100644 --- a/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreIT.java +++ b/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreIT.java @@ -28,6 +28,8 @@ import io.weaviate.client.Config; import io.weaviate.client.WeaviateClient; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -359,6 +361,77 @@ public void addAndSearchWithCustomContentFieldName() { }); } + @ParameterizedTest(name = "{0} : {displayName} ") + @ValueSource(strings = { "custom_", "" }) + public void addAndSearchWithCustomMetaFieldPrefix(String metaFieldPrefix) { + WeaviateVectorStoreOptions optionsWithCustomContentFieldName = new WeaviateVectorStoreOptions(); + optionsWithCustomContentFieldName.setMetaFieldPrefix(metaFieldPrefix); + + this.contextRunner.run(context -> { + VectorStore vectorStore = context.getBean(VectorStore.class); + resetCollection(vectorStore); + }); + + this.contextRunner.run(context -> { + WeaviateClient weaviateClient = context.getBean(WeaviateClient.class); + EmbeddingModel embeddingModel = context.getBean(EmbeddingModel.class); + + VectorStore customVectorStore = WeaviateVectorStore.builder(weaviateClient, embeddingModel) + .filterMetadataFields(List.of(WeaviateVectorStore.MetadataField.text("country"))) + .options(optionsWithCustomContentFieldName) + .build(); + + var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2020)); + var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "NL")); + var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner", + Map.of("country", "BG", "year", 2023)); + + customVectorStore.add(List.of(bgDocument, nlDocument, bgDocument2)); + + List results = customVectorStore + .similaritySearch(SearchRequest.builder().query("The World").topK(5).build()); + assertThat(results).hasSize(3); + + results = customVectorStore.similaritySearch(SearchRequest.builder() + .query("The World") + .topK(5) + .similarityThresholdAll() + .filterExpression("country == 'NL'") + .build()); + assertThat(results).hasSize(1); + assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId()); + }); + + this.contextRunner.run(context -> { + VectorStore vectorStore = context.getBean(VectorStore.class); + List results = vectorStore.similaritySearch(SearchRequest.builder() + .query("The World") + .topK(5) + .similarityThresholdAll() + .filterExpression("country == 'NL'") + .build()); + assertThat(results).hasSize(0); + }); + + // remove documents for parameterized test + this.contextRunner.run(context -> { + WeaviateClient weaviateClient = context.getBean(WeaviateClient.class); + EmbeddingModel embeddingModel = context.getBean(EmbeddingModel.class); + + VectorStore customVectorStore = WeaviateVectorStore.builder(weaviateClient, embeddingModel) + .filterMetadataFields(List.of(WeaviateVectorStore.MetadataField.text("country"))) + .options(optionsWithCustomContentFieldName) + .build(); + + List results = customVectorStore + .similaritySearch(SearchRequest.builder().query("The World").topK(5).build()); + + customVectorStore.delete(results.stream().map(Document::getId).toList()); + }); + } + @SpringBootConfiguration @EnableAutoConfiguration public static class TestApplication { diff --git a/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptionsTests.java b/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptionsTests.java index 954622d7a42..1e96b39c41a 100644 --- a/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptionsTests.java +++ b/vector-stores/spring-ai-weaviate-store/src/test/java/org/springframework/ai/vectorstore/weaviate/WeaviateVectorStoreOptionsTests.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** @@ -66,4 +67,19 @@ void shouldFailWithEmptyContentFieldName() { .hasMessage("contentFieldName cannot be null or empty"); } + @Test + void shouldFailWithNullMetaFieldPrefix() { + WeaviateVectorStoreOptions options = new WeaviateVectorStoreOptions(); + + assertThatThrownBy(() -> options.setMetaFieldPrefix(null)).isInstanceOf(IllegalArgumentException.class) + .hasMessage("metaFieldPrefix can be empty but not null"); + } + + @Test + void shouldPassWithEmptyMetaFieldPrefix() { + WeaviateVectorStoreOptions options = new WeaviateVectorStoreOptions(); + options.setMetaFieldPrefix(""); + assertThat(options.getMetaFieldPrefix()).isEqualTo(""); + } + }