Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ WeaviateVectorStoreOptions mappingPropertiesToOptions(WeaviateVectorStorePropert
PropertyMapper mapper = PropertyMapper.get();
mapper.from(properties::getContentFieldName).whenHasText().to(weaviateVectorStoreOptions::setContentFieldName);
mapper.from(properties::getObjectClass).whenHasText().to(weaviateVectorStoreOptions::setObjectClass);
mapper.from(properties::getMetaFieldPrefix).whenHasText().to(weaviateVectorStoreOptions::setMetaFieldPrefix);

return weaviateVectorStoreOptions;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ public class WeaviateVectorStoreProperties {

private String contentFieldName = "content";

private String metaFieldPrefix = "meta_";

private ConsistentLevel consistencyLevel = WeaviateVectorStore.ConsistentLevel.ONE;

/**
Expand Down Expand Up @@ -99,6 +101,20 @@ public void setContentFieldName(String contentFieldName) {
this.contentFieldName = contentFieldName;
}

/**
* @since 1.1.0
*/
public String getMetaFieldPrefix() {
return metaFieldPrefix;
}

/**
* @since 1.1.0
*/
public void setMetaFieldPrefix(String metaFieldPrefix) {
this.metaFieldPrefix = metaFieldPrefix;
}

public ConsistentLevel getConsistencyLevel() {
return this.consistencyLevel;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,8 @@ public void autoConfigurationEnabledWhenTypeIsWeaviate() {
public void testMappingPropertiesToOptions() {
this.contextRunner
.withPropertyValues("spring.ai.vectorstore.weaviate.object-class=CustomObjectClass",
"spring.ai.vectorstore.weaviate.content-field-name=customContentFieldName")
"spring.ai.vectorstore.weaviate.content-field-name=customContentFieldName",
"spring.ai.vectorstore.weaviate.meta-field-prefix=custom_")
.run(context -> {
WeaviateVectorStoreAutoConfiguration autoConfiguration = context
.getBean(WeaviateVectorStoreAutoConfiguration.class);
Expand All @@ -189,6 +190,7 @@ public void testMappingPropertiesToOptions() {

assertThat(options.getObjectClass()).isEqualTo("CustomObjectClass");
assertThat(options.getContentFieldName()).isEqualTo("customContentFieldName");
assertThat(options.getMetaFieldPrefix()).isEqualTo("custom_");
});
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ You can use the following properties in your Spring Boot configuration to custom
|`spring.ai.vectorstore.weaviate.api-key`|The API key for authentication|
|`spring.ai.vectorstore.weaviate.object-class`|The class name for storing documents. |SpringAiWeaviate
|`spring.ai.vectorstore.weaviate.content-field-name`|The field name for content|content
|`spring.ai.vectorstore.weaviate.meta-field-prefix`|The field prefix for metadata|meta_
|`spring.ai.vectorstore.weaviate.consistency-level`|Desired tradeoff between consistency and speed|ConsistentLevel.ONE
|`spring.ai.vectorstore.weaviate.filter-field`|Configures metadata fields that can be used in filters. Format: spring.ai.vectorstore.weaviate.filter-field.<field-name>=<field-type>|
|===
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2023-2024 the original author or authors.
* Copyright 2023-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -35,19 +35,42 @@
* (https://weaviate.io/developers/weaviate/api/graphql/filters)
*
* @author Christian Tzolov
* @author Jonghoon Park
*/
public class WeaviateFilterExpressionConverter extends AbstractFilterExpressionConverter {

// https://weaviate.io/developers/weaviate/api/graphql/filters#special-cases
private static final List<String> SYSTEM_IDENTIFIERS = List.of("id", "_creationTimeUnix", "_lastUpdateTimeUnix");

private static final String DEFAULT_META_FIELD_PREFIX = "meta_";

private boolean mapIntegerToNumberValue = true;

private List<String> allowedIdentifierNames;

private final String metaFieldPrefix;

/**
* Constructs a new instance of the {@code WeaviateFilterExpressionConverter} class.
* This constructor uses the default meta field prefix
* ({@link #DEFAULT_META_FIELD_PREFIX}).
* @param allowedIdentifierNames A {@code List} of allowed identifier names.
*/
public WeaviateFilterExpressionConverter(List<String> allowedIdentifierNames) {
this(allowedIdentifierNames, DEFAULT_META_FIELD_PREFIX);
}

/**
* Constructs a new instance of the {@code WeaviateFilterExpressionConverter} class.
* @param allowedIdentifierNames A {@code List} of allowed identifier names.
* @param metaFieldPrefix the prefix for meta fields
* @since 1.1.0
*/
public WeaviateFilterExpressionConverter(List<String> allowedIdentifierNames, String metaFieldPrefix) {
Assert.notNull(allowedIdentifierNames, "List can be empty but not null.");
Assert.notNull(metaFieldPrefix, "metaFieldPrefix can be empty but not null.");
this.allowedIdentifierNames = allowedIdentifierNames;
this.metaFieldPrefix = metaFieldPrefix;
}

public void setAllowedIdentifierNames(List<String> allowedIdentifierNames) {
Expand Down Expand Up @@ -112,7 +135,7 @@ public String withMetaPrefix(String identifier) {
}

if (this.allowedIdentifierNames.contains(identifier)) {
return "meta_" + identifier;
return this.metaFieldPrefix + identifier;
}

throw new IllegalArgumentException("Not allowed filter identifier name: " + identifier
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,6 @@ public class WeaviateVectorStore extends AbstractObservationVectorStore {

private static final Logger logger = LoggerFactory.getLogger(WeaviateVectorStore.class);

private static final String METADATA_FIELD_PREFIX = "meta_";

private static final String METADATA_FIELD_NAME = "metadata";

private static final String ADDITIONAL_FIELD_NAME = "_additional";
Expand Down Expand Up @@ -162,7 +160,8 @@ protected WeaviateVectorStore(Builder builder) {
this.consistencyLevel = builder.consistencyLevel;
this.filterMetadataFields = builder.filterMetadataFields;
this.filterExpressionConverter = new WeaviateFilterExpressionConverter(
this.filterMetadataFields.stream().map(MetadataField::name).toList());
this.filterMetadataFields.stream().map(MetadataField::name).toList(),
this.options.getMetaFieldPrefix());
this.weaviateSimilaritySearchFields = buildWeaviateSimilaritySearchFields();
}

Expand All @@ -182,7 +181,7 @@ private Field[] buildWeaviateSimilaritySearchFields() {
searchWeaviateFieldList.add(Field.builder().name(this.options.getContentFieldName()).build());
searchWeaviateFieldList.add(Field.builder().name(METADATA_FIELD_NAME).build());
searchWeaviateFieldList.addAll(this.filterMetadataFields.stream()
.map(mf -> Field.builder().name(METADATA_FIELD_PREFIX + mf.name()).build())
.map(mf -> Field.builder().name(this.options.getMetaFieldPrefix() + mf.name()).build())
.toList());
searchWeaviateFieldList.add(Field.builder()
.name(ADDITIONAL_FIELD_NAME)
Expand Down Expand Up @@ -260,7 +259,7 @@ private WeaviateObject toWeaviateObject(Document document, List<Document> docume
// expressions on them.
for (MetadataField mf : this.filterMetadataFields) {
if (document.getMetadata().containsKey(mf.name())) {
fields.put(METADATA_FIELD_PREFIX + mf.name(), document.getMetadata().get(mf.name()));
fields.put(this.options.getMetaFieldPrefix() + mf.name(), document.getMetadata().get(mf.name()));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,16 @@
* Provided Weaviate vector option configuration.
*
* @author Jonghoon Park
* @since 1.1.0.
* @since 1.1.0
*/
public class WeaviateVectorStoreOptions {

private String objectClass = "SpringAiWeaviate";

private String contentFieldName = "content";

private String metaFieldPrefix = "meta_";

public String getObjectClass() {
return objectClass;
}
Expand All @@ -48,4 +50,13 @@ public void setContentFieldName(String contentFieldName) {
this.contentFieldName = contentFieldName;
}

public String getMetaFieldPrefix() {
return metaFieldPrefix;
}

public void setMetaFieldPrefix(String metaFieldPrefix) {
Assert.notNull(metaFieldPrefix, "metaFieldPrefix can be empty but not null");
this.metaFieldPrefix = metaFieldPrefix;
}
Comment on lines +53 to +60
Copy link
Contributor Author

@dev-jonghoonpark dev-jonghoonpark Jun 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These methods are newly created, but since the class Javadoc already includes the @since 1.1.0 tag, the @since tag was not added to the methods.


}
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ void shouldBuildWithCustomConfiguration() {
WeaviateVectorStoreOptions options = new WeaviateVectorStoreOptions();
options.setObjectClass("CustomObjectClass");
options.setContentFieldName("customContentFieldName");
options.setMetaFieldPrefix("custom_");

WeaviateVectorStore vectorStore = WeaviateVectorStore.builder(weaviateClient, this.embeddingModel)
.options(options)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
import io.weaviate.client.Config;
import io.weaviate.client.WeaviateClient;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
Expand Down Expand Up @@ -359,6 +361,77 @@ public void addAndSearchWithCustomContentFieldName() {
});
}

@ParameterizedTest(name = "{0} : {displayName} ")
@ValueSource(strings = { "custom_", "" })
public void addAndSearchWithCustomMetaFieldPrefix(String metaFieldPrefix) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you help me understand where the configurable metadata prefix is used in this test and then assert that it takes effect?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sobychacko

In this test, the following behavior is expected:

  1. Verify that data is stored and retrieved correctly using a customVectorStore with a modified prefix.
  2. Confirm that data cannot be retrieved using a vectorStore with the default prefix (as the prefixes differ, retrieval should fail).
  3. Remove the data from the customVectorStore to ensure it does not affect subsequent tests.

If there’s a better approach, I’m open to suggestions for improvement. 👍

WeaviateVectorStoreOptions optionsWithCustomContentFieldName = new WeaviateVectorStoreOptions();
optionsWithCustomContentFieldName.setMetaFieldPrefix(metaFieldPrefix);

this.contextRunner.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
resetCollection(vectorStore);
});

this.contextRunner.run(context -> {
WeaviateClient weaviateClient = context.getBean(WeaviateClient.class);
EmbeddingModel embeddingModel = context.getBean(EmbeddingModel.class);

VectorStore customVectorStore = WeaviateVectorStore.builder(weaviateClient, embeddingModel)
.filterMetadataFields(List.of(WeaviateVectorStore.MetadataField.text("country")))
.options(optionsWithCustomContentFieldName)
.build();

var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2020));
var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "NL"));
var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner",
Map.of("country", "BG", "year", 2023));

customVectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));

List<Document> results = customVectorStore
.similaritySearch(SearchRequest.builder().query("The World").topK(5).build());
assertThat(results).hasSize(3);

results = customVectorStore.similaritySearch(SearchRequest.builder()
.query("The World")
.topK(5)
.similarityThresholdAll()
.filterExpression("country == 'NL'")
.build());
assertThat(results).hasSize(1);
assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId());
});

this.contextRunner.run(context -> {
VectorStore vectorStore = context.getBean(VectorStore.class);
List<Document> results = vectorStore.similaritySearch(SearchRequest.builder()
.query("The World")
.topK(5)
.similarityThresholdAll()
.filterExpression("country == 'NL'")
.build());
assertThat(results).hasSize(0);
});

// remove documents for parameterized test
this.contextRunner.run(context -> {
WeaviateClient weaviateClient = context.getBean(WeaviateClient.class);
EmbeddingModel embeddingModel = context.getBean(EmbeddingModel.class);

VectorStore customVectorStore = WeaviateVectorStore.builder(weaviateClient, embeddingModel)
.filterMetadataFields(List.of(WeaviateVectorStore.MetadataField.text("country")))
.options(optionsWithCustomContentFieldName)
.build();

List<Document> results = customVectorStore
.similaritySearch(SearchRequest.builder().query("The World").topK(5).build());

customVectorStore.delete(results.stream().map(Document::getId).toList());
});
}

@SpringBootConfiguration
@EnableAutoConfiguration
public static class TestApplication {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import org.junit.jupiter.api.Test;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

/**
Expand Down Expand Up @@ -66,4 +67,19 @@ void shouldFailWithEmptyContentFieldName() {
.hasMessage("contentFieldName cannot be null or empty");
}

@Test
void shouldFailWithNullMetaFieldPrefix() {
WeaviateVectorStoreOptions options = new WeaviateVectorStoreOptions();

assertThatThrownBy(() -> options.setMetaFieldPrefix(null)).isInstanceOf(IllegalArgumentException.class)
.hasMessage("metaFieldPrefix can be empty but not null");
}

@Test
void shouldPassWithEmptyMetaFieldPrefix() {
WeaviateVectorStoreOptions options = new WeaviateVectorStoreOptions();
options.setMetaFieldPrefix("");
assertThat(options.getMetaFieldPrefix()).isEqualTo("");
}

}