From 8d888a52549e8f277668f6de546c010208076d4e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 9 May 2025 09:33:19 +0200 Subject: [PATCH 1/6] add dependency to sqlite-vec package --- dotnet/Directory.Packages.props | 1 + .../Connectors.Memory.SqliteVec.csproj | 1 + 2 files changed, 2 insertions(+) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 0b99c3da23ff..9eb820558f36 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -154,6 +154,7 @@ + diff --git a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj index 2c427f636975..b1f47faf8414 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj @@ -27,6 +27,7 @@ + From bdc8e3ec74bb4572a63c9912e11ffd9ff225f269 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 9 May 2025 09:45:33 +0200 Subject: [PATCH 2/6] use it --- .../Support/SqliteTestEnvironment.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/dotnet/src/VectorDataIntegrationTests/SqliteVecIntegrationTests/Support/SqliteTestEnvironment.cs b/dotnet/src/VectorDataIntegrationTests/SqliteVecIntegrationTests/Support/SqliteTestEnvironment.cs index 4f6968e54962..a0485852a6d7 100644 --- a/dotnet/src/VectorDataIntegrationTests/SqliteVecIntegrationTests/Support/SqliteTestEnvironment.cs +++ b/dotnet/src/VectorDataIntegrationTests/SqliteVecIntegrationTests/Support/SqliteTestEnvironment.cs @@ -7,12 +7,6 @@ namespace SqliteVecIntegrationTests.Support; internal static class SqliteTestEnvironment { - /// - /// SQLite extension name for vector search. - /// More information here: . - /// - private const string VectorSearchExtensionName = "vec0"; - private static bool? s_isSqliteVecInstalled; internal static bool TryLoadSqliteVec(SqliteConnection connection) @@ -26,7 +20,7 @@ internal static bool TryLoadSqliteVec(SqliteConnection connection) try { - connection.LoadExtension(VectorSearchExtensionName); + connection.LoadVector(); s_isSqliteVecInstalled = true; } catch (SqliteException) From b19df98d9d71f410db2469db749720ce74c1d16d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 9 May 2025 09:49:20 +0200 Subject: [PATCH 3/6] bump Microsoft.Data.Sqlite as well, so we consume a fix that is unblocking the scenario! --- dotnet/Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 9eb820558f36..d298252fc82b 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -135,7 +135,7 @@ - + From e6c44a83b9c71976ce9109d3b8696d569b939983 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 9 May 2025 11:10:14 +0200 Subject: [PATCH 4/6] add a workaround for compiler bug --- .../Connectors.Memory.SqliteVec.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj index b1f47faf8414..24c074206355 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj @@ -6,6 +6,7 @@ $(AssemblyName) net8.0;netstandard2.0;net462 preview + $(NoWarn);CS1591 From 4970221b5591e3211825d0f60d2fa8c6b29f8d01 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 9 May 2025 16:55:38 +0200 Subject: [PATCH 5/6] address code review feedback: - remove the ability to specify extension name (now it's always vec0) - use LoadVector after opening connection rather than LoadExtension($userDefinedName) - use 9.0.4 of Microsoft.Data.Sqlite --- dotnet/Directory.Packages.props | 2 +- .../SqliteCollection.cs | 16 ++++------------ .../SqliteCommandBuilder.cs | 5 ++--- .../SqliteConstants.cs | 6 ------ .../SqliteVectorStore.cs | 5 ----- .../SqliteVectorStoreOptions.cs | 7 ------- .../SqliteCommandBuilderTests.cs | 5 ++--- 7 files changed, 9 insertions(+), 37 deletions(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index d298252fc82b..8731a694d0d3 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -135,7 +135,7 @@ - + diff --git a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteCollection.cs b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteCollection.cs index cef87c3abdfa..d0926b10ee90 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteCollection.cs @@ -53,9 +53,6 @@ public sealed class SqliteCollection : VectorStoreCollectionTable name in SQLite for vector properties. private readonly string _vectorTableName; - /// The sqlite_vec extension name to use. - private readonly string _vectorSearchExtensionName; - /// public override string Name { get; } @@ -84,7 +81,6 @@ public SqliteCollection( this.Name = name; options ??= SqliteCollectionOptions.Default; - this._vectorSearchExtensionName = options.VectorSearchExtensionName ?? SqliteConstants.VectorSearchExtensionName; // Escape both table names before exposing them to anything that may build SQL commands. this._dataTableName = name.EscapeIdentifier(); @@ -543,7 +539,7 @@ private async ValueTask GetConnectionAsync(CancellationToken c { var connection = new SqliteConnection(this._connectionString); await connection.OpenAsync(cancellationToken).ConfigureAwait(false); - connection.LoadExtension(this._vectorSearchExtensionName); + connection.LoadVector(); return connection; } @@ -599,13 +595,9 @@ await this.CreateTableAsync(connection, this._dataTableName, dataTableColumns, i if (this._vectorPropertiesExist) { - var extensionName = !string.IsNullOrWhiteSpace(this._vectorSearchExtensionName) ? - this._vectorSearchExtensionName : - SqliteConstants.VectorSearchExtensionName; - List vectorTableColumns = SqlitePropertyMapping.GetColumns(this._model.Properties, data: false); - await this.CreateVirtualTableAsync(connection, this._vectorTableName, vectorTableColumns, ifNotExists, extensionName!, cancellationToken) + await this.CreateVirtualTableAsync(connection, this._vectorTableName, vectorTableColumns, ifNotExists, cancellationToken) .ConfigureAwait(false); } } @@ -623,11 +615,11 @@ private Task CreateTableAsync(SqliteConnection connection, string tableName cancellationToken); } - private Task CreateVirtualTableAsync(SqliteConnection connection, string tableName, List columns, bool ifNotExists, string extensionName, CancellationToken cancellationToken) + private Task CreateVirtualTableAsync(SqliteConnection connection, string tableName, List columns, bool ifNotExists, CancellationToken cancellationToken) { const string OperationName = "CreateVirtualTable"; - using var command = SqliteCommandBuilder.BuildCreateVirtualTableCommand(connection, tableName, columns, ifNotExists, extensionName); + using var command = SqliteCommandBuilder.BuildCreateVirtualTableCommand(connection, tableName, columns, ifNotExists); return connection.ExecuteWithErrorHandlingAsync( this._collectionMetadata, diff --git a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteCommandBuilder.cs b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteCommandBuilder.cs index aa21b2dc0529..65560df178b9 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteCommandBuilder.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteCommandBuilder.cs @@ -69,12 +69,11 @@ public static DbCommand BuildCreateVirtualTableCommand( SqliteConnection connection, string tableName, IReadOnlyList columns, - bool ifNotExists, - string extensionName) + bool ifNotExists) { var builder = new StringBuilder(); - builder.AppendLine($"CREATE VIRTUAL TABLE {(ifNotExists ? "IF NOT EXISTS " : string.Empty)}\"{tableName}\" USING {extensionName}("); + builder.AppendLine($"CREATE VIRTUAL TABLE {(ifNotExists ? "IF NOT EXISTS " : string.Empty)}\"{tableName}\" USING vec0("); // The vector extension is currently uncapable of handling quoted identifiers. builder.AppendLine(string.Join(",\n", columns.Select(column => GetColumnDefinition(column, quote: false)))); diff --git a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteConstants.cs b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteConstants.cs index d1f9316aa46f..a09599a9a14f 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteConstants.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteConstants.cs @@ -10,12 +10,6 @@ internal static class SqliteConstants { internal const string VectorStoreSystemName = "sqlite"; - /// - /// SQLite extension name for vector search. - /// More information here: . - /// - public const string VectorSearchExtensionName = "vec0"; - /// A of types that vector properties on the provided model may have. public static readonly HashSet SupportedVectorTypes = [ diff --git a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteVectorStore.cs index cddf6724477b..e1aa4303f921 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/SqliteVectorStore.cs @@ -28,9 +28,6 @@ public sealed class SqliteVectorStore : VectorStore /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. private static readonly VectorStoreRecordDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string))] }; - /// SQLite extension name for vector search operations. - private readonly string? _vectorSearchExtensionName; - /// Custom virtual table name to store vectors. private readonly string? _vectorVirtualTableName; @@ -48,7 +45,6 @@ public SqliteVectorStore(string connectionString, SqliteVectorStoreOptions? opti this._connectionString = connectionString; options ??= SqliteVectorStoreOptions.Default; - this._vectorSearchExtensionName = options.VectorSearchExtensionName; this._vectorVirtualTableName = options.VectorVirtualTableName; this._embeddingGenerator = options.EmbeddingGenerator; @@ -74,7 +70,6 @@ public override VectorStoreCollection GetCollection - /// SQLite extension name for vector search operations. - /// If not specified, the default "vec0" extension name will be used. - /// More information here: . - /// - public string? VectorSearchExtensionName { get; set; } - /// /// Custom virtual table name to store vectors. /// diff --git a/dotnet/src/Connectors/Connectors.SqliteVec.UnitTests/SqliteCommandBuilderTests.cs b/dotnet/src/Connectors/Connectors.SqliteVec.UnitTests/SqliteCommandBuilderTests.cs index 5c5674ba93d2..6580a089b302 100644 --- a/dotnet/src/Connectors/Connectors.SqliteVec.UnitTests/SqliteCommandBuilderTests.cs +++ b/dotnet/src/Connectors/Connectors.SqliteVec.UnitTests/SqliteCommandBuilderTests.cs @@ -73,7 +73,6 @@ public void ItBuildsCreateVirtualTableCommand(bool ifNotExists) { // Arrange const string TableName = "TestTable"; - const string ExtensionName = "TestExtension"; var columns = new List { @@ -82,12 +81,12 @@ public void ItBuildsCreateVirtualTableCommand(bool ifNotExists) }; // Act - var command = SqliteCommandBuilder.BuildCreateVirtualTableCommand(this._connection, TableName, columns, ifNotExists, ExtensionName); + var command = SqliteCommandBuilder.BuildCreateVirtualTableCommand(this._connection, TableName, columns, ifNotExists); // Assert Assert.Contains("CREATE VIRTUAL TABLE", command.CommandText); Assert.Contains(TableName, command.CommandText); - Assert.Contains($"USING {ExtensionName}", command.CommandText); + Assert.Contains("USING vec0", command.CommandText); Assert.Equal(ifNotExists, command.CommandText.Contains("IF NOT EXISTS")); From f41000ef67cc90be9db2693e9b563a33b311e320 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 9 May 2025 18:21:30 +0200 Subject: [PATCH 6/6] update to latest version and remove the workaround --- dotnet/Directory.Packages.props | 2 +- .../Connectors.Memory.SqliteVec.csproj | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 8731a694d0d3..258d01f64551 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -154,7 +154,7 @@ - + diff --git a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj index 24c074206355..b1f47faf8414 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.SqliteVec/Connectors.Memory.SqliteVec.csproj @@ -6,7 +6,6 @@ $(AssemblyName) net8.0;netstandard2.0;net462 preview - $(NoWarn);CS1591