Skip to content

Commit d5b1d50

Browse files
authored
Add vector-set API (#2939)
Adds support for [Redis vector sets](https://redis.io/docs/latest/develop/data-types/vector-sets/)
1 parent 6fdbc88 commit d5b1d50

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+3699
-65
lines changed

Directory.Build.props

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
1111
<CodeAnalysisRuleset>$(MSBuildThisFileDirectory)Shared.ruleset</CodeAnalysisRuleset>
1212
<MSBuildWarningsAsMessages>NETSDK1069</MSBuildWarningsAsMessages>
13-
<NoWarn>NU5105;NU1507</NoWarn>
13+
<NoWarn>$(NoWarn);NU5105;NU1507;SER001</NoWarn>
1414
<PackageReleaseNotes>https://stackexchange.github.io/StackExchange.Redis/ReleaseNotes</PackageReleaseNotes>
1515
<PackageProjectUrl>https://stackexchange.github.io/StackExchange.Redis/</PackageProjectUrl>
1616
<PackageLicenseExpression>MIT</PackageLicenseExpression>

Directory.Packages.props

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88
<PackageVersion Include="System.Threading.Channels" Version="5.0.0" />
99
<PackageVersion Include="System.Runtime.InteropServices.RuntimeInformation" Version="4.3.0" />
1010
<PackageVersion Include="System.IO.Compression" Version="4.3.0" />
11+
12+
<!-- For analyzers, tied to the consumer's build SDK; at the moment, that means "us" -->
13+
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.12.0" />
14+
1115
<!-- Packages only used in the solution, upgrade at will -->
1216
<PackageVersion Include="BenchmarkDotNet" Version="0.15.2" />
1317
<PackageVersion Include="GitHubActionsTestLogger" Version="2.4.1" />
@@ -23,6 +27,7 @@
2327
<PackageVersion Include="StyleCop.Analyzers" Version="1.2.0-beta.556" />
2428
<PackageVersion Include="System.Collections.Immutable" Version="9.0.0" />
2529
<PackageVersion Include="System.Reflection.Metadata" Version="9.0.0" />
30+
2631
<!-- For binding redirect testing, main package gets this transitively -->
2732
<PackageVersion Include="System.IO.Pipelines" Version="9.0.0" />
2833
<PackageVersion Include="System.Runtime.Caching" Version="9.0.0" />

StackExchange.Redis.sln

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs", "docs\docs.csproj",
122122
EndProject
123123
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StackExchange.Redis.Benchmarks", "tests\StackExchange.Redis.Benchmarks\StackExchange.Redis.Benchmarks.csproj", "{59889284-FFEE-82E7-94CB-3B43E87DA6CF}"
124124
EndProject
125+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{5FA0958E-6EBD-45F4-808E-3447A293F96F}"
126+
EndProject
127+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StackExchange.Redis.Build", "eng\StackExchange.Redis.Build\StackExchange.Redis.Build.csproj", "{190742E1-FA50-4E36-A8C4-88AE87654340}"
128+
EndProject
125129
Global
126130
GlobalSection(SolutionConfigurationPlatforms) = preSolution
127131
Debug|Any CPU = Debug|Any CPU
@@ -180,6 +184,10 @@ Global
180184
{59889284-FFEE-82E7-94CB-3B43E87DA6CF}.Debug|Any CPU.Build.0 = Debug|Any CPU
181185
{59889284-FFEE-82E7-94CB-3B43E87DA6CF}.Release|Any CPU.ActiveCfg = Release|Any CPU
182186
{59889284-FFEE-82E7-94CB-3B43E87DA6CF}.Release|Any CPU.Build.0 = Release|Any CPU
187+
{190742E1-FA50-4E36-A8C4-88AE87654340}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
188+
{190742E1-FA50-4E36-A8C4-88AE87654340}.Debug|Any CPU.Build.0 = Debug|Any CPU
189+
{190742E1-FA50-4E36-A8C4-88AE87654340}.Release|Any CPU.ActiveCfg = Release|Any CPU
190+
{190742E1-FA50-4E36-A8C4-88AE87654340}.Release|Any CPU.Build.0 = Release|Any CPU
183191
EndGlobalSection
184192
GlobalSection(SolutionProperties) = preSolution
185193
HideSolutionNode = FALSE
@@ -202,6 +210,7 @@ Global
202210
{A0F89B8B-32A3-4C28-8F1B-ADE343F16137} = {73A5C363-CA1F-44C4-9A9B-EF791A76BA6A}
203211
{69A0ACF2-DF1F-4F49-B554-F732DCA938A3} = {73A5C363-CA1F-44C4-9A9B-EF791A76BA6A}
204212
{59889284-FFEE-82E7-94CB-3B43E87DA6CF} = {73A5C363-CA1F-44C4-9A9B-EF791A76BA6A}
213+
{190742E1-FA50-4E36-A8C4-88AE87654340} = {5FA0958E-6EBD-45F4-808E-3447A293F96F}
205214
EndGlobalSection
206215
GlobalSection(ExtensibilityGlobals) = postSolution
207216
SolutionGuid = {193AA352-6748-47C1-A5FC-C9AA6B5F000B}
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
22
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=OK/@EntryIndexedValue">OK</s:String>
33
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=PONG/@EntryIndexedValue">PONG</s:String>
4-
<s:Boolean x:Key="/Default/UserDictionary/Words/=pubsub/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
4+
<s:Boolean x:Key="/Default/UserDictionary/Words/=pubsub/@EntryIndexedValue">True</s:Boolean>
5+
<s:Boolean x:Key="/Default/UserDictionary/Words/=vectorset/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

docs/ReleaseNotes.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ Current package versions:
1717
- Add `Condition.SortedSet[Not]ContainsStarting` condition for transactions ([#2638 by ArnoKoll](https://github.com/StackExchange/StackExchange.Redis/pull/2638))
1818
- Add support for XPENDING Idle time filter ([#2822 by david-brink-talogy](https://github.com/StackExchange/StackExchange.Redis/pull/2822))
1919
- Improve `double` formatting performance on net8+ ([#2928 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2928))
20+
- Add vector-set support ([#2939 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2939))
2021
- Add `GetServer(RedisKey, ...)` API ([#2936 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2936))
2122
- Fix error constructing `StreamAdd` message ([#2941 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2941))
2223

docs/exp/SER001.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
At the current time, [Redis documents that](https://redis.io/docs/latest/commands/vadd/):
2+
3+
> Vector set is a new data type that is currently in preview and may be subject to change.
4+
5+
As such, the corresponding library feature must also be considered subject to change:
6+
7+
1. Existing bindings may cease working correctly if the underlying server API changes.
8+
2. Changes to the server API may require changes to the library API, manifesting in either/both of build-time
9+
or run-time breaks.
10+
11+
While this seems *unlikely*, it must be considered a possibility. If you acknowledge this, you can suppress
12+
this warning by adding the following to your `csproj` file:
13+
14+
```xml
15+
<NoWarn>$(NoWarn);SER001</NoWarn>
16+
```
17+
18+
or more granularly / locally in C#:
19+
20+
``` c#
21+
#pragma warning disable SER001
22+
```
Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
using System.Buffers;
2+
using System.Collections.Immutable;
3+
using System.Reflection;
4+
using System.Text;
5+
using Microsoft.CodeAnalysis;
6+
using Microsoft.CodeAnalysis.CSharp;
7+
using Microsoft.CodeAnalysis.CSharp.Syntax;
8+
9+
namespace StackExchange.Redis.Build;
10+
11+
[Generator(LanguageNames.CSharp)]
12+
public class FastHashGenerator : IIncrementalGenerator
13+
{
14+
public void Initialize(IncrementalGeneratorInitializationContext context)
15+
{
16+
var literals = context.SyntaxProvider
17+
.CreateSyntaxProvider(Predicate, Transform)
18+
.Where(pair => pair.Name is { Length: > 0 })
19+
.Collect();
20+
21+
context.RegisterSourceOutput(literals, Generate);
22+
}
23+
24+
private bool Predicate(SyntaxNode node, CancellationToken cancellationToken)
25+
{
26+
// looking for [FastHash] partial static class Foo { }
27+
if (node is ClassDeclarationSyntax decl
28+
&& decl.Modifiers.Any(SyntaxKind.StaticKeyword)
29+
&& decl.Modifiers.Any(SyntaxKind.PartialKeyword))
30+
{
31+
foreach (var attribList in decl.AttributeLists)
32+
{
33+
foreach (var attrib in attribList.Attributes)
34+
{
35+
if (attrib.Name.ToString() is "FastHashAttribute" or "FastHash") return true;
36+
}
37+
}
38+
}
39+
40+
return false;
41+
}
42+
43+
private static string GetName(INamedTypeSymbol type)
44+
{
45+
if (type.ContainingType is null) return type.Name;
46+
var stack = new Stack<string>();
47+
while (true)
48+
{
49+
stack.Push(type.Name);
50+
if (type.ContainingType is null) break;
51+
type = type.ContainingType;
52+
}
53+
var sb = new StringBuilder(stack.Pop());
54+
while (stack.Count != 0)
55+
{
56+
sb.Append('.').Append(stack.Pop());
57+
}
58+
return sb.ToString();
59+
}
60+
61+
private (string Namespace, string ParentType, string Name, string Value) Transform(
62+
GeneratorSyntaxContext ctx,
63+
CancellationToken cancellationToken)
64+
{
65+
// extract the name and value (defaults to name, but can be overridden via attribute) and the location
66+
if (ctx.SemanticModel.GetDeclaredSymbol(ctx.Node) is not INamedTypeSymbol named) return default;
67+
string ns = "", parentType = "";
68+
if (named.ContainingType is { } containingType)
69+
{
70+
parentType = GetName(containingType);
71+
ns = containingType.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat);
72+
}
73+
else if (named.ContainingNamespace is { } containingNamespace)
74+
{
75+
ns = containingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat);
76+
}
77+
78+
string name = named.Name, value = "";
79+
foreach (var attrib in named.GetAttributes())
80+
{
81+
if (attrib.AttributeClass?.Name == "FastHashAttribute")
82+
{
83+
if (attrib.ConstructorArguments.Length == 1)
84+
{
85+
if (attrib.ConstructorArguments[0].Value?.ToString() is { Length: > 0 } val)
86+
{
87+
value = val;
88+
break;
89+
}
90+
}
91+
}
92+
}
93+
94+
if (string.IsNullOrWhiteSpace(value))
95+
{
96+
value = name.Replace("_", "-"); // if nothing explicit: infer from name
97+
}
98+
99+
return (ns, parentType, name, value);
100+
}
101+
102+
private string GetVersion()
103+
{
104+
var asm = GetType().Assembly;
105+
if (asm.GetCustomAttributes(typeof(AssemblyFileVersionAttribute), false).FirstOrDefault() is
106+
AssemblyFileVersionAttribute { Version: { Length: > 0 } } version)
107+
{
108+
return version.Version;
109+
}
110+
111+
return asm.GetName().Version?.ToString() ?? "??";
112+
}
113+
114+
private void Generate(
115+
SourceProductionContext ctx,
116+
ImmutableArray<(string Namespace, string ParentType, string Name, string Value)> literals)
117+
{
118+
if (literals.IsDefaultOrEmpty) return;
119+
120+
var sb = new StringBuilder("// <auto-generated />")
121+
.AppendLine().Append("// ").Append(GetType().Name).Append(" v").Append(GetVersion()).AppendLine();
122+
123+
// lease a buffer that is big enough for the longest string
124+
var buffer = ArrayPool<byte>.Shared.Rent(
125+
Encoding.UTF8.GetMaxByteCount(literals.Max(l => l.Value.Length)));
126+
int indent = 0;
127+
128+
StringBuilder NewLine() => sb.AppendLine().Append(' ', indent * 4);
129+
NewLine().Append("using System;");
130+
NewLine().Append("using StackExchange.Redis;");
131+
NewLine().Append("#pragma warning disable CS8981");
132+
foreach (var grp in literals.GroupBy(l => (l.Namespace, l.ParentType)))
133+
{
134+
NewLine();
135+
int braces = 0;
136+
if (!string.IsNullOrWhiteSpace(grp.Key.Namespace))
137+
{
138+
NewLine().Append("namespace ").Append(grp.Key.Namespace);
139+
NewLine().Append("{");
140+
indent++;
141+
braces++;
142+
}
143+
if (!string.IsNullOrWhiteSpace(grp.Key.ParentType))
144+
{
145+
if (grp.Key.ParentType.Contains('.')) // nested types
146+
{
147+
foreach (var part in grp.Key.ParentType.Split('.'))
148+
{
149+
NewLine().Append("partial class ").Append(part);
150+
NewLine().Append("{");
151+
indent++;
152+
braces++;
153+
}
154+
}
155+
else
156+
{
157+
NewLine().Append("partial class ").Append(grp.Key.ParentType);
158+
NewLine().Append("{");
159+
indent++;
160+
braces++;
161+
}
162+
}
163+
164+
foreach (var literal in grp)
165+
{
166+
int len;
167+
unsafe
168+
{
169+
fixed (byte* bPtr = buffer) // netstandard2.0 forces fallback API
170+
{
171+
fixed (char* cPtr = literal.Value)
172+
{
173+
len = Encoding.UTF8.GetBytes(cPtr, literal.Value.Length, bPtr, buffer.Length);
174+
}
175+
}
176+
}
177+
178+
// perform string escaping on the generated value (this includes the quotes, note)
179+
var csValue = SyntaxFactory.LiteralExpression(SyntaxKind.StringLiteralExpression, SyntaxFactory.Literal(literal.Value)).ToFullString();
180+
181+
var hash = FastHash.Hash64(buffer.AsSpan(0, len));
182+
NewLine().Append("static partial class ").Append(literal.Name);
183+
NewLine().Append("{");
184+
indent++;
185+
NewLine().Append("public const int Length = ").Append(len).Append(';');
186+
NewLine().Append("public const long Hash = ").Append(hash).Append(';');
187+
NewLine().Append("public static ReadOnlySpan<byte> U8 => ").Append(csValue).Append("u8;");
188+
NewLine().Append("public const string Text = ").Append(csValue).Append(';');
189+
if (len <= 8)
190+
{
191+
// the hash enforces all the values
192+
NewLine().Append("public static bool Is(long hash, in RawResult value) => hash == Hash && value.Payload.Length == Length;");
193+
NewLine().Append("public static bool Is(long hash, ReadOnlySpan<byte> value) => hash == Hash & value.Length == Length;");
194+
}
195+
else
196+
{
197+
NewLine().Append("public static bool Is(long hash, in RawResult value) => hash == Hash && value.IsEqual(U8);");
198+
NewLine().Append("public static bool Is(long hash, ReadOnlySpan<byte> value) => hash == Hash && value.SequenceEqual(U8);");
199+
}
200+
indent--;
201+
NewLine().Append("}");
202+
}
203+
204+
// handle any closing braces
205+
while (braces-- > 0)
206+
{
207+
indent--;
208+
NewLine().Append("}");
209+
}
210+
}
211+
212+
ArrayPool<byte>.Shared.Return(buffer);
213+
ctx.AddSource("FastHash.generated.cs", sb.ToString());
214+
}
215+
}
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# FastHashGenerator
2+
3+
Efficient matching of well-known short string tokens is a high-volume scenario, for example when matching RESP literals.
4+
5+
The purpose of this generator is to interpret inputs like:
6+
7+
``` c#
8+
[FastHash] public static partial class bin { }
9+
[FastHash] public static partial class f32 { }
10+
```
11+
12+
Usually the token is inferred from the name; `[FastHash("real value")]` can be used if the token is not a valid identifier.
13+
Underscore is replaced with hyphen, so a field called `my_token` has the default value `"my-token"`.
14+
The generator demands *all* of `[FastHash] public static partial class`, and note that any *containing* types must
15+
*also* be declared `partial`.
16+
17+
The output is of the form:
18+
19+
``` c#
20+
static partial class bin
21+
{
22+
public const int Length = 3;
23+
public const long Hash = 7235938;
24+
public static ReadOnlySpan<byte> U8 => @"bin"u8;
25+
public static string Text => @"bin";
26+
public static bool Is(long hash, in RawResult value) => ...
27+
public static bool Is(long hash, in ReadOnlySpan<byte> value) => ...
28+
}
29+
static partial class f32
30+
{
31+
public const int Length = 3;
32+
public const long Hash = 3289958;
33+
public static ReadOnlySpan<byte> U8 => @"f32"u8;
34+
public const string Text = @"f32";
35+
public static bool Is(long hash, in RawResult value) => ...
36+
public static bool Is(long hash, in ReadOnlySpan<byte> value) => ...
37+
}
38+
```
39+
40+
(this API is strictly an internal implementation detail, and can change at any time)
41+
42+
This generated code allows for fast, efficient, and safe matching of well-known tokens, for example:
43+
44+
``` c#
45+
var key = ...
46+
var hash = key.Hash64();
47+
switch (key.Length)
48+
{
49+
case bin.Length when bin.Is(hash, key):
50+
// handle bin
51+
break;
52+
case f32.Length when f32.Is(hash, key):
53+
// handle f32
54+
break;
55+
}
56+
```
57+
58+
The switch on the `Length` is optional, but recommended - these low values can often be implemented (by the compiler)
59+
as a simple jump-table, which is very fast. However, switching on the hash itself is also valid. All hash matches
60+
must also perform a sequence equality check - the `Is(hash, value)` convenience method validates both hash and equality.
61+
62+
Note that `switch` requires `const` values, hence why we use generated *types* rather than partial-properties
63+
that emit an instance with the known values. Also, the `"..."u8` syntax emits a span which is awkward to store, but
64+
easy to return via a property.

0 commit comments

Comments
 (0)