diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index c038ab327..e9c1e2e79 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -8,6 +8,10 @@ Current package versions: ## 2.11.unreleased +- Add support for `VRANGE` ([#3011 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3011)) + +## 2.11.0 + - Add support for `HOTKEYS` ([#3008 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3008)) - Add support for keyspace notifications ([#2995 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2995)) - Add support for idempotent stream entry (`XADD IDMP[AUTO]`) support ([#3006 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/3006)) diff --git a/docs/VectorSets.md b/docs/VectorSets.md new file mode 100644 index 000000000..9a362ec15 --- /dev/null +++ b/docs/VectorSets.md @@ -0,0 +1,394 @@ +# Redis Vector Sets + +Redis Vector Sets provide efficient storage and similarity search for vector data. SE.Redis provides a strongly-typed API for working with vector sets. + +## Prerequisites + +### Redis Version + +Vector Sets require Redis 8.0 or later. + +## Quick Start + +Note that the vectors used in these examples are small for illustrative purposes. In practice, you would commonly use much +larger vectors. The API is designed to efficiently handle large vectors - in particular, the use of `ReadOnlyMemory` +rather than arrays allows you to work with vectors in "pooled" memory buffers (such as `ArrayPool`), which can be more +efficient than creating arrays - or even working with raw memory for example memory-mapped-files. + +### Adding Vectors + +Add vectors to a vector set using `VectorSetAddAsync`: + +```csharp +var db = conn.GetDatabase(); +var key = "product-embeddings"; + +// Create a vector (e.g., from an ML model) +var vector = new[] { 0.1f, 0.2f, 0.3f, 0.4f }; + +// Add a member with its vector +var request = VectorSetAddRequest.Member("product-123", vector.AsMemory()); +bool added = await db.VectorSetAddAsync(key, request); +``` + +### Adding Vectors with Attributes + +You can attach JSON metadata to vectors for filtering: + +```csharp +var vector = new[] { 0.1f, 0.2f, 0.3f, 0.4f }; +var request = VectorSetAddRequest.Member( + "product-123", + vector.AsMemory(), + attributesJson: """{"category":"electronics","price":299.99}""" +); +await db.VectorSetAddAsync(key, request); +``` + +### Similarity Search + +Find similar vectors using `VectorSetSimilaritySearchAsync`: + +```csharp +// Search by an existing member +var query = VectorSetSimilaritySearchRequest.ByMember("product-123"); +query.Count = 10; +query.WithScores = true; + +using var results = await db.VectorSetSimilaritySearchAsync(key, query); +if (results is not null) +{ + foreach (var result in results.Value.Results) + { + Console.WriteLine($"Member: {result.Member}, Score: {result.Score}"); + } +} +``` + +Or search by a vector directly: + +```csharp +var queryVector = new[] { 0.15f, 0.25f, 0.35f, 0.45f }; +var query = VectorSetSimilaritySearchRequest.ByVector(queryVector.AsMemory()); +query.Count = 10; +query.WithScores = true; + +using var results = await db.VectorSetSimilaritySearchAsync(key, query); +``` + +### Filtered Search + +Use JSON path expressions to filter results: + +```csharp +var query = VectorSetSimilaritySearchRequest.ByVector(queryVector.AsMemory()); +query.Count = 10; +query.FilterExpression = "$.category == 'electronics' && $.price < 500"; +query.WithAttributes = true; // Include attributes in results + +using var results = await db.VectorSetSimilaritySearchAsync(key, query); +``` + +See [Redis filtered search documentation](https://redis.io/docs/latest/develop/data-types/vector-sets/filtered-search/) for filter syntax. + +## Vector Set Operations + +### Getting Vector Set Information + +```csharp +var info = await db.VectorSetInfoAsync(key); +if (info is not null) +{ + Console.WriteLine($"Dimension: {info.Value.Dimension}"); + Console.WriteLine($"Length: {info.Value.Length}"); + Console.WriteLine($"Quantization: {info.Value.Quantization}"); +} +``` + +### Checking Membership + +```csharp +bool exists = await db.VectorSetContainsAsync(key, "product-123"); +``` + +### Removing Members + +```csharp +bool removed = await db.VectorSetRemoveAsync(key, "product-123"); +``` + +### Getting Random Members + +```csharp +// Get a single random member +var member = await db.VectorSetRandomMemberAsync(key); + +// Get multiple random members +var members = await db.VectorSetRandomMembersAsync(key, count: 5); +``` + +## Range Queries + +### Getting Members by Lexicographical Range + +Retrieve members in lexicographical order: + +```csharp +// Get all members +using var allMembers = await db.VectorSetRangeAsync(key); +// ... access allMembers.Span, etc + +// Get members in a specific range +using var rangeMembers = await db.VectorSetRangeAsync( + key, + start: "product-100", + end: "product-200", + count: 50 +); +// ... access rangeMembers.Span, etc + +// Exclude boundaries +using var members = await db.VectorSetRangeAsync( + key, + start: "product-100", + end: "product-200", + exclude: Exclude.Both +); +// ... access members.Span, etc +``` + +### Enumerating Large Result Sets + +For large vector sets, use enumeration to process results in batches: + +```csharp +await foreach (var member in db.VectorSetRangeEnumerateAsync(key, count: 100)) +{ + Console.WriteLine($"Processing: {member}"); +} +``` + +The enumeration of results is done in batches, so that the client does not need to buffer the entire result set in memory; +if you exit the loop early, the client and server will stop processing and sending results. This also supports async cancellation: + +```csharp +using var cts = new CancellationTokenSource(); // cancellation not shown + +await foreach (var member in db.VectorSetRangeEnumerateAsync(key, count: 100) + .WithCancellation(cts.Token)) +{ + // ... +} +``` + +## Advanced Configuration + +### Quantization + +Control vector compression: + +```csharp +var request = VectorSetAddRequest.Member("product-123", vector.AsMemory()); +request.Quantization = VectorSetQuantization.Int8; // Default +// or VectorSetQuantization.None +// or VectorSetQuantization.Binary +await db.VectorSetAddAsync(key, request); +``` + +### Dimension Reduction + +Use projection to reduce vector dimensions: + +```csharp +var request = VectorSetAddRequest.Member("product-123", vector.AsMemory()); +request.ReducedDimensions = 128; // Reduce from original dimension +await db.VectorSetAddAsync(key, request); +``` + +### HNSW Parameters + +Fine-tune the HNSW index: + +```csharp +var request = VectorSetAddRequest.Member("product-123", vector.AsMemory()); +request.MaxConnections = 32; // M parameter (default: 16) +request.BuildExplorationFactor = 400; // EF parameter (default: 200) +await db.VectorSetAddAsync(key, request); +``` + +### Search Parameters + +Control search behavior: + +```csharp +var query = VectorSetSimilaritySearchRequest.ByVector(queryVector.AsMemory()); +query.SearchExplorationFactor = 500; // Higher = more accurate, slower +query.Epsilon = 0.1; // Only return similarity >= 0.9 +query.UseExactSearch = true; // Use linear scan instead of HNSW +await db.VectorSetSimilaritySearchAsync(key, query); +``` + +## Working with Vector Data + +### Retrieving Vectors + +Get the approximate vector for a member: + +```csharp +using var vectorLease = await db.VectorSetGetApproximateVectorAsync(key, "product-123"); +if (vectorLease != null) +{ + ReadOnlySpan vector = vectorLease.Value.Span; + // Use the vector data +} +``` + +### Managing Attributes + +Get and set JSON attributes: + +```csharp +// Get attributes +var json = await db.VectorSetGetAttributesJsonAsync(key, "product-123"); + +// Set attributes +await db.VectorSetSetAttributesJsonAsync( + key, + "product-123", + """{"category":"electronics","updated":"2024-01-15"}""" +); +``` + +### Graph Links + +Inspect HNSW graph connections: + +```csharp +// Get linked members +using var links = await db.VectorSetGetLinksAsync(key, "product-123"); +if (links != null) +{ + foreach (var link in links.Value.Span) + { + Console.WriteLine($"Linked to: {link}"); + } +} + +// Get links with similarity scores +using var linksWithScores = await db.VectorSetGetLinksWithScoresAsync(key, "product-123"); +if (linksWithScores != null) +{ + foreach (var link in linksWithScores.Value.Span) + { + Console.WriteLine($"Linked to: {link.Member}, Score: {link.Score}"); + } +} +``` + +## Memory Management + +Vector operations return `Lease` for efficient memory pooling. Always dispose leases: + +```csharp +// Using statement (recommended) +using var results = await db.VectorSetSimilaritySearchAsync(key, query); + +// Or explicit disposal +var results = await db.VectorSetSimilaritySearchAsync(key, query); +try +{ + // Use results +} +finally +{ + results?.Dispose(); +} +``` + +## Performance Considerations + +### Batch Operations + +For bulk inserts, consider using pipelining: + +```csharp +var batch = db.CreateBatch(); +var tasks = new List>(); + +foreach (var (member, vector) in vectorData) +{ + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + tasks.Add(batch.VectorSetAddAsync(key, request)); +} + +batch.Execute(); +await Task.WhenAll(tasks); +``` + +### Search Optimization + +- Use **quantization** to reduce memory usage and improve search speed +- Tune **SearchExplorationFactor** based on accuracy vs. speed requirements +- Use **filters** to reduce the search space +- Consider **dimension reduction** for very high-dimensional vectors + +### Range Query Pagination + +Prefer enumeration for large result sets to avoid loading everything into memory: + +```csharp +// Good: loads results in batches, processes items individually +await foreach (var member in db.VectorSetRangeEnumerateAsync(key)) +{ + await ProcessMemberAsync(member); +} + +// Avoid: loads all results at once +using var allMembers1 = await db.VectorSetRangeAsync(key); + +// Avoid: loads results in batches, but still loads everything into memory at once +var allMembers2 = await db.VectorSetRangeEnumerateAsync(key).ToArrayAsync(); +``` + +## Common Patterns + +### Semantic Search + +```csharp +// 1. Store document embeddings +var embedding = await GetEmbeddingFromMLModel(document); +var request = VectorSetAddRequest.Member( + documentId, + embedding.AsMemory(), + attributesJson: $$"""{"title":"{{document.Title}}","date":"{{document.Date}}"}""" +); +await db.VectorSetAddAsync("documents", request); + +// 2. Search for similar documents +var queryEmbedding = await GetEmbeddingFromMLModel(searchQuery); +var query = VectorSetSimilaritySearchRequest.ByVector(queryEmbedding.AsMemory()); +query.Count = 10; +query.WithScores = true; +query.WithAttributes = true; + +using var results = await db.VectorSetSimilaritySearchAsync("documents", query); +``` + +### Recommendation System + +```csharp +// Find similar items based on an item the user liked +var query = VectorSetSimilaritySearchRequest.ByMember(userLikedItemId); +query.Count = 20; +query.FilterExpression = "$.inStock == true && $.price < 100"; +query.WithScores = true; + +using var recommendations = await db.VectorSetSimilaritySearchAsync("products", query); +``` + +## See Also + +- [Redis Vector Sets Documentation](https://redis.io/docs/latest/develop/data-types/vector-sets/) +- [HNSW Algorithm](https://arxiv.org/abs/1603.09320) +- [Filtered Search Syntax](https://redis.io/docs/latest/develop/data-types/vector-sets/filtered-search/) + diff --git a/docs/index.md b/docs/index.md index 25bdd942e..8a846cf4f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,6 +45,7 @@ Documentation - [Using RESP3](Resp3) - information on using RESP3 - [ServerMaintenanceEvent](ServerMaintenanceEvent) - how to listen and prepare for hosted server maintenance (e.g. Azure Cache for Redis) - [Streams](Streams) - how to use the Stream data type +- [Vector Sets](VectorSets) - how to use Vector Sets for similarity search with embeddings - [Where are `KEYS` / `SCAN` / `FLUSH*`?](KeysScan) - how to use server-based commands - [Profiling](Profiling) - profiling interfaces, as well as how to profile in an `async` world - [Scripting](Scripting) - running Lua scripts with convenient named parameter replacement diff --git a/src/StackExchange.Redis/Enums/RedisCommand.cs b/src/StackExchange.Redis/Enums/RedisCommand.cs index c55a39d8a..2a4d1180f 100644 --- a/src/StackExchange.Redis/Enums/RedisCommand.cs +++ b/src/StackExchange.Redis/Enums/RedisCommand.cs @@ -219,6 +219,7 @@ internal enum RedisCommand VISMEMBER, VLINKS, VRANDMEMBER, + VRANGE, VREM, VSETATTR, VSIM, @@ -533,6 +534,7 @@ internal static bool IsPrimaryOnly(this RedisCommand command) case RedisCommand.VISMEMBER: case RedisCommand.VLINKS: case RedisCommand.VRANDMEMBER: + case RedisCommand.VRANGE: case RedisCommand.VSIM: // Writable commands, but allowed for the writable-replicas scenario case RedisCommand.COPY: diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs b/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs index 039075ec8..1c163f315 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs @@ -180,4 +180,44 @@ bool VectorSetSetAttributesJson( RedisKey key, VectorSetSimilaritySearchRequest query, CommandFlags flags = CommandFlags.None); + + /// + /// Get a range of members from a vectorset by lexicographical order. + /// + /// The key of the vectorset. + /// The minimum value to filter by (inclusive by default). + /// The maximum value to filter by (inclusive by default). + /// The maximum number of members to return (-1 for all). + /// Whether to exclude the start and/or end values. + /// The flags to use for this operation. + /// Members in the specified range as a pooled memory lease. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Lease VectorSetRange( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = -1, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None); + + /// + /// Enumerate members from a vectorset by lexicographical order in batches. + /// + /// The key of the vectorset. + /// The minimum value to filter by (inclusive by default). + /// The maximum value to filter by (inclusive by default). + /// The batch size for each iteration. + /// Whether to exclude the start and/or end values. + /// The flags to use for this operation. + /// An enumerable of members in the specified range. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + System.Collections.Generic.IEnumerable VectorSetRangeEnumerate( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = 100, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None); } diff --git a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs index 3ac67d40f..7b8825e4c 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs @@ -93,6 +93,26 @@ Task VectorSetSetAttributesJsonAsync( VectorSetSimilaritySearchRequest query, CommandFlags flags = CommandFlags.None); + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task?> VectorSetRangeAsync( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = -1, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + System.Collections.Generic.IAsyncEnumerable VectorSetRangeEnumerateAsync( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = 100, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None); + /// Task StreamReadGroupAsync(StreamPosition[] streamPositions, RedisValue groupName, RedisValue consumerName, int? countPerStream = null, bool noAck = false, TimeSpan? claimMinIdleTime = null, CommandFlags flags = CommandFlags.None); } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.VectorSets.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.VectorSets.cs index 809adad97..ae7498401 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.VectorSets.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.VectorSets.cs @@ -56,4 +56,22 @@ public Task VectorSetSetAttributesJsonAsync(RedisKey key, RedisValue membe VectorSetSimilaritySearchRequest query, CommandFlags flags = CommandFlags.None) => Inner.VectorSetSimilaritySearchAsync(ToInner(key), query, flags); + + public Task?> VectorSetRangeAsync( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = -1, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRangeAsync(ToInner(key), start, end, count, exclude, flags); + + public System.Collections.Generic.IAsyncEnumerable VectorSetRangeEnumerateAsync( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = 100, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRangeEnumerateAsync(ToInner(key), start, end, count, exclude, flags); } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.VectorSets.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.VectorSets.cs index 62f4e9202..83fbb2f85 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.VectorSets.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.VectorSets.cs @@ -53,4 +53,22 @@ public bool VectorSetSetAttributesJson(RedisKey key, RedisValue member, string a VectorSetSimilaritySearchRequest query, CommandFlags flags = CommandFlags.None) => Inner.VectorSetSimilaritySearch(ToInner(key), query, flags); + + public Lease VectorSetRange( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = -1, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRange(ToInner(key), start, end, count, exclude, flags); + + public System.Collections.Generic.IEnumerable VectorSetRangeEnumerate( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = 100, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRangeEnumerate(ToInner(key), start, end, count, exclude, flags); } diff --git a/src/StackExchange.Redis/Lease.cs b/src/StackExchange.Redis/Lease.cs index 91495dd08..a5a88e4eb 100644 --- a/src/StackExchange.Redis/Lease.cs +++ b/src/StackExchange.Redis/Lease.cs @@ -18,6 +18,11 @@ public sealed class Lease : IMemoryOwner private T[]? _arr; + /// + /// Gets whether this lease is empty. + /// + public bool IsEmpty => Length == 0; + /// /// The length of the lease. /// diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index ab058de62..b12b9f826 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,6 @@ #nullable enable +StackExchange.Redis.Lease.IsEmpty.get -> bool +[SER001]StackExchange.Redis.IDatabase.VectorSetRange(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue start = default(StackExchange.Redis.RedisValue), StackExchange.Redis.RedisValue end = default(StackExchange.Redis.RedisValue), long count = -1, StackExchange.Redis.Exclude exclude = StackExchange.Redis.Exclude.None, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.Lease! +[SER001]StackExchange.Redis.IDatabase.VectorSetRangeEnumerate(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue start = default(StackExchange.Redis.RedisValue), StackExchange.Redis.RedisValue end = default(StackExchange.Redis.RedisValue), long count = 100, StackExchange.Redis.Exclude exclude = StackExchange.Redis.Exclude.None, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Collections.Generic.IEnumerable! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetRangeAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue start = default(StackExchange.Redis.RedisValue), StackExchange.Redis.RedisValue end = default(StackExchange.Redis.RedisValue), long count = -1, StackExchange.Redis.Exclude exclude = StackExchange.Redis.Exclude.None, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task?>! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetRangeEnumerateAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue start = default(StackExchange.Redis.RedisValue), StackExchange.Redis.RedisValue end = default(StackExchange.Redis.RedisValue), long count = 100, StackExchange.Redis.Exclude exclude = StackExchange.Redis.Exclude.None, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Collections.Generic.IAsyncEnumerable! diff --git a/src/StackExchange.Redis/RedisDatabase.VectorSets.cs b/src/StackExchange.Redis/RedisDatabase.VectorSets.cs index 9b3f1b43b..f10693dc5 100644 --- a/src/StackExchange.Redis/RedisDatabase.VectorSets.cs +++ b/src/StackExchange.Redis/RedisDatabase.VectorSets.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; // ReSharper disable once CheckNamespace @@ -188,4 +191,107 @@ public Task VectorSetSetAttributesJsonAsync(RedisKey key, RedisValue membe var msg = query.ToMessage(key, Database, flags); return ExecuteAsync(msg, msg.GetResultProcessor()); } + + private Message GetVectorSetRangeMessage( + in RedisKey key, + in RedisValue start, + in RedisValue end, + long count, + Exclude exclude, + CommandFlags flags) + { + static RedisValue GetTerminator(RedisValue value, Exclude exclude, bool isStart) + { + if (value.IsNull) return isStart ? RedisLiterals.MinusSymbol : RedisLiterals.PlusSymbol; + var mask = isStart ? Exclude.Start : Exclude.Stop; + var isExclusive = (exclude & mask) != 0; + return (isExclusive ? "(" : "[") + value; + } + + var from = GetTerminator(start, exclude, true); + var to = GetTerminator(end, exclude, false); + return count < 0 + ? Message.Create(Database, flags, RedisCommand.VRANGE, key, from, to) + : Message.Create(Database, flags, RedisCommand.VRANGE, key, from, to, count); + } + + public Lease VectorSetRange( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = -1, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None) + { + var msg = GetVectorSetRangeMessage(key, start, end, count, exclude, flags); + return ExecuteSync(msg, ResultProcessor.LeaseRedisValue)!; + } + + public Task?> VectorSetRangeAsync( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = -1, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None) + { + var msg = GetVectorSetRangeMessage(key, start, end, count, exclude, flags); + return ExecuteAsync(msg, ResultProcessor.LeaseRedisValue); + } + + public IEnumerable VectorSetRangeEnumerate( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = 100, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None) + { + // intentionally not using "scan" naming in case a VSCAN command is added later + while (true) + { + using var batch = VectorSetRange(key, start, end, count, exclude, flags); + exclude |= Exclude.Start; // on subsequent iterations, exclude the start (we've already yielded it) + + if (batch is null || batch.IsEmpty) yield break; + var segment = batch.ArraySegment; + for (int i = 0; i < segment.Count; i++) + { + // note side effect: use the last value as the exclusive start of the next batch + yield return start = segment.Array![segment.Offset + i]; + } + if (batch.Length < count || (!end.IsNull && end == start)) yield break; // no need to issue a final query + } + } + + public IAsyncEnumerable VectorSetRangeEnumerateAsync( + RedisKey key, + RedisValue start = default, + RedisValue end = default, + long count = 100, + Exclude exclude = Exclude.None, + CommandFlags flags = CommandFlags.None) + { + // intentionally not using "scan" naming in case a VSCAN command is added later + return WithCancellationSupport(CancellationToken.None); + + async IAsyncEnumerable WithCancellationSupport([EnumeratorCancellation] CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + using var batch = await VectorSetRangeAsync(key, start, end, count, exclude, flags); + exclude |= Exclude.Start; // on subsequent iterations, exclude the start (we've already yielded it) + + if (batch is null || batch.IsEmpty) yield break; + var segment = batch.ArraySegment; + for (int i = 0; i < segment.Count; i++) + { + // note side effect: use the last value as the exclusive start of the next batch + yield return start = segment.Array![segment.Offset + i]; + } + if (batch.Length < count || (!end.IsNull && end == start)) yield break; // no need to issue a final query + } + } + } } diff --git a/src/StackExchange.Redis/ResultProcessor.VectorSets.cs b/src/StackExchange.Redis/ResultProcessor.VectorSets.cs index 8743ebd0b..b10e5fd93 100644 --- a/src/StackExchange.Redis/ResultProcessor.VectorSets.cs +++ b/src/StackExchange.Redis/ResultProcessor.VectorSets.cs @@ -11,6 +11,8 @@ internal abstract partial class ResultProcessor public static readonly ResultProcessor?> VectorSetLinks = new VectorSetLinksProcessor(); + public static readonly ResultProcessor?> LeaseRedisValue = new LeaseRedisValueProcessor(); + public static ResultProcessor VectorSetInfo = new VectorSetInfoProcessor(); private sealed class VectorSetLinksWithScoresProcessor : FlattenedLeaseProcessor @@ -43,6 +45,15 @@ protected override bool TryReadOne(in RawResult result, out RedisValue value) } } + private sealed class LeaseRedisValueProcessor : LeaseProcessor + { + protected override bool TryParse(in RawResult raw, out RedisValue parsed) + { + parsed = raw.AsRedisValue(); + return true; + } + } + private sealed partial class VectorSetInfoProcessor : ResultProcessor { protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) diff --git a/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs b/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs index 12eda7147..fb8e5d52a 100644 --- a/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs +++ b/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs @@ -1,6 +1,7 @@ using System; using System.Globalization; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; using Xunit; @@ -672,4 +673,488 @@ public async Task VectorSetGetLinksWithScores() Assert.True(linksArray.First(l => l.Member == "element2").Score > 0.9); // similar Assert.True(linksArray.First(l => l.Member == "element3").Score < 0.8); // less-so } + + [Fact] + public async Task VectorSetRange_BasicOperation() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + // Add members with lexicographically ordered names + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var members = new[] { "alpha", "beta", "delta", "gamma" }; // note: delta before gamma because lexicographical + + foreach (var member in members) + { + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Get all members - should be in lexicographical order + using var result = await db.VectorSetRangeAsync(key); + + Assert.NotNull(result); + Assert.Equal(4, result.Length); + // Lexicographical order: alpha, beta, delta, gamma + Assert.Equal(new[] { "alpha", "beta", "delta", "gamma" }, result.Span.ToArray().Select(r => (string?)r).ToArray()); + } + + [Fact] + public async Task VectorSetRange_WithStartAndEnd() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var members = new[] { "apple", "banana", "cherry", "date", "elderberry" }; + + foreach (var member in members) + { + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Get range from "banana" to "date" (inclusive) + using var result = await db.VectorSetRangeAsync(key, start: "banana", end: "date"); + + Assert.NotNull(result); + Assert.Equal(3, result.Length); + Assert.Equal(new[] { "banana", "cherry", "date" }, result.Span.ToArray().Select(r => (string?)r).ToArray()); + } + + [Fact] + public async Task VectorSetRange_WithCount() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + + // Add 10 members + for (int i = 0; i < 10; i++) + { + var request = VectorSetAddRequest.Member($"member{i}", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Get only 5 members + using var result = await db.VectorSetRangeAsync(key, count: 5); + + Assert.NotNull(result); + Assert.Equal(5, result.Length); + } + + [Fact] + public async Task VectorSetRange_WithExcludeStart() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var members = new[] { "a", "b", "c", "d" }; + + foreach (var member in members) + { + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Get range excluding start + using var result = await db.VectorSetRangeAsync(key, start: "a", end: "d", exclude: Exclude.Start); + + Assert.NotNull(result); + Assert.Equal(3, result.Length); + Assert.Equal(new[] { "b", "c", "d" }, result.Span.ToArray().Select(r => (string?)r).ToArray()); + } + + [Fact] + public async Task VectorSetRange_WithExcludeEnd() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var members = new[] { "a", "b", "c", "d" }; + + foreach (var member in members) + { + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Get range excluding end + using var result = await db.VectorSetRangeAsync(key, start: "a", end: "d", exclude: Exclude.Stop); + + Assert.NotNull(result); + Assert.Equal(3, result.Length); + Assert.Equal(new[] { "a", "b", "c" }, result.Span.ToArray().Select(r => (string?)r).ToArray()); + } + + [Fact] + public async Task VectorSetRange_WithExcludeBoth() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var members = new[] { "a", "b", "c", "d", "e" }; + + foreach (var member in members) + { + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Get range excluding both boundaries + using var result = await db.VectorSetRangeAsync(key, start: "a", end: "e", exclude: Exclude.Both); + + Assert.NotNull(result); + Assert.Equal(3, result.Length); + Assert.Equal(new[] { "b", "c", "d" }, result.Span.ToArray().Select(r => (string?)r).ToArray()); + } + + [Fact] + public async Task VectorSetRange_EmptySet() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + // Don't add any members + using var result = await db.VectorSetRangeAsync(key); + + Assert.NotNull(result); + Assert.Empty(result.Span.ToArray()); + } + + [Fact] + public async Task VectorSetRange_NoMatches() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var members = new[] { "a", "b", "c" }; + + foreach (var member in members) + { + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Query range with no matching members + using var result = await db.VectorSetRangeAsync(key, start: "x", end: "z"); + + Assert.NotNull(result); + Assert.Empty(result.Span.ToArray()); + } + + [Fact] + public async Task VectorSetRange_OpenStart() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var members = new[] { "alpha", "beta", "gamma" }; + + foreach (var member in members) + { + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Get from beginning to "beta" + using var result = await db.VectorSetRangeAsync(key, end: "beta"); + + Assert.NotNull(result); + Assert.Equal(2, result.Length); + Assert.Equal(new[] { "alpha", "beta" }, result.Span.ToArray().Select(r => (string?)r).ToArray()); + } + + [Fact] + public async Task VectorSetRange_OpenEnd() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var members = new[] { "alpha", "beta", "gamma" }; + + foreach (var member in members) + { + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Get from "beta" to end + using var result = await db.VectorSetRangeAsync(key, start: "beta"); + + Assert.NotNull(result); + Assert.Equal(2, result.Length); + Assert.Equal(new[] { "beta", "gamma" }, result.Span.ToArray().Select(r => (string?)r).ToArray()); + } + + [Fact] + public async Task VectorSetRange_SyncVsAsync() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + + // Add 20 members + for (int i = 0; i < 20; i++) + { + var request = VectorSetAddRequest.Member($"m{i:D2}", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Call both sync and async + using var syncResult = db.VectorSetRange(key, start: "m05", end: "m15"); + using var asyncResult = await db.VectorSetRangeAsync(key, start: "m05", end: "m15"); + + Assert.NotNull(syncResult); + Assert.NotNull(asyncResult); + Assert.Equal(syncResult.Length, asyncResult.Length); + Assert.Equal(syncResult.Span.ToArray().Select(r => (string?)r), asyncResult.Span.ToArray().Select(r => (string?)r)); + } + + [Fact] + public async Task VectorSetRange_WithNumericLexOrder() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var members = new[] { "1", "10", "2", "20", "3" }; + + foreach (var member in members) + { + var request = VectorSetAddRequest.Member(member, vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Get all - should be in lexicographical order, not numeric + using var result = await db.VectorSetRangeAsync(key); + + Assert.NotNull(result); + Assert.Equal(5, result.Length); + // Lexicographical order: "1", "10", "2", "20", "3" + Assert.Equal(new[] { "1", "10", "2", "20", "3" }, result.Span.ToArray().Select(r => (string?)r).ToArray()); + } + + [Fact] + public async Task VectorSetRangeEnumerate_BasicIteration() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + + // Add 50 members + for (int i = 0; i < 50; i++) + { + var request = VectorSetAddRequest.Member($"member{i:D3}", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Enumerate with batch size of 10 + var allMembers = new System.Collections.Generic.List(); + foreach (var member in db.VectorSetRangeEnumerate(key, count: 10)) + { + allMembers.Add(member); + } + + Assert.Equal(50, allMembers.Count); + + // Verify lexicographical order + var sorted = allMembers.OrderBy(m => (string?)m, StringComparer.Ordinal).ToList(); + Assert.Equal(sorted, allMembers); + } + + [Fact] + public async Task VectorSetRangeEnumerate_WithRange() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + + // Add members "a" through "z" + for (char c = 'a'; c <= 'z'; c++) + { + var request = VectorSetAddRequest.Member(c.ToString(), vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Enumerate from "f" to "p" with batch size 5 + var allMembers = new System.Collections.Generic.List(); + foreach (var member in db.VectorSetRangeEnumerate(key, start: "f", end: "p", count: 5)) + { + allMembers.Add(member); + } + + // Should get "f" through "p" inclusive (11 members) + Assert.Equal(11, allMembers.Count); + Assert.Equal("f", (string?)allMembers.First()); + Assert.Equal("p", (string?)allMembers.Last()); + } + + [Fact] + public async Task VectorSetRangeEnumerate_EarlyBreak() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + + // Add 100 members + for (int i = 0; i < 100; i++) + { + var request = VectorSetAddRequest.Member($"member{i:D3}", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Take only first 25 members + var limitedMembers = db.VectorSetRangeEnumerate(key, count: 10).Take(25).ToList(); + + Assert.Equal(25, limitedMembers.Count); + } + + [Fact] + public async Task VectorSetRangeEnumerate_EmptyBatches() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + // Don't add any members + var allMembers = new System.Collections.Generic.List(); + foreach (var member in db.VectorSetRangeEnumerate(key)) + { + allMembers.Add(member); + } + + Assert.Empty(allMembers); + } + + [Fact] + public async Task VectorSetRangeEnumerateAsync_BasicIteration() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + + // Add 50 members + for (int i = 0; i < 50; i++) + { + var request = VectorSetAddRequest.Member($"member{i:D3}", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + // Enumerate with batch size of 10 + var allMembers = new System.Collections.Generic.List(); + await foreach (var member in db.VectorSetRangeEnumerateAsync(key, count: 10)) + { + allMembers.Add(member); + } + + Assert.Equal(50, allMembers.Count); + + // Verify lexicographical order + var sorted = allMembers.OrderBy(m => (string?)m, StringComparer.Ordinal).ToList(); + Assert.Equal(sorted, allMembers); + } + + [Fact] + public async Task VectorSetRangeEnumerateAsync_WithCancellation() + { + await using var conn = Create(require: RedisFeatures.v8_4_0_rc1); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + + // Add 100 members + for (int i = 0; i < 100; i++) + { + var request = VectorSetAddRequest.Member($"member{i:D3}", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + } + + using var cts = new CancellationTokenSource(); + var allMembers = new System.Collections.Generic.List(); + + // Start enumeration and cancel after collecting some members + await Assert.ThrowsAnyAsync(async () => + { + await foreach (var member in db.VectorSetRangeEnumerateAsync(key, count: 10).WithCancellation(cts.Token)) + { + allMembers.Add(member); + + // Cancel after we've collected 25 members + if (allMembers.Count == 25) + { + cts.Cancel(); + } + } + }); + + // Should have stopped at or shortly after 25 members + Log($"Expected ~25 members, got {allMembers.Count}"); + Assert.True(allMembers.Count >= 25 && allMembers.Count <= 35, $"Expected ~25 members, got {allMembers.Count}"); + } }