From 7951715091ecde32c11f5c82d37658d84aba0d47 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 15 Oct 2026 11:50:43 +0100 Subject: [PATCH 1/2] Search: SQ8 compression and training threshold for HNSW vector fields (Redis 8.12) RediSearch MOD-14958 adds `COMPRESSION SQ8` and `TRAINING_THRESHOLD` to HNSW vector fields: 8-bit scalar quantization of the stored vectors for FLOAT32/FLOAT16 with L2, IP or cosine, with an optional mean learned from the first N vectors (server default 10240, maximum 102400, 0 = no training) while they stay searchable in the flat tier. HnswVectorField gains CompressionAlgorithm and TrainingThreshold, emitted after the graph parameters when set, and AddHnswVectorField gains the matching optional parameters (appended, so positional callers are unaffected at the source level). VectorCompressionAlgorithm gains SQ8; the token mapping is now shared with the SVS-VAMANA field. No client-side validation of type/metric/threshold combinations: the server owns those rules and reports them clearly ("COMPRESSION is only supported for FLOAT32 and FLOAT16 vector types", "TRAINING_THRESHOLD is irrelevant when compression was not requested"), and the tests pin that the errors surface as RedisServerException. FT.INFO's new `compression` and `training_threshold` attribute keys flow through the existing untyped Attributes dictionaries. Tests are gated to 8.12.0 and were run against the 8.12-m02-int image, standalone and cluster, RESP2 and RESP3. --- .../PublicAPI/PublicAPI.Unshipped.txt | 7 ++ src/NRedisStack/Search/Schema.cs | 70 ++++++++++++--- .../Search/IndexCreationTests.cs | 88 +++++++++++++++++++ 3 files changed, 152 insertions(+), 13 deletions(-) diff --git a/src/NRedisStack/PublicAPI/PublicAPI.Unshipped.txt b/src/NRedisStack/PublicAPI/PublicAPI.Unshipped.txt index 7dc5c581..097d8891 100644 --- a/src/NRedisStack/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/NRedisStack/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,8 @@ #nullable enable +NRedisStack.Search.Schema.AddHnswVectorField(NRedisStack.Search.FieldName! name, NRedisStack.Search.Schema.VectorField.VectorType type, int dimensions, NRedisStack.Search.Schema.VectorField.VectorDistanceMetric distanceMetric, int maxOutgoingConnections = 16, int maxConnectedNeighbors = 200, int maxTopCandidates = 10, double boundaryFactor = 0.01, System.Collections.Generic.Dictionary? attributes = null, bool missingIndex = false, NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm compressionAlgorithm = NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm.NotSpecified, int trainingThreshold = 0) -> NRedisStack.Search.Schema! +NRedisStack.Search.Schema.HnswVectorField.CompressionAlgorithm.get -> NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm +NRedisStack.Search.Schema.HnswVectorField.CompressionAlgorithm.set -> void +NRedisStack.Search.Schema.HnswVectorField.TrainingThreshold.get -> int +NRedisStack.Search.Schema.HnswVectorField.TrainingThreshold.set -> void +NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm.SQ8 = 7 -> NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm +*REMOVED*NRedisStack.Search.Schema.AddHnswVectorField(NRedisStack.Search.FieldName! name, NRedisStack.Search.Schema.VectorField.VectorType type, int dimensions, NRedisStack.Search.Schema.VectorField.VectorDistanceMetric distanceMetric, int maxOutgoingConnections = 16, int maxConnectedNeighbors = 200, int maxTopCandidates = 10, double boundaryFactor = 0.01, System.Collections.Generic.Dictionary? attributes = null, bool missingIndex = false) -> NRedisStack.Search.Schema! diff --git a/src/NRedisStack/Search/Schema.cs b/src/NRedisStack/Search/Schema.cs index 716d8b9b..8f3757c4 100644 --- a/src/NRedisStack/Search/Schema.cs +++ b/src/NRedisStack/Search/Schema.cs @@ -426,7 +426,9 @@ public enum VectorDistanceMetric } /// - /// The distance metric to use for vector similarity search. + /// The vector compression ("COMPRESSION") to apply to stored vectors. Which values an algorithm accepts is + /// decided by the server: the LVQ and LeanVec values are for , and + /// is for ; the client sends what it is given. /// public enum VectorCompressionAlgorithm { @@ -444,7 +446,26 @@ public enum VectorCompressionAlgorithm LeanVec4x8 = 5, // ReSharper disable once InconsistentNaming LeanVec8x8 = 6, + /// + /// 8-bit scalar quantization of the stored vectors, for fields of type + /// or (Redis 8.12). An optional mean is + /// learned from the first vectors indexed; see . + /// + SQ8 = 7, } + + /// The "COMPRESSION" token for a . + internal static string CompressionToken(VectorCompressionAlgorithm compression) => compression switch + { + VectorCompressionAlgorithm.LVQ8 => "LVQ8", + VectorCompressionAlgorithm.LVQ4 => "LVQ4", + VectorCompressionAlgorithm.LVQ4x4 => "LVQ4x4", + VectorCompressionAlgorithm.LVQ4x8 => "LVQ4x8", + VectorCompressionAlgorithm.LeanVec4x8 => "LeanVec4x8", + VectorCompressionAlgorithm.LeanVec8x8 => "LeanVec8x8", + VectorCompressionAlgorithm.SQ8 => "SQ8", + _ => compression.ToString(), // fallback + }; } @@ -493,6 +514,22 @@ public class HnswVectorField( /// public double BoundaryFactor { get; set; } = DEFAULT_EPSILON; + /// + /// "COMPRESSION"; how stored vectors are compressed. HNSW accepts + /// (Redis 8.12), for and with the L2, IP or + /// cosine metric; anything else is rejected by the server. Not sent when . + /// + public VectorCompressionAlgorithm CompressionAlgorithm { get; set; } = VectorCompressionAlgorithm.NotSpecified; + + /// + /// "TRAINING_THRESHOLD"; with , how many vectors are collected before the + /// quantization mean is learned and vectors migrate into the compressed graph. Vectors remain searchable + /// meanwhile. The server's default is 10240 and its maximum 102400; zero on the server means "do not train". + /// Not sent when zero here - set to send an explicit zero. Only valid + /// together with a compression; the server rejects it otherwise. + /// + public int TrainingThreshold { get; set; } + internal const int DEFAULT_M = 16, DEFAULT_EF_CONSTRUCTION = 200, DEFAULT_EF_RUNTIME = 10; internal const double DEFAULT_EPSILON = 0.01; internal override int DirectAttributeCount @@ -501,7 +538,9 @@ internal override int DirectAttributeCount + (MaxConnectedNeighbors == DEFAULT_EF_CONSTRUCTION ? 0 : 1) + (MaxTopCandidates == DEFAULT_EF_RUNTIME ? 0 : 1) // ReSharper disable once CompareOfFloatsByEqualityOperator - + (BoundaryFactor == DEFAULT_EPSILON ? 0 : 1); + + (BoundaryFactor == DEFAULT_EPSILON ? 0 : 1) + + (CompressionAlgorithm == VectorCompressionAlgorithm.NotSpecified ? 0 : 1) + + (TrainingThreshold == 0 ? 0 : 1); internal override void AddDirectAttributes(List args) { @@ -527,6 +566,16 @@ internal override void AddDirectAttributes(List args) args.Add("EPSILON"); args.Add(BoundaryFactor); } + if (CompressionAlgorithm != VectorCompressionAlgorithm.NotSpecified) + { + args.Add("COMPRESSION"); + args.Add(CompressionToken(CompressionAlgorithm)); + } + if (TrainingThreshold != 0) + { + args.Add("TRAINING_THRESHOLD"); + args.Add(TrainingThreshold); + } } } @@ -560,16 +609,7 @@ internal override void AddDirectAttributes(List args) if (CompressionAlgorithm != VectorCompressionAlgorithm.NotSpecified) { args.Add("COMPRESSION"); - args.Add(CompressionAlgorithm switch - { - VectorCompressionAlgorithm.LVQ8 => "LVQ8", - VectorCompressionAlgorithm.LVQ4 => "LVQ4", - VectorCompressionAlgorithm.LVQ4x4 => "LVQ4x4", - VectorCompressionAlgorithm.LVQ4x8 => "LVQ4x8", - VectorCompressionAlgorithm.LeanVec4x8 => "LeanVec4x8", - VectorCompressionAlgorithm.LeanVec8x8 => "LeanVec8x8", - _ => CompressionAlgorithm.ToString(), // fallback - }); + args.Add(CompressionToken(CompressionAlgorithm)); } base.AddDirectAttributes(args); if (ConstructionWindowSize != DEFAULT_CONSTRUCTION_WINDOW_SIZE) @@ -874,7 +914,9 @@ public Schema AddHnswVectorField(FieldName name, VectorType type, int dimensions int maxConnectedNeighbors = HnswVectorField.DEFAULT_EF_CONSTRUCTION, int maxTopCandidates = HnswVectorField.DEFAULT_EF_RUNTIME, double boundaryFactor = HnswVectorField.DEFAULT_EPSILON, - Dictionary? attributes = null, bool missingIndex = false) + Dictionary? attributes = null, bool missingIndex = false, + VectorCompressionAlgorithm compressionAlgorithm = VectorCompressionAlgorithm.NotSpecified, + int trainingThreshold = 0) { Fields.Add(new HnswVectorField(name, type, dimensions, distanceMetric, attributes, missingIndex) { @@ -882,6 +924,8 @@ public Schema AddHnswVectorField(FieldName name, VectorType type, int dimensions MaxConnectedNeighbors = maxConnectedNeighbors, MaxTopCandidates = maxTopCandidates, BoundaryFactor = boundaryFactor, + CompressionAlgorithm = compressionAlgorithm, + TrainingThreshold = trainingThreshold, }); return this; } diff --git a/tests/NRedisStack.Tests/Search/IndexCreationTests.cs b/tests/NRedisStack.Tests/Search/IndexCreationTests.cs index 22bc2f0f..55ac24d6 100644 --- a/tests/NRedisStack.Tests/Search/IndexCreationTests.cs +++ b/tests/NRedisStack.Tests/Search/IndexCreationTests.cs @@ -379,6 +379,94 @@ public void TestCreate_Float16_Int32_VectorField_Svs_WithCompression(string endp Assert.Equal(2, res.TotalResults); } + [Fact] + public void TestIndexingCreation_Hnsw_Sq8() + { + // COMPRESSION and TRAINING_THRESHOLD are emitted only when set, after the graph parameters + Schema sc = new Schema() + .AddHnswVectorField("v1", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8) + .AddHnswVectorField("v2", Schema.VectorField.VectorType.FLOAT16, 4, + Schema.VectorField.VectorDistanceMetric.CosineDistance, + maxOutgoingConnections: 8, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8, trainingThreshold: 2048); + + var cmd = SearchCommandBuilder.Create("IDX_NAME", FTCreateParams.CreateParams(), sc).ToString(); + + Assert.Equal( + "FT.CREATE IDX_NAME SCHEMA v1 VECTOR HNSW 8 DIM 4 TYPE FLOAT32 DISTANCE_METRIC L2 COMPRESSION SQ8 " + + "v2 VECTOR HNSW 12 DIM 4 TYPE FLOAT16 DISTANCE_METRIC COSINE M 8 COMPRESSION SQ8 TRAINING_THRESHOLD 2048", + cmd); + } + + // HNSW SQ8 compression (RediSearch MOD-14958): FLOAT32/FLOAT16 with L2/IP/COSINE; the server owns the validation. + [SkipIfRedisTheory(Comparison.LessThan, "8.12.0")] + [MemberData(nameof(EndpointsFixture.Env.AllEnvironments), MemberType = typeof(EndpointsFixture.Env))] + public void TestCreate_Hnsw_Sq8(string endpointId) + { + IDatabase db = GetCleanDatabase(endpointId); + var ft = db.FT(); + var schema = new Schema() + .AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8) + .AddHnswVectorField("v2", Schema.VectorField.VectorType.FLOAT16, 4, + Schema.VectorField.VectorDistanceMetric.CosineDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8, trainingThreshold: 2048) + .AddHnswVectorField("plain", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance); + + Assert.True(ft.Create("idx", new FTCreateParams(), schema)); + + // FT.INFO reports the stored configuration; the server default threshold is 10240, and an + // uncompressed field carries neither key + var attrs = ft.Info("idx").Attributes.ToDictionary(a => a["identifier"].ToString()!); + Assert.Equal("SQ8", attrs["v"]["compression"].ToString()); + Assert.Equal(10240, (long)attrs["v"]["training_threshold"]); + Assert.Equal("SQ8", attrs["v2"]["compression"].ToString()); + Assert.Equal(2048, (long)attrs["v2"]["training_threshold"]); + Assert.False(attrs["plain"].ContainsKey("compression")); + Assert.False(attrs["plain"].ContainsKey("training_threshold")); + + // vectors are searchable while still below the training threshold + byte[] vec = MemoryMarshal.AsBytes(stackalloc float[] { 1, 2, 3, 4 }).ToArray(); + byte[] vec16 = MemoryMarshal.AsBytes(stackalloc short[] { 1, 2, 3, 4 }).ToArray(); + for (int i = 0; i < 3; i++) + { + db.HashSet($"doc:{i}", [new("v", vec), new("v2", vec16), new("plain", vec)]); + } + var res = ft.Search("idx", new Query("*=>[KNN 2 @v $vec]").ReturnFields("__v_score").AddParam("vec", vec)); + Assert.Equal(2, res.TotalResults); + res = ft.Search("idx", new Query("*=>[KNN 2 @v2 $vec]").ReturnFields("__v2_score").AddParam("vec", vec16)); + Assert.Equal(2, res.TotalResults); + } + + [SkipIfRedisTheory(Comparison.LessThan, "8.12.0")] + [MemberData(nameof(EndpointsFixture.Env.StandaloneOnly), MemberType = typeof(EndpointsFixture.Env))] + public void TestCreate_Hnsw_Sq8_ServerRejectsInvalidCombinations(string endpointId) + { + // the client does not validate type/metric/threshold combinations; these are the server's errors, surfaced + IDatabase db = GetCleanDatabase(endpointId); + var ft = db.FT(); + + var float64 = new Schema().AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT64, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8); + var ex = Assert.Throws(() => ft.Create("idx64", new FTCreateParams(), float64)); + Assert.Contains("COMPRESSION is only supported for FLOAT32 and FLOAT16", ex.Message); + + var thresholdOnly = new Schema().AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, trainingThreshold: 100); + ex = Assert.Throws(() => ft.Create("idxThreshold", new FTCreateParams(), thresholdOnly)); + Assert.Contains("TRAINING_THRESHOLD is irrelevant when compression was not requested", ex.Message); + + var svsOnlyValue = new Schema().AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.LVQ8); + Assert.Throws(() => ft.Create("idxLvq", new FTCreateParams(), svsOnlyValue)); + } + [Fact] public void TestIndexingCreation_Default() { From a7dfe50bc9347e5cb7e5248f00ad3ecc22c0eaf0 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 15 Oct 2026 13:49:56 +0100 Subject: [PATCH 2/2] CI: add Redis 8.12 (client-libs-test:8.12-m02-int) to the integration matrix --- .github/workflows/integration.yml | 6 +++--- tests/dockers/.env.v8.12 | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 tests/dockers/.env.v8.12 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7572f458..914ea9c4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -39,10 +39,10 @@ jobs: fail-fast: false matrix: # Full version set runs only on the nightly schedule; PRs and pushes - # test a reduced set (7.2 LTS, 7.4 LTS, 8.2 LTS, 8.8, 8.10 current stable) to keep CI fast. + # test a reduced set (7.2 LTS, 7.4 LTS, 8.2 LTS, 8.8, 8.10 current stable, 8.12 preview) to keep CI fast. redis-version: ${{ fromJSON(github.event_name == 'schedule' - && '["8.10", "8.8", "8.6", "8.4", "8.2", "7.4", "7.2", "6.2"]' - || '["8.10", "8.8", "8.2", "7.4", "7.2"]') }} + && '["8.12", "8.10", "8.8", "8.6", "8.4", "8.2", "7.4", "7.2", "6.2"]' + || '["8.12", "8.10", "8.8", "8.2", "7.4", "7.2"]') }} dotnet-version: ['8.0', '9.0', '10.0'] name: Redis ${{ matrix.redis-version }}; .NET ${{ matrix.dotnet-version }}; steps: diff --git a/tests/dockers/.env.v8.12 b/tests/dockers/.env.v8.12 new file mode 100644 index 00000000..8866d9ec --- /dev/null +++ b/tests/dockers/.env.v8.12 @@ -0,0 +1,6 @@ +# Environment variables for Redis 8.12 +# Used by the run-tests GitHub Action + +# Redis CE image version (Redis 8+ includes modules natively) +# 8.12 is pre-release: this is a milestone build, move to the GA tag when it ships +CLIENT_LIBS_TEST_IMAGE=redislabs/client-libs-test:8.12-m02-int