diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7572f458..914ea9c4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -39,10 +39,10 @@ jobs: fail-fast: false matrix: # Full version set runs only on the nightly schedule; PRs and pushes - # test a reduced set (7.2 LTS, 7.4 LTS, 8.2 LTS, 8.8, 8.10 current stable) to keep CI fast. + # test a reduced set (7.2 LTS, 7.4 LTS, 8.2 LTS, 8.8, 8.10 current stable, 8.12 preview) to keep CI fast. redis-version: ${{ fromJSON(github.event_name == 'schedule' - && '["8.10", "8.8", "8.6", "8.4", "8.2", "7.4", "7.2", "6.2"]' - || '["8.10", "8.8", "8.2", "7.4", "7.2"]') }} + && '["8.12", "8.10", "8.8", "8.6", "8.4", "8.2", "7.4", "7.2", "6.2"]' + || '["8.12", "8.10", "8.8", "8.2", "7.4", "7.2"]') }} dotnet-version: ['8.0', '9.0', '10.0'] name: Redis ${{ matrix.redis-version }}; .NET ${{ matrix.dotnet-version }}; steps: diff --git a/src/NRedisStack/PublicAPI/PublicAPI.Unshipped.txt b/src/NRedisStack/PublicAPI/PublicAPI.Unshipped.txt index 7dc5c581..097d8891 100644 --- a/src/NRedisStack/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/NRedisStack/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,8 @@ #nullable enable +NRedisStack.Search.Schema.AddHnswVectorField(NRedisStack.Search.FieldName! name, NRedisStack.Search.Schema.VectorField.VectorType type, int dimensions, NRedisStack.Search.Schema.VectorField.VectorDistanceMetric distanceMetric, int maxOutgoingConnections = 16, int maxConnectedNeighbors = 200, int maxTopCandidates = 10, double boundaryFactor = 0.01, System.Collections.Generic.Dictionary? attributes = null, bool missingIndex = false, NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm compressionAlgorithm = NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm.NotSpecified, int trainingThreshold = 0) -> NRedisStack.Search.Schema! +NRedisStack.Search.Schema.HnswVectorField.CompressionAlgorithm.get -> NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm +NRedisStack.Search.Schema.HnswVectorField.CompressionAlgorithm.set -> void +NRedisStack.Search.Schema.HnswVectorField.TrainingThreshold.get -> int +NRedisStack.Search.Schema.HnswVectorField.TrainingThreshold.set -> void +NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm.SQ8 = 7 -> NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm +*REMOVED*NRedisStack.Search.Schema.AddHnswVectorField(NRedisStack.Search.FieldName! name, NRedisStack.Search.Schema.VectorField.VectorType type, int dimensions, NRedisStack.Search.Schema.VectorField.VectorDistanceMetric distanceMetric, int maxOutgoingConnections = 16, int maxConnectedNeighbors = 200, int maxTopCandidates = 10, double boundaryFactor = 0.01, System.Collections.Generic.Dictionary? attributes = null, bool missingIndex = false) -> NRedisStack.Search.Schema! diff --git a/src/NRedisStack/Search/Schema.cs b/src/NRedisStack/Search/Schema.cs index 716d8b9b..8f3757c4 100644 --- a/src/NRedisStack/Search/Schema.cs +++ b/src/NRedisStack/Search/Schema.cs @@ -426,7 +426,9 @@ public enum VectorDistanceMetric } /// - /// The distance metric to use for vector similarity search. + /// The vector compression ("COMPRESSION") to apply to stored vectors. Which values an algorithm accepts is + /// decided by the server: the LVQ and LeanVec values are for , and + /// is for ; the client sends what it is given. /// public enum VectorCompressionAlgorithm { @@ -444,7 +446,26 @@ public enum VectorCompressionAlgorithm LeanVec4x8 = 5, // ReSharper disable once InconsistentNaming LeanVec8x8 = 6, + /// + /// 8-bit scalar quantization of the stored vectors, for fields of type + /// or (Redis 8.12). An optional mean is + /// learned from the first vectors indexed; see . + /// + SQ8 = 7, } + + /// The "COMPRESSION" token for a . + internal static string CompressionToken(VectorCompressionAlgorithm compression) => compression switch + { + VectorCompressionAlgorithm.LVQ8 => "LVQ8", + VectorCompressionAlgorithm.LVQ4 => "LVQ4", + VectorCompressionAlgorithm.LVQ4x4 => "LVQ4x4", + VectorCompressionAlgorithm.LVQ4x8 => "LVQ4x8", + VectorCompressionAlgorithm.LeanVec4x8 => "LeanVec4x8", + VectorCompressionAlgorithm.LeanVec8x8 => "LeanVec8x8", + VectorCompressionAlgorithm.SQ8 => "SQ8", + _ => compression.ToString(), // fallback + }; } @@ -493,6 +514,22 @@ public class HnswVectorField( /// public double BoundaryFactor { get; set; } = DEFAULT_EPSILON; + /// + /// "COMPRESSION"; how stored vectors are compressed. HNSW accepts + /// (Redis 8.12), for and with the L2, IP or + /// cosine metric; anything else is rejected by the server. Not sent when . + /// + public VectorCompressionAlgorithm CompressionAlgorithm { get; set; } = VectorCompressionAlgorithm.NotSpecified; + + /// + /// "TRAINING_THRESHOLD"; with , how many vectors are collected before the + /// quantization mean is learned and vectors migrate into the compressed graph. Vectors remain searchable + /// meanwhile. The server's default is 10240 and its maximum 102400; zero on the server means "do not train". + /// Not sent when zero here - set to send an explicit zero. Only valid + /// together with a compression; the server rejects it otherwise. + /// + public int TrainingThreshold { get; set; } + internal const int DEFAULT_M = 16, DEFAULT_EF_CONSTRUCTION = 200, DEFAULT_EF_RUNTIME = 10; internal const double DEFAULT_EPSILON = 0.01; internal override int DirectAttributeCount @@ -501,7 +538,9 @@ internal override int DirectAttributeCount + (MaxConnectedNeighbors == DEFAULT_EF_CONSTRUCTION ? 0 : 1) + (MaxTopCandidates == DEFAULT_EF_RUNTIME ? 0 : 1) // ReSharper disable once CompareOfFloatsByEqualityOperator - + (BoundaryFactor == DEFAULT_EPSILON ? 0 : 1); + + (BoundaryFactor == DEFAULT_EPSILON ? 0 : 1) + + (CompressionAlgorithm == VectorCompressionAlgorithm.NotSpecified ? 0 : 1) + + (TrainingThreshold == 0 ? 0 : 1); internal override void AddDirectAttributes(List args) { @@ -527,6 +566,16 @@ internal override void AddDirectAttributes(List args) args.Add("EPSILON"); args.Add(BoundaryFactor); } + if (CompressionAlgorithm != VectorCompressionAlgorithm.NotSpecified) + { + args.Add("COMPRESSION"); + args.Add(CompressionToken(CompressionAlgorithm)); + } + if (TrainingThreshold != 0) + { + args.Add("TRAINING_THRESHOLD"); + args.Add(TrainingThreshold); + } } } @@ -560,16 +609,7 @@ internal override void AddDirectAttributes(List args) if (CompressionAlgorithm != VectorCompressionAlgorithm.NotSpecified) { args.Add("COMPRESSION"); - args.Add(CompressionAlgorithm switch - { - VectorCompressionAlgorithm.LVQ8 => "LVQ8", - VectorCompressionAlgorithm.LVQ4 => "LVQ4", - VectorCompressionAlgorithm.LVQ4x4 => "LVQ4x4", - VectorCompressionAlgorithm.LVQ4x8 => "LVQ4x8", - VectorCompressionAlgorithm.LeanVec4x8 => "LeanVec4x8", - VectorCompressionAlgorithm.LeanVec8x8 => "LeanVec8x8", - _ => CompressionAlgorithm.ToString(), // fallback - }); + args.Add(CompressionToken(CompressionAlgorithm)); } base.AddDirectAttributes(args); if (ConstructionWindowSize != DEFAULT_CONSTRUCTION_WINDOW_SIZE) @@ -874,7 +914,9 @@ public Schema AddHnswVectorField(FieldName name, VectorType type, int dimensions int maxConnectedNeighbors = HnswVectorField.DEFAULT_EF_CONSTRUCTION, int maxTopCandidates = HnswVectorField.DEFAULT_EF_RUNTIME, double boundaryFactor = HnswVectorField.DEFAULT_EPSILON, - Dictionary? attributes = null, bool missingIndex = false) + Dictionary? attributes = null, bool missingIndex = false, + VectorCompressionAlgorithm compressionAlgorithm = VectorCompressionAlgorithm.NotSpecified, + int trainingThreshold = 0) { Fields.Add(new HnswVectorField(name, type, dimensions, distanceMetric, attributes, missingIndex) { @@ -882,6 +924,8 @@ public Schema AddHnswVectorField(FieldName name, VectorType type, int dimensions MaxConnectedNeighbors = maxConnectedNeighbors, MaxTopCandidates = maxTopCandidates, BoundaryFactor = boundaryFactor, + CompressionAlgorithm = compressionAlgorithm, + TrainingThreshold = trainingThreshold, }); return this; } diff --git a/tests/NRedisStack.Tests/Search/IndexCreationTests.cs b/tests/NRedisStack.Tests/Search/IndexCreationTests.cs index 22bc2f0f..55ac24d6 100644 --- a/tests/NRedisStack.Tests/Search/IndexCreationTests.cs +++ b/tests/NRedisStack.Tests/Search/IndexCreationTests.cs @@ -379,6 +379,94 @@ public void TestCreate_Float16_Int32_VectorField_Svs_WithCompression(string endp Assert.Equal(2, res.TotalResults); } + [Fact] + public void TestIndexingCreation_Hnsw_Sq8() + { + // COMPRESSION and TRAINING_THRESHOLD are emitted only when set, after the graph parameters + Schema sc = new Schema() + .AddHnswVectorField("v1", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8) + .AddHnswVectorField("v2", Schema.VectorField.VectorType.FLOAT16, 4, + Schema.VectorField.VectorDistanceMetric.CosineDistance, + maxOutgoingConnections: 8, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8, trainingThreshold: 2048); + + var cmd = SearchCommandBuilder.Create("IDX_NAME", FTCreateParams.CreateParams(), sc).ToString(); + + Assert.Equal( + "FT.CREATE IDX_NAME SCHEMA v1 VECTOR HNSW 8 DIM 4 TYPE FLOAT32 DISTANCE_METRIC L2 COMPRESSION SQ8 " + + "v2 VECTOR HNSW 12 DIM 4 TYPE FLOAT16 DISTANCE_METRIC COSINE M 8 COMPRESSION SQ8 TRAINING_THRESHOLD 2048", + cmd); + } + + // HNSW SQ8 compression (RediSearch MOD-14958): FLOAT32/FLOAT16 with L2/IP/COSINE; the server owns the validation. + [SkipIfRedisTheory(Comparison.LessThan, "8.12.0")] + [MemberData(nameof(EndpointsFixture.Env.AllEnvironments), MemberType = typeof(EndpointsFixture.Env))] + public void TestCreate_Hnsw_Sq8(string endpointId) + { + IDatabase db = GetCleanDatabase(endpointId); + var ft = db.FT(); + var schema = new Schema() + .AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8) + .AddHnswVectorField("v2", Schema.VectorField.VectorType.FLOAT16, 4, + Schema.VectorField.VectorDistanceMetric.CosineDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8, trainingThreshold: 2048) + .AddHnswVectorField("plain", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance); + + Assert.True(ft.Create("idx", new FTCreateParams(), schema)); + + // FT.INFO reports the stored configuration; the server default threshold is 10240, and an + // uncompressed field carries neither key + var attrs = ft.Info("idx").Attributes.ToDictionary(a => a["identifier"].ToString()!); + Assert.Equal("SQ8", attrs["v"]["compression"].ToString()); + Assert.Equal(10240, (long)attrs["v"]["training_threshold"]); + Assert.Equal("SQ8", attrs["v2"]["compression"].ToString()); + Assert.Equal(2048, (long)attrs["v2"]["training_threshold"]); + Assert.False(attrs["plain"].ContainsKey("compression")); + Assert.False(attrs["plain"].ContainsKey("training_threshold")); + + // vectors are searchable while still below the training threshold + byte[] vec = MemoryMarshal.AsBytes(stackalloc float[] { 1, 2, 3, 4 }).ToArray(); + byte[] vec16 = MemoryMarshal.AsBytes(stackalloc short[] { 1, 2, 3, 4 }).ToArray(); + for (int i = 0; i < 3; i++) + { + db.HashSet($"doc:{i}", [new("v", vec), new("v2", vec16), new("plain", vec)]); + } + var res = ft.Search("idx", new Query("*=>[KNN 2 @v $vec]").ReturnFields("__v_score").AddParam("vec", vec)); + Assert.Equal(2, res.TotalResults); + res = ft.Search("idx", new Query("*=>[KNN 2 @v2 $vec]").ReturnFields("__v2_score").AddParam("vec", vec16)); + Assert.Equal(2, res.TotalResults); + } + + [SkipIfRedisTheory(Comparison.LessThan, "8.12.0")] + [MemberData(nameof(EndpointsFixture.Env.StandaloneOnly), MemberType = typeof(EndpointsFixture.Env))] + public void TestCreate_Hnsw_Sq8_ServerRejectsInvalidCombinations(string endpointId) + { + // the client does not validate type/metric/threshold combinations; these are the server's errors, surfaced + IDatabase db = GetCleanDatabase(endpointId); + var ft = db.FT(); + + var float64 = new Schema().AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT64, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8); + var ex = Assert.Throws(() => ft.Create("idx64", new FTCreateParams(), float64)); + Assert.Contains("COMPRESSION is only supported for FLOAT32 and FLOAT16", ex.Message); + + var thresholdOnly = new Schema().AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, trainingThreshold: 100); + ex = Assert.Throws(() => ft.Create("idxThreshold", new FTCreateParams(), thresholdOnly)); + Assert.Contains("TRAINING_THRESHOLD is irrelevant when compression was not requested", ex.Message); + + var svsOnlyValue = new Schema().AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT32, 4, + Schema.VectorField.VectorDistanceMetric.EuclideanDistance, + compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.LVQ8); + Assert.Throws(() => ft.Create("idxLvq", new FTCreateParams(), svsOnlyValue)); + } + [Fact] public void TestIndexingCreation_Default() { diff --git a/tests/dockers/.env.v8.12 b/tests/dockers/.env.v8.12 new file mode 100644 index 00000000..8866d9ec --- /dev/null +++ b/tests/dockers/.env.v8.12 @@ -0,0 +1,6 @@ +# Environment variables for Redis 8.12 +# Used by the run-tests GitHub Action + +# Redis CE image version (Redis 8+ includes modules natively) +# 8.12 is pre-release: this is a milestone build, move to the GA tag when it ships +CLIENT_LIBS_TEST_IMAGE=redislabs/client-libs-test:8.12-m02-int