Search: SQ8 compression and training threshold for HNSW vector fields (Redis 8.12)

.github/workflows/integration.yml

@@ -39,10 +39,10 @@ jobs:
         fail-fast: false
         matrix:
           # Full version set runs only on the nightly schedule; PRs and pushes
-          # test a reduced set (7.2 LTS, 7.4 LTS, 8.2 LTS, 8.8, 8.10 current stable) to keep CI fast.
+          # test a reduced set (7.2 LTS, 7.4 LTS, 8.2 LTS, 8.8, 8.10 current stable, 8.12 preview) to keep CI fast.
           redis-version: ${{ fromJSON(github.event_name == 'schedule'
-            && '["8.10", "8.8", "8.6", "8.4", "8.2", "7.4", "7.2", "6.2"]'
-            || '["8.10", "8.8", "8.2", "7.4", "7.2"]') }}
+            && '["8.12", "8.10", "8.8", "8.6", "8.4", "8.2", "7.4", "7.2", "6.2"]'
+            || '["8.12", "8.10", "8.8", "8.2", "7.4", "7.2"]') }}
           dotnet-version: ['8.0', '9.0', '10.0']
       name: Redis ${{ matrix.redis-version }}; .NET ${{ matrix.dotnet-version }};
       steps:

src/NRedisStack/PublicAPI/PublicAPI.Unshipped.txt

@@ -1 +1,8 @@
 #nullable enable
+NRedisStack.Search.Schema.AddHnswVectorField(NRedisStack.Search.FieldName! name, NRedisStack.Search.Schema.VectorField.VectorType type, int dimensions, NRedisStack.Search.Schema.VectorField.VectorDistanceMetric distanceMetric, int maxOutgoingConnections = 16, int maxConnectedNeighbors = 200, int maxTopCandidates = 10, double boundaryFactor = 0.01, System.Collections.Generic.Dictionary<string!, object!>? attributes = null, bool missingIndex = false, NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm compressionAlgorithm = NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm.NotSpecified, int trainingThreshold = 0) -> NRedisStack.Search.Schema!
+NRedisStack.Search.Schema.HnswVectorField.CompressionAlgorithm.get -> NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm
+NRedisStack.Search.Schema.HnswVectorField.CompressionAlgorithm.set -> void
+NRedisStack.Search.Schema.HnswVectorField.TrainingThreshold.get -> int
+NRedisStack.Search.Schema.HnswVectorField.TrainingThreshold.set -> void
+NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm.SQ8 = 7 -> NRedisStack.Search.Schema.VectorField.VectorCompressionAlgorithm
+*REMOVED*NRedisStack.Search.Schema.AddHnswVectorField(NRedisStack.Search.FieldName! name, NRedisStack.Search.Schema.VectorField.VectorType type, int dimensions, NRedisStack.Search.Schema.VectorField.VectorDistanceMetric distanceMetric, int maxOutgoingConnections = 16, int maxConnectedNeighbors = 200, int maxTopCandidates = 10, double boundaryFactor = 0.01, System.Collections.Generic.Dictionary<string!, object!>? attributes = null, bool missingIndex = false) -> NRedisStack.Search.Schema!

src/NRedisStack/Search/Schema.cs

@@ -426,7 +426,9 @@ public enum VectorDistanceMetric
         }
 
         /// <summary>
-        /// The distance metric to use for vector similarity search.
+        /// The vector compression ("COMPRESSION") to apply to stored vectors. Which values an algorithm accepts is
+        /// decided by the server: the LVQ and LeanVec values are for <see cref="VectorAlgo.SVS_VAMANA"/>, and
+        /// <see cref="SQ8"/> is for <see cref="VectorAlgo.HNSW"/>; the client sends what it is given.
         /// </summary>
         public enum VectorCompressionAlgorithm
         {
@@ -444,7 +446,26 @@ public enum VectorCompressionAlgorithm
             LeanVec4x8 = 5,
             // ReSharper disable once InconsistentNaming
             LeanVec8x8 = 6,
+            /// <summary>
+            /// 8-bit scalar quantization of the stored vectors, for <see cref="VectorAlgo.HNSW"/> fields of type
+            /// <see cref="VectorType.FLOAT32"/> or <see cref="VectorType.FLOAT16"/> (Redis 8.12). An optional mean is
+            /// learned from the first vectors indexed; see <see cref="HnswVectorField.TrainingThreshold"/>.
+            /// </summary>
+            SQ8 = 7,
         }
+
+        /// <summary>The "COMPRESSION" token for a <see cref="VectorCompressionAlgorithm"/>.</summary>
+        internal static string CompressionToken(VectorCompressionAlgorithm compression) => compression switch
+        {
+            VectorCompressionAlgorithm.LVQ8 => "LVQ8",
+            VectorCompressionAlgorithm.LVQ4 => "LVQ4",
+            VectorCompressionAlgorithm.LVQ4x4 => "LVQ4x4",
+            VectorCompressionAlgorithm.LVQ4x8 => "LVQ4x8",
+            VectorCompressionAlgorithm.LeanVec4x8 => "LeanVec4x8",
+            VectorCompressionAlgorithm.LeanVec8x8 => "LeanVec8x8",
+            VectorCompressionAlgorithm.SQ8 => "SQ8",
+            _ => compression.ToString(), // fallback
+        };
     }
 
 
@@ -493,6 +514,22 @@ public class HnswVectorField(
         /// </summary>
         public double BoundaryFactor { get; set; } = DEFAULT_EPSILON;
 
+        /// <summary>
+        /// "COMPRESSION"; how stored vectors are compressed. HNSW accepts <see cref="VectorCompressionAlgorithm.SQ8"/>
+        /// (Redis 8.12), for <see cref="VectorType.FLOAT32"/> and <see cref="VectorType.FLOAT16"/> with the L2, IP or
+        /// cosine metric; anything else is rejected by the server. Not sent when <see cref="VectorCompressionAlgorithm.NotSpecified"/>.
+        /// </summary>
+        public VectorCompressionAlgorithm CompressionAlgorithm { get; set; } = VectorCompressionAlgorithm.NotSpecified;
+
+        /// <summary>
+        /// "TRAINING_THRESHOLD"; with <see cref="CompressionAlgorithm"/>, how many vectors are collected before the
+        /// quantization mean is learned and vectors migrate into the compressed graph. Vectors remain searchable
+        /// meanwhile. The server's default is 10240 and its maximum 102400; zero on the server means "do not train".
+        /// Not sent when zero here - set <see cref="VectorField.Attributes"/> to send an explicit zero. Only valid
+        /// together with a compression; the server rejects it otherwise.
+        /// </summary>
+        public int TrainingThreshold { get; set; }
+
         internal const int DEFAULT_M = 16, DEFAULT_EF_CONSTRUCTION = 200, DEFAULT_EF_RUNTIME = 10;
         internal const double DEFAULT_EPSILON = 0.01;
         internal override int DirectAttributeCount
@@ -501,7 +538,9 @@ internal override int DirectAttributeCount
                + (MaxConnectedNeighbors == DEFAULT_EF_CONSTRUCTION ? 0 : 1)
                + (MaxTopCandidates == DEFAULT_EF_RUNTIME ? 0 : 1)
                // ReSharper disable once CompareOfFloatsByEqualityOperator
-               + (BoundaryFactor == DEFAULT_EPSILON ? 0 : 1);
+               + (BoundaryFactor == DEFAULT_EPSILON ? 0 : 1)
+               + (CompressionAlgorithm == VectorCompressionAlgorithm.NotSpecified ? 0 : 1)
+               + (TrainingThreshold == 0 ? 0 : 1);
 
         internal override void AddDirectAttributes(List<object> args)
         {
@@ -527,6 +566,16 @@ internal override void AddDirectAttributes(List<object> args)
                 args.Add("EPSILON");
                 args.Add(BoundaryFactor);
             }
+            if (CompressionAlgorithm != VectorCompressionAlgorithm.NotSpecified)
+            {
+                args.Add("COMPRESSION");
+                args.Add(CompressionToken(CompressionAlgorithm));
+            }
+            if (TrainingThreshold != 0)
+            {
+                args.Add("TRAINING_THRESHOLD");
+                args.Add(TrainingThreshold);
+            }
         }
     }
 
@@ -560,16 +609,7 @@ internal override void AddDirectAttributes(List<object> args)
             if (CompressionAlgorithm != VectorCompressionAlgorithm.NotSpecified)
             {
                 args.Add("COMPRESSION");
-                args.Add(CompressionAlgorithm switch
-                {
-                    VectorCompressionAlgorithm.LVQ8 => "LVQ8",
-                    VectorCompressionAlgorithm.LVQ4 => "LVQ4",
-                    VectorCompressionAlgorithm.LVQ4x4 => "LVQ4x4",
-                    VectorCompressionAlgorithm.LVQ4x8 => "LVQ4x8",
-                    VectorCompressionAlgorithm.LeanVec4x8 => "LeanVec4x8",
-                    VectorCompressionAlgorithm.LeanVec8x8 => "LeanVec8x8",
-                    _ => CompressionAlgorithm.ToString(), // fallback
-                });
+                args.Add(CompressionToken(CompressionAlgorithm));
             }
             base.AddDirectAttributes(args);
             if (ConstructionWindowSize != DEFAULT_CONSTRUCTION_WINDOW_SIZE)
@@ -874,14 +914,18 @@ public Schema AddHnswVectorField(FieldName name, VectorType type, int dimensions
         int maxConnectedNeighbors = HnswVectorField.DEFAULT_EF_CONSTRUCTION,
         int maxTopCandidates = HnswVectorField.DEFAULT_EF_RUNTIME,
         double boundaryFactor = HnswVectorField.DEFAULT_EPSILON,
-        Dictionary<string, object>? attributes = null, bool missingIndex = false)
+        Dictionary<string, object>? attributes = null, bool missingIndex = false,
+        VectorCompressionAlgorithm compressionAlgorithm = VectorCompressionAlgorithm.NotSpecified,
+        int trainingThreshold = 0)
     {
         Fields.Add(new HnswVectorField(name, type, dimensions, distanceMetric, attributes, missingIndex)
         {
             MaxOutgoingConnections = maxOutgoingConnections,
             MaxConnectedNeighbors = maxConnectedNeighbors,
             MaxTopCandidates = maxTopCandidates,
             BoundaryFactor = boundaryFactor,
+            CompressionAlgorithm = compressionAlgorithm,
+            TrainingThreshold = trainingThreshold,
         });
         return this;
     }

tests/NRedisStack.Tests/Search/IndexCreationTests.cs

@@ -379,6 +379,94 @@ public void TestCreate_Float16_Int32_VectorField_Svs_WithCompression(string endp
         Assert.Equal(2, res.TotalResults);
     }
 
+    [Fact]
+    public void TestIndexingCreation_Hnsw_Sq8()
+    {
+        // COMPRESSION and TRAINING_THRESHOLD are emitted only when set, after the graph parameters
+        Schema sc = new Schema()
+            .AddHnswVectorField("v1", Schema.VectorField.VectorType.FLOAT32, 4,
+                Schema.VectorField.VectorDistanceMetric.EuclideanDistance,
+                compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8)
+            .AddHnswVectorField("v2", Schema.VectorField.VectorType.FLOAT16, 4,
+                Schema.VectorField.VectorDistanceMetric.CosineDistance,
+                maxOutgoingConnections: 8,
+                compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8, trainingThreshold: 2048);
+
+        var cmd = SearchCommandBuilder.Create("IDX_NAME", FTCreateParams.CreateParams(), sc).ToString();
+
+        Assert.Equal(
+            "FT.CREATE IDX_NAME SCHEMA v1 VECTOR HNSW 8 DIM 4 TYPE FLOAT32 DISTANCE_METRIC L2 COMPRESSION SQ8 " +
+            "v2 VECTOR HNSW 12 DIM 4 TYPE FLOAT16 DISTANCE_METRIC COSINE M 8 COMPRESSION SQ8 TRAINING_THRESHOLD 2048",
+            cmd);
+    }
+
+    // HNSW SQ8 compression (RediSearch MOD-14958): FLOAT32/FLOAT16 with L2/IP/COSINE; the server owns the validation.
+    [SkipIfRedisTheory(Comparison.LessThan, "8.12.0")]
+    [MemberData(nameof(EndpointsFixture.Env.AllEnvironments), MemberType = typeof(EndpointsFixture.Env))]
+    public void TestCreate_Hnsw_Sq8(string endpointId)
+    {
+        IDatabase db = GetCleanDatabase(endpointId);
+        var ft = db.FT();
+        var schema = new Schema()
+            .AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT32, 4,
+                Schema.VectorField.VectorDistanceMetric.EuclideanDistance,
+                compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8)
+            .AddHnswVectorField("v2", Schema.VectorField.VectorType.FLOAT16, 4,
+                Schema.VectorField.VectorDistanceMetric.CosineDistance,
+                compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8, trainingThreshold: 2048)
+            .AddHnswVectorField("plain", Schema.VectorField.VectorType.FLOAT32, 4,
+                Schema.VectorField.VectorDistanceMetric.EuclideanDistance);
+
+        Assert.True(ft.Create("idx", new FTCreateParams(), schema));
+
+        // FT.INFO reports the stored configuration; the server default threshold is 10240, and an
+        // uncompressed field carries neither key
+        var attrs = ft.Info("idx").Attributes.ToDictionary(a => a["identifier"].ToString()!);
+        Assert.Equal("SQ8", attrs["v"]["compression"].ToString());
+        Assert.Equal(10240, (long)attrs["v"]["training_threshold"]);
+        Assert.Equal("SQ8", attrs["v2"]["compression"].ToString());
+        Assert.Equal(2048, (long)attrs["v2"]["training_threshold"]);
+        Assert.False(attrs["plain"].ContainsKey("compression"));
+        Assert.False(attrs["plain"].ContainsKey("training_threshold"));
+
+        // vectors are searchable while still below the training threshold
+        byte[] vec = MemoryMarshal.AsBytes(stackalloc float[] { 1, 2, 3, 4 }).ToArray();
+        byte[] vec16 = MemoryMarshal.AsBytes(stackalloc short[] { 1, 2, 3, 4 }).ToArray();
+        for (int i = 0; i < 3; i++)
+        {
+            db.HashSet($"doc:{i}", [new("v", vec), new("v2", vec16), new("plain", vec)]);
+        }
+        var res = ft.Search("idx", new Query("*=>[KNN 2 @v $vec]").ReturnFields("__v_score").AddParam("vec", vec));
+        Assert.Equal(2, res.TotalResults);
+        res = ft.Search("idx", new Query("*=>[KNN 2 @v2 $vec]").ReturnFields("__v2_score").AddParam("vec", vec16));
+        Assert.Equal(2, res.TotalResults);
+    }
+
+    [SkipIfRedisTheory(Comparison.LessThan, "8.12.0")]
+    [MemberData(nameof(EndpointsFixture.Env.StandaloneOnly), MemberType = typeof(EndpointsFixture.Env))]
+    public void TestCreate_Hnsw_Sq8_ServerRejectsInvalidCombinations(string endpointId)
+    {
+        // the client does not validate type/metric/threshold combinations; these are the server's errors, surfaced
+        IDatabase db = GetCleanDatabase(endpointId);
+        var ft = db.FT();
+
+        var float64 = new Schema().AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT64, 4,
+            Schema.VectorField.VectorDistanceMetric.EuclideanDistance,
+            compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.SQ8);
+        var ex = Assert.Throws<RedisServerException>(() => ft.Create("idx64", new FTCreateParams(), float64));
+        Assert.Contains("COMPRESSION is only supported for FLOAT32 and FLOAT16", ex.Message);
+
+        var thresholdOnly = new Schema().AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT32, 4,
+            Schema.VectorField.VectorDistanceMetric.EuclideanDistance, trainingThreshold: 100);
+        ex = Assert.Throws<RedisServerException>(() => ft.Create("idxThreshold", new FTCreateParams(), thresholdOnly));
+        Assert.Contains("TRAINING_THRESHOLD is irrelevant when compression was not requested", ex.Message);
+
+        var svsOnlyValue = new Schema().AddHnswVectorField("v", Schema.VectorField.VectorType.FLOAT32, 4,
+            Schema.VectorField.VectorDistanceMetric.EuclideanDistance,
+            compressionAlgorithm: Schema.VectorField.VectorCompressionAlgorithm.LVQ8);
+        Assert.Throws<RedisServerException>(() => ft.Create("idxLvq", new FTCreateParams(), svsOnlyValue));
+    }
+
     [Fact]
     public void TestIndexingCreation_Default()
     {

tests/dockers/.env.v8.12

@@ -0,0 +1,6 @@
+# Environment variables for Redis 8.12
+# Used by the run-tests GitHub Action
+
+# Redis CE image version (Redis 8+ includes modules natively)
+# 8.12 is pre-release: this is a milestone build, move to the GA tag when it ships
+CLIENT_LIBS_TEST_IMAGE=redislabs/client-libs-test:8.12-m02-int