Add compressed vector storage (Int8 SQ, RaBitQ, TurboQUANT)

src/Build5Nines.SharpVector/BasicDiskMemoryVectorDatabaseBase.cs

@@ -3,6 +3,7 @@
 using Build5Nines.SharpVector.Vocabulary;
 using Build5Nines.SharpVector.Vectorization;
 using Build5Nines.SharpVector.VectorCompare;
+using Build5Nines.SharpVector.VectorEncoding;
 using Build5Nines.SharpVector.VectorStore;
 
 namespace Build5Nines.SharpVector;
@@ -26,4 +27,8 @@ public abstract class BasicDiskMemoryVectorDatabaseBase<TId, TMetadata, TVectorS
     protected BasicDiskMemoryVectorDatabaseBase(TVectorStore vectorStore)
         : base(vectorStore)
     { }
+
+    protected BasicDiskMemoryVectorDatabaseBase(TVectorStore vectorStore, IVectorEncoding encoding)
+        : base(vectorStore, encoding)
+    { }
 }

src/Build5Nines.SharpVector/BasicDiskVectorDatabase.cs

@@ -3,6 +3,7 @@
 using Build5Nines.SharpVector.Preprocessing;
 using Build5Nines.SharpVector.Vectorization;
 using Build5Nines.SharpVector.VectorCompare;
+using Build5Nines.SharpVector.VectorEncoding;
 using Build5Nines.SharpVector.VectorStore;
 
 namespace Build5Nines.SharpVector;
@@ -33,6 +34,20 @@ public BasicDiskVectorDatabase(string rootPath)
         )
     { }
 
+    /// <summary>
+    /// Create a disk-backed database that compresses vectors with the supplied
+    /// encoding before storing them.
+    /// </summary>
+    public BasicDiskVectorDatabase(string rootPath, IVectorEncoding encoding)
+        : base(
+            new BasicDiskVectorStore<int, TMetadata, BasicDiskVocabularyStore<string>, string, int>(
+                rootPath,
+                new BasicDiskVocabularyStore<string>(rootPath)
+            ),
+            encoding
+        )
+    { }
+
     [Obsolete("Use DeserializeFromBinaryStreamAsync instead.")]
     public override async Task DeserializeFromJsonStreamAsync(Stream stream)
     {

src/Build5Nines.SharpVector/BasicMemoryVectorDatabase.cs

@@ -1,7 +1,21 @@
+using Build5Nines.SharpVector.VectorEncoding;
+
 namespace Build5Nines.SharpVector;
 
 /// <summary>
 /// A basic implementation of an vector database that uses an in-memory dictionary to store vectors, with integer keys and string metadata values.
 /// </summary>
 public class BasicMemoryVectorDatabase : MemoryVectorDatabase<string>
-{ }
+{
+    public BasicMemoryVectorDatabase()
+        : base()
+    { }
+
+    /// <summary>
+    /// Create a database that compresses vectors with the supplied encoding
+    /// before storing them.
+    /// </summary>
+    public BasicMemoryVectorDatabase(IVectorEncoding encoding)
+        : base(encoding)
+    { }
+}

src/Build5Nines.SharpVector/DatabaseInfo.cs

@@ -1,3 +1,5 @@
+using System.Text.Json.Serialization;
+
 namespace Build5Nines.SharpVector;
 
 public class DatabaseInfo
@@ -22,4 +24,12 @@ public DatabaseInfo(string? schema, string? version, string? classType)
     public string? Schema { get; set; }
     public string? Version { get; set; }
     public string? ClassType { get; set; }
-}
+
+    /// <summary>
+    /// The id of the vector encoding used for this database, when not the
+    /// default raw float32 encoding. Omitted from the JSON when null so
+    /// raw-encoded databases continue to produce the legacy on-disk shape.
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? VectorEncodingId { get; set; }
+}

src/Build5Nines.SharpVector/MemoryVectorDatabase.cs

@@ -3,6 +3,7 @@
 using Build5Nines.SharpVector.Preprocessing;
 using Build5Nines.SharpVector.Vectorization;
 using Build5Nines.SharpVector.VectorCompare;
+using Build5Nines.SharpVector.VectorEncoding;
 using Build5Nines.SharpVector.VectorStore;
 
 namespace Build5Nines.SharpVector;
@@ -37,6 +38,19 @@ public MemoryVectorDatabase()
         )
     { }
 
+    /// <summary>
+    /// Create an in-memory vector database that compresses vectors with the
+    /// supplied encoding before storing them.
+    /// </summary>
+    public MemoryVectorDatabase(IVectorEncoding encoding)
+        : base(
+            new MemoryDictionaryVectorStoreWithVocabulary<int, TMetadata, DictionaryVocabularyStore<string>, string, int>(
+                new DictionaryVocabularyStore<string>()
+            ),
+            encoding
+        )
+    { }
+
 
     [Obsolete("Use DeserializeFromBinaryStreamAsync instead.")]
     public override async Task DeserializeFromJsonStreamAsync(Stream stream)

src/Build5Nines.SharpVector/MemoryVectorDatabaseBase.cs

@@ -3,6 +3,7 @@
 using Build5Nines.SharpVector.Vocabulary;
 using Build5Nines.SharpVector.Vectorization;
 using Build5Nines.SharpVector.VectorCompare;
+using Build5Nines.SharpVector.VectorEncoding;
 using Build5Nines.SharpVector.VectorStore;
 using System.Collections.Concurrent;
 using System.IO.Compression;
@@ -43,6 +44,10 @@ public abstract class MemoryVectorDatabaseBase<TId, TMetadata, TVectorStore, TVo
     protected MemoryVectorDatabaseBase(TVectorStore vectorStore)
         : base(vectorStore)
     { }
+
+    protected MemoryVectorDatabaseBase(TVectorStore vectorStore, IVectorEncoding encoding)
+        : base(vectorStore, encoding)
+    { }
 }
 
 /// <summary>
@@ -63,4 +68,8 @@ public abstract class MemoryVectorDatabaseBase<TId, TMetadata, TVectorStore, TId
     public MemoryVectorDatabaseBase(IEmbeddingsGenerator embeddingsGenerator, TVectorStore vectorStore)
         : base(embeddingsGenerator, vectorStore)
     { }
+
+    public MemoryVectorDatabaseBase(IEmbeddingsGenerator embeddingsGenerator, TVectorStore vectorStore, IVectorEncoding encoding)
+        : base(embeddingsGenerator, vectorStore, encoding)
+    { }
 }

src/Build5Nines.SharpVector/VectorCompare/CosineSimilarityVectorComparerAsync.cs

@@ -5,6 +5,8 @@ namespace Build5Nines.SharpVector.VectorCompare;
 
 public class CosineSimilarityVectorComparer : IVectorComparer
 {
+    public VectorComparison MetricKind => VectorComparison.CosineSimilarity;
+
     /// <summary>
     /// Calculates the cosine similarity between two vectors.
     /// 

src/Build5Nines.SharpVector/VectorCompare/EuclideanDistanceVectorComparerAsync.cs

@@ -4,6 +4,8 @@ namespace Build5Nines.SharpVector.VectorCompare;
 
 public class EuclideanDistanceVectorComparer : IVectorComparer
 {
+    public VectorComparison MetricKind => VectorComparison.EuclideanDistance;
+
     /// <summary>
     /// Calculates the Euclidean distance between two vectors.
     /// </summary>

src/Build5Nines.SharpVector/VectorCompare/IVectorComparer.cs

@@ -2,6 +2,13 @@ namespace Build5Nines.SharpVector.VectorCompare;
 
 public interface IVectorComparer
 {
+    /// <summary>
+    /// The kind of metric this comparer implements. The database uses this to
+    /// dispatch to the matching fast-path in an <see cref="VectorEncoding.IVectorEncoding"/>
+    /// when stored vectors are compressed.
+    /// </summary>
+    VectorComparison MetricKind { get; }
+
     /// <summary>
     /// Calculates a comparison between two vectors
     /// </summary>

src/Build5Nines.SharpVector/VectorComparison.cs

@@ -1,3 +1,15 @@
 namespace Build5Nines.SharpVector;
 
-//public record VectorComparison<TId>(TId Id, float vectorComparison);
+/// <summary>
+/// The kind of similarity/distance metric a <see cref="VectorCompare.IVectorComparer"/>
+/// represents. Used by the encoding subsystem to dispatch to the correct
+/// fast-path implementation for an encoded vector.
+/// </summary>
+public enum VectorComparison
+{
+    /// <summary>Cosine similarity: higher is more similar, range [-1, 1].</summary>
+    CosineSimilarity,
+
+    /// <summary>Euclidean distance: lower is more similar, range [0, infinity).</summary>
+    EuclideanDistance
+}