feat(server)!: pgvecto.rs 0.2 and pgvector compatibility (#6785)

* basic changes

update version check

set ef_search for clip

* pgvector compatibility

Revert "pgvector compatibility"

This reverts commit 2b66a52aa4097dd27da58138c5288fd87cb9b24a.

pgvector compatibility: minimal edition

pgvector startup check

* update extension at startup

* wording

shortened vector extension variable name

* nightly docker

* fixed version checks

* update tests

add tests for updating extension

remove unnecessary check

* simplify `getRuntimeConfig`

* wording

* reindex on minor version update

* 0.2 upgrade testing

update prod compose

* acquire lock for init

* wip vector down on shutdown

* use upgrade helper

* update image tag

* refine restart check

check error message

* test reindex

testing

upstream fix

formatting

fixed reindexing

* use enum in signature

* fix tests

remove unused code

* add reindexing tests

* update to official 0.2

remove alpha from version name

* add warning test if restart required

* update test image to 0.2.0

* linting and test cleanup

* formatting

* update sql

* wording

* handle setting search path for new and existing databases

* handle new db in reindex check

* fix post-update reindexing

* get dim size

* formatting

* use vbase

* handle different db name

* update sql

* linting

* fix suggested env

server/src/domain/repositories/database.repository.ts

@@ -3,21 +3,47 @@ import { Version } from '../domain.constant';
 export enum DatabaseExtension {
   CUBE = 'cube',
   EARTH_DISTANCE = 'earthdistance',
+  VECTOR = 'vector',
   VECTORS = 'vectors',
 }
 
+export type VectorExtension = DatabaseExtension.VECTOR | DatabaseExtension.VECTORS;
+
+export enum VectorIndex {
+  CLIP = 'clip_index',
+  FACE = 'face_index',
+}
+
 export enum DatabaseLock {
   GeodataImport = 100,
+  Migrations = 200,
   StorageTemplateMigration = 420,
   CLIPDimSize = 512,
 }
 
+export const extName: Record<DatabaseExtension, string> = {
+  cube: 'cube',
+  earthdistance: 'earthdistance',
+  vector: 'pgvector',
+  vectors: 'pgvecto.rs',
+} as const;
+
+export interface VectorUpdateResult {
+  restartRequired: boolean;
+}
+
 export const IDatabaseRepository = 'IDatabaseRepository';
 
 export interface IDatabaseRepository {
   getExtensionVersion(extensionName: string): Promise<Version | null>;
+  getAvailableExtensionVersion(extension: DatabaseExtension): Promise<Version | null>;
+  getPreferredVectorExtension(): VectorExtension;
   getPostgresVersion(): Promise<Version>;
   createExtension(extension: DatabaseExtension): Promise<void>;
+  updateExtension(extension: DatabaseExtension, version?: Version): Promise<void>;
+  updateVectorExtension(extension: VectorExtension, version?: Version): Promise<VectorUpdateResult>;
+  reindex(index: VectorIndex): Promise<void>;
+  shouldReindex(name: VectorIndex): Promise<boolean>;
   runMigrations(options?: { transaction?: 'all' | 'none' | 'each' }): Promise<void>;
   withLock<R>(lock: DatabaseLock, callback: () => Promise<R>): Promise<R>;
   isBusy(lock: DatabaseLock): boolean;

server/src/domain/repositories/smart-info.repository.ts

@@ -7,7 +7,7 @@ export type Embedding = number[];
 export interface EmbeddingSearch {
   userIds: string[];
   embedding: Embedding;
-  numResults?: number;
+  numResults: number;
   withArchived?: boolean;
 }