Merge remote-tracking branch 'origin/lighter_buckets_web' into lighter_buckets_server

server/src/utils/database.ts

@@ -18,10 +18,10 @@ import { parse } from 'pg-connection-string';
 import postgres, { Notice } from 'postgres';
 import { columns, Exif, Person } from 'src/database';
 import { DB } from 'src/db';
-import { AssetFileType } from 'src/enum';
+import { AssetFileType, DatabaseExtension } from 'src/enum';
 import { TimeBucketSize } from 'src/repositories/asset.repository';
 import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
-import { DatabaseConnectionParams } from 'src/types';
+import { DatabaseConnectionParams, VectorExtension } from 'src/types';
 
 type Ssl = 'require' | 'allow' | 'prefer' | 'verify-full' | boolean | object;
 
@@ -379,3 +379,28 @@ export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuild
     .$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople))
     .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null));
 }
+
+type VectorIndexOptions = { vectorExtension: VectorExtension; table: string; indexName: string };
+
+export function vectorIndexQuery({ vectorExtension, table, indexName }: VectorIndexOptions): string {
+  switch (vectorExtension) {
+    case DatabaseExtension.VECTORS: {
+      return `
+        CREATE INDEX IF NOT EXISTS ${indexName} ON ${table}
+        USING vectors (embedding vector_cos_ops) WITH (options = $$
+        [indexing.hnsw]
+        m = 16
+        ef_construction = 300
+        $$)`;
+    }
+    case DatabaseExtension.VECTOR: {
+      return `
+        CREATE INDEX IF NOT EXISTS ${indexName} ON ${table}
+        USING hnsw (embedding vector_cosine_ops)
+        WITH (ef_construction = 300, m = 16)`;
+    }
+    default: {
+      throw new Error(`Unsupported vector extension: '${vectorExtension}'`);
+    }
+  }
+}

server/src/utils/pagination.ts

@@ -8,22 +8,6 @@ export interface PaginationResult<T> {
   hasNextPage: boolean;
 }
 
-export type Paginated<T> = Promise<PaginationResult<T>>;
-
-/** @deprecated use `this.db. ... .stream()` instead */
-export async function* usePagination<T>(
-  pageSize: number,
-  getNextPage: (pagination: PaginationOptions) => PaginationResult<T> | Paginated<T>,
-) {
-  let hasNextPage = true;
-
-  for (let skip = 0; hasNextPage; skip += pageSize) {
-    const result = await getNextPage({ take: pageSize, skip });
-    hasNextPage = result.hasNextPage;
-    yield result.items;
-  }
-}
-
 export function paginationHelper<Entity extends object>(items: Entity[], take: number): PaginationResult<Entity> {
   const hasNextPage = items.length > take;
   items.splice(take);