fix(server): Split database queries based on PostgreSQL bound params limit (#6034)
* fix(server): Split database queries based on PostgreSQL bound params limit PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the maximum number of parameters for any query is 65535. Any query that tries to bind more than that (e.g. searching by a list of IDs) requires splitting the query into multiple chunks. This change includes refactoring every Repository that runs queries using a list of ids, and either flattening or merging results. Fixes #5788, #5997. Also, potentially a fix for #4648 (at least based on [this comment](https://github.com/immich-app/immich/issues/4648#issuecomment-1826134027)). References: * https://github.com/typeorm/typeorm/issues/7565 * [PostgreSQL message format - Bind](https://www.postgresql.org/docs/15/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BIND) * misc: Create Chunked decorator to simplify implementation * feat: Add ChunkedArray/ChunkedSet decorators
This commit is contained in:
committed by
GitHub
parent
6835d4519a
commit
e262298090
@@ -27,7 +27,7 @@ import { DateTime } from 'luxon';
|
||||
import { And, FindOptionsRelations, FindOptionsWhere, In, IsNull, LessThan, Not, Repository } from 'typeorm';
|
||||
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
|
||||
import { DummyValue, GenerateSql } from '../infra.util';
|
||||
import { OptionalBetween, paginate } from '../infra.utils';
|
||||
import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils';
|
||||
|
||||
const DEFAULT_SEARCH_SIZE = 250;
|
||||
|
||||
@@ -248,6 +248,7 @@ export class AssetRepository implements IAssetRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID]] })
|
||||
@ChunkedArray()
|
||||
getByIds(ids: string[], relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity[]> {
|
||||
if (!relations) {
|
||||
relations = {
|
||||
@@ -301,6 +302,7 @@ export class AssetRepository implements IAssetRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID]] })
|
||||
@ChunkedArray()
|
||||
getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]> {
|
||||
return this.repository.find({
|
||||
where: { library: { id: In(libraryIds) } },
|
||||
@@ -380,14 +382,17 @@ export class AssetRepository implements IAssetRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] })
|
||||
@Chunked()
|
||||
async updateAll(ids: string[], options: Partial<AssetEntity>): Promise<void> {
|
||||
await this.repository.update({ id: In(ids) }, options);
|
||||
}
|
||||
|
||||
@Chunked()
|
||||
async softDeleteAll(ids: string[]): Promise<void> {
|
||||
await this.repository.softDelete({ id: In(ids), isExternal: false });
|
||||
}
|
||||
|
||||
@Chunked()
|
||||
async restoreAll(ids: string[]): Promise<void> {
|
||||
await this.repository.restore({ id: In(ids) });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user