Files
immich/server/src/domain/domain.util.ts
Michael Manganiello e262298090 fix(server): Split database queries based on PostgreSQL bound params limit (#6034)
* fix(server): Split database queries based on PostgreSQL bound params limit

PostgreSQL uses a 16-bit integer to indicate the number of bound
parameters.

This means that the maximum number of parameters for any query is 65535.
Any query that tries to bind more than that (e.g. searching by a list of
IDs) requires splitting the query into multiple chunks.

This change includes refactoring every Repository that runs queries
using a list of ids, and either flattening or merging results.

Fixes #5788, #5997.

Also, potentially a fix for #4648 (at least based on
[this comment](https://github.com/immich-app/immich/issues/4648#issuecomment-1826134027)).

References:

* https://github.com/typeorm/typeorm/issues/7565
* [PostgreSQL message format - Bind](https://www.postgresql.org/docs/15/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BIND)

* misc: Create Chunked decorator to simplify implementation

* feat: Add ChunkedArray/ChunkedSet decorators
2024-01-06 20:36:12 -05:00

241 lines
5.9 KiB
TypeScript

import { applyDecorators } from '@nestjs/common';
import { ApiProperty } from '@nestjs/swagger';
import { Transform, Type } from 'class-transformer';
import {
IsArray,
IsBoolean,
IsDate,
IsNotEmpty,
IsOptional,
IsString,
IsUUID,
ValidateIf,
ValidationOptions,
} from 'class-validator';
import { CronJob } from 'cron';
import _ from 'lodash';
import { basename, extname } from 'node:path';
import sanitize from 'sanitize-filename';
export enum CacheControl {
PRIVATE_WITH_CACHE = 'private_with_cache',
PRIVATE_WITHOUT_CACHE = 'private_without_cache',
NONE = 'none',
}
export class ImmichFileResponse {
public readonly path!: string;
public readonly contentType!: string;
public readonly cacheControl!: CacheControl;
constructor(response: ImmichFileResponse) {
Object.assign(this, response);
}
}
export interface OpenGraphTags {
title: string;
description: string;
imageUrl?: string;
}
export type Options = {
optional?: boolean;
each?: boolean;
};
export const isConnectionAborted = (error: Error | any) => error.code === 'ECONNABORTED';
export function ValidateUUID({ optional, each }: Options = { optional: false, each: false }) {
return applyDecorators(
IsUUID('4', { each }),
ApiProperty({ format: 'uuid' }),
optional ? Optional() : IsNotEmpty(),
each ? IsArray() : IsString(),
);
}
export function validateCronExpression(expression: string) {
try {
new CronJob(expression, () => {});
} catch (error) {
return false;
}
return true;
}
interface IValue {
value?: string;
}
export const QueryBoolean = ({ optional }: { optional?: boolean }) => {
const decorators = [IsBoolean(), Transform(toBoolean)];
if (optional) {
decorators.push(Optional());
}
return applyDecorators(...decorators);
};
export const QueryDate = ({ optional }: { optional?: boolean }) => {
const decorators = [IsDate(), Type(() => Date)];
if (optional) {
decorators.push(Optional());
}
return applyDecorators(...decorators);
};
export const toBoolean = ({ value }: IValue) => {
if (value == 'true') {
return true;
} else if (value == 'false') {
return false;
}
return value;
};
export const toEmail = ({ value }: IValue) => value?.toLowerCase();
export const toSanitized = ({ value }: IValue) => sanitize((value || '').replace(/\./g, ''));
export function getFileNameWithoutExtension(path: string): string {
return basename(path, extname(path));
}
export function getLivePhotoMotionFilename(stillName: string, motionName: string) {
return getFileNameWithoutExtension(stillName) + extname(motionName);
}
const KiB = Math.pow(1024, 1);
const MiB = Math.pow(1024, 2);
const GiB = Math.pow(1024, 3);
const TiB = Math.pow(1024, 4);
const PiB = Math.pow(1024, 5);
export const HumanReadableSize = { KiB, MiB, GiB, TiB, PiB };
export function asHumanReadable(bytes: number, precision = 1): string {
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB'];
let magnitude = 0;
let remainder = bytes;
while (remainder >= 1024) {
if (magnitude + 1 < units.length) {
magnitude++;
remainder /= 1024;
} else {
break;
}
}
return `${remainder.toFixed(magnitude == 0 ? 0 : precision)} ${units[magnitude]}`;
}
export interface PaginationOptions {
take: number;
skip?: number;
}
export interface PaginationResult<T> {
items: T[];
hasNextPage: boolean;
}
export type Paginated<T> = Promise<PaginationResult<T>>;
export async function* usePagination<T>(
pageSize: number,
getNextPage: (pagination: PaginationOptions) => Paginated<T>,
) {
let hasNextPage = true;
for (let skip = 0; hasNextPage; skip += pageSize) {
const result = await getNextPage({ take: pageSize, skip });
hasNextPage = result.hasNextPage;
yield result.items;
}
}
export interface OptionalOptions extends ValidationOptions {
nullable?: boolean;
}
/**
* Checks if value is missing and if so, ignores all validators.
*
* @param validationOptions {@link OptionalOptions}
*
* @see IsOptional exported from `class-validator.
*/
// https://stackoverflow.com/a/71353929
export function Optional({ nullable, ...validationOptions }: OptionalOptions = {}) {
if (nullable === true) {
return IsOptional(validationOptions);
}
return ValidateIf((obj: any, v: any) => v !== undefined, validationOptions);
}
/**
* Chunks an array or set into smaller arrays of the specified size.
*
* @param collection The collection to chunk.
* @param size The size of each chunk.
*/
export function chunks<T>(collection: Array<T> | Set<T>, size: number): T[][] {
if (collection instanceof Set) {
const result = [];
let chunk = [];
for (const elem of collection) {
chunk.push(elem);
if (chunk.length === size) {
result.push(chunk);
chunk = [];
}
}
if (chunk.length > 0) {
result.push(chunk);
}
return result;
} else {
return _.chunk(collection, size);
}
}
// NOTE: The following Set utils have been added here, to easily determine where they are used.
// They should be replaced with native Set operations, when they are added to the language.
// Proposal reference: https://github.com/tc39/proposal-set-methods
export const setUnion = <T>(...sets: Set<T>[]): Set<T> => {
const union = new Set(sets[0]);
for (const set of sets.slice(1)) {
for (const elem of set) {
union.add(elem);
}
}
return union;
};
export const setDifference = <T>(setA: Set<T>, ...sets: Set<T>[]): Set<T> => {
const difference = new Set(setA);
for (const set of sets) {
for (const elem of set) {
difference.delete(elem);
}
}
return difference;
};
export const setIsSuperset = <T>(set: Set<T>, subset: Set<T>): boolean => {
for (const elem of subset) {
if (!set.has(elem)) {
return false;
}
}
return true;
};
export const setIsEqual = <T>(setA: Set<T>, setB: Set<T>): boolean => {
return setA.size === setB.size && setIsSuperset(setA, setB);
};