cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'lighter_buckets_web' into lighter_buckets_server

Browse Source
This commit is contained in:
Min Idzelis
2025-04-29 01:58:00 +00:00
parent bfefa36f04 580a0117c4
commit 077703adcc
328 changed files with 6090 additions and 2169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/repositories/access.repository.ts
+22
View File
@@ -279,6 +279,26 @@ class AuthDeviceAccess {
}
}
class NotificationAccess {
constructor(private db: Kysely<DB>) {}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID_SET] })
@ChunkedSet({ paramIndex: 1 })
async checkOwnerAccess(userId: string, notificationIds: Set<string>) {
if (notificationIds.size === 0) {
return new Set<string>();
}
return this.db
.selectFrom('notifications')
.select('notifications.id')
.where('notifications.id', 'in', [...notificationIds])
.where('notifications.userId', '=', userId)
.execute()
.then((stacks) => new Set(stacks.map((stack) => stack.id)));
}
}
class StackAccess {
constructor(private db: Kysely<DB>) {}
@@ -426,6 +446,7 @@ export class AccessRepository {
asset: AssetAccess;
authDevice: AuthDeviceAccess;
memory: MemoryAccess;
notification: NotificationAccess;
person: PersonAccess;
partner: PartnerAccess;
stack: StackAccess;
@@ -438,6 +459,7 @@ export class AccessRepository {
this.asset = new AssetAccess(db);
this.authDevice = new AuthDeviceAccess(db);
this.memory = new MemoryAccess(db);
this.notification = new NotificationAccess(db);
this.person = new PersonAccess(db);
this.partner = new PartnerAccess(db);
this.stack = new StackAccess(db);
server/src/repositories/asset-job.repository.ts
+91 -3
View File
@@ -2,12 +2,21 @@ import { Injectable } from '@nestjs/common';
import { Kysely } from 'kysely';
import { jsonArrayFrom } from 'kysely/helpers/postgres';
import { InjectKysely } from 'nestjs-kysely';
import { columns } from 'src/database';
import { Asset, columns } from 'src/database';
import { DB } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetFileType } from 'src/enum';
import { AssetFileType, AssetType } from 'src/enum';
import { StorageAsset } from 'src/types';
import { anyUuid, asUuid, withExifInner, withFaces, withFiles } from 'src/utils/database';
import {
anyUuid,
asUuid,
toJson,
withExif,
withExifInner,
withFaces,
withFacesAndPeople,
withFiles,
} from 'src/utils/database';
@Injectable()
export class AssetJobRepository {
@@ -148,6 +157,7 @@ export class AssetJobRepository {
.executeTakeFirst();
}
@GenerateSql({ params: [[DummyValue.UUID]] })
getForSyncAssets(ids: string[]) {
return this.db
.selectFrom('assets')
@@ -163,6 +173,84 @@ export class AssetJobRepository {
.execute();
}
@GenerateSql({ params: [DummyValue.UUID] })
getForAssetDeletion(id: string) {
return this.db
.selectFrom('assets')
.select([
'assets.id',
'assets.isVisible',
'assets.libraryId',
'assets.ownerId',
'assets.livePhotoVideoId',
'assets.sidecarPath',
'assets.encodedVideoPath',
'assets.originalPath',
])
.$call(withExif)
.select(withFacesAndPeople)
.select(withFiles)
.leftJoin('asset_stack', 'asset_stack.id', 'assets.stackId')
.leftJoinLateral(
(eb) =>
eb
.selectFrom('assets as stacked')
.select(['asset_stack.id', 'asset_stack.primaryAssetId'])
.select((eb) => eb.fn<Asset[]>('array_agg', [eb.table('stacked')]).as('assets'))
.where('stacked.deletedAt', 'is not', null)
.where('stacked.isArchived', '=', false)
.whereRef('stacked.stackId', '=', 'asset_stack.id')
.groupBy('asset_stack.id')
.as('stacked_assets'),
(join) => join.on('asset_stack.id', 'is not', null),
)
.select((eb) => toJson(eb, 'stacked_assets').as('stack'))
.where('assets.id', '=', id)
.executeTakeFirst();
}
@GenerateSql({ params: [], stream: true })
streamForVideoConversion(force?: boolean) {
return this.db
.selectFrom('assets')
.select(['assets.id'])
.where('assets.type', '=', AssetType.VIDEO)
.$if(!force, (qb) =>
qb
.where((eb) => eb.or([eb('assets.encodedVideoPath', 'is', null), eb('assets.encodedVideoPath', '=', '')]))
.where('assets.isVisible', '=', true),
)
.where('assets.deletedAt', 'is', null)
.stream();
}
@GenerateSql({ params: [DummyValue.UUID] })
getForVideoConversion(id: string) {
return this.db
.selectFrom('assets')
.select(['assets.id', 'assets.ownerId', 'assets.originalPath', 'assets.encodedVideoPath'])
.where('assets.id', '=', id)
.where('assets.type', '=', AssetType.VIDEO)
.executeTakeFirst();
}
@GenerateSql({ params: [], stream: true })
streamForMetadataExtraction(force?: boolean) {
return this.db
.selectFrom('assets')
.select(['assets.id'])
.$if(!force, (qb) =>
qb
.leftJoin('asset_job_status', 'asset_job_status.assetId', 'assets.id')
.where((eb) =>
eb.or([eb('asset_job_status.metadataExtractedAt', 'is', null), eb('asset_job_status.assetId', 'is', null)]),
)
.where('assets.isVisible', '=', true),
)
.where('assets.deletedAt', 'is', null)
.stream();
}
private storageTemplateAssetQuery() {
return this.db
.selectFrom('assets')
server/src/repositories/config.repository.spec.ts
+9 -97
View File
@@ -80,21 +80,12 @@ describe('getEnv', () => {
const { database } = getEnv();
expect(database).toEqual({
config: {
kysely: expect.objectContaining({
host: 'database',
port: 5432,
database: 'immich',
username: 'postgres',
password: 'postgres',
}),
typeorm: expect.objectContaining({
type: 'postgres',
host: 'database',
port: 5432,
database: 'immich',
username: 'postgres',
password: 'postgres',
}),
connectionType: 'parts',
host: 'database',
port: 5432,
database: 'immich',
username: 'postgres',
password: 'postgres',
},
skipMigrations: false,
vectorExtension: 'vectors',
@@ -110,88 +101,9 @@ describe('getEnv', () => {
it('should use DB_URL', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({
host: 'database1',
password: 'postgres2',
user: 'postgres1',
port: 54_320,
database: 'immich',
});
});
it('should handle sslmode=require', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=require';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: {} });
});
it('should handle sslmode=prefer', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=prefer';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: {} });
});
it('should handle sslmode=verify-ca', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-ca';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: {} });
});
it('should handle sslmode=verify-full', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=verify-full';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: {} });
});
it('should handle sslmode=no-verify', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?sslmode=no-verify';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: { rejectUnauthorized: false } });
});
it('should handle ssl=true', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?ssl=true';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({ ssl: true });
});
it('should reject invalid ssl', () => {
process.env.DB_URL = 'postgres://postgres1:postgres2@database1:54320/immich?ssl=invalid';
expect(() => getEnv()).toThrowError('Invalid ssl option: invalid');
});
it('should handle socket: URLs', () => {
process.env.DB_URL = 'socket:/run/postgresql?db=database1';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({
host: '/run/postgresql',
database: 'database1',
});
});
it('should handle sockets in postgres: URLs', () => {
process.env.DB_URL = 'postgres:///database2?host=/path/to/socket';
const { database } = getEnv();
expect(database.config.kysely).toMatchObject({
host: '/path/to/socket',
database: 'database2',
expect(database.config).toMatchObject({
connectionType: 'url',
url: 'postgres://postgres1:postgres2@database1:54320/immich',
});
});
});
server/src/repositories/config.repository.ts
+13 -48
View File
@@ -7,8 +7,7 @@ import { Request, Response } from 'express';
import { RedisOptions } from 'ioredis';
import { CLS_ID, ClsModuleOptions } from 'nestjs-cls';
import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
import { join, resolve } from 'node:path';
import { parse } from 'pg-connection-string';
import { join } from 'node:path';
import { citiesFile, excludePaths, IWorker } from 'src/constants';
import { Telemetry } from 'src/decorators';
import { EnvDto } from 'src/dtos/env.dto';
@@ -22,9 +21,7 @@ import {
QueueName,
} from 'src/enum';
import { DatabaseConnectionParams, VectorExtension } from 'src/types';
import { isValidSsl, PostgresConnectionConfig } from 'src/utils/database';
import { setDifference } from 'src/utils/set';
import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions.js';
export interface EnvData {
host?: string;
@@ -59,7 +56,7 @@ export interface EnvData {
};
database: {
config: { typeorm: PostgresConnectionOptions & DatabaseConnectionParams; kysely: PostgresConnectionConfig };
config: DatabaseConnectionParams;
skipMigrations: boolean;
vectorExtension: VectorExtension;
};
@@ -152,14 +149,10 @@ const getEnv = (): EnvData => {
const isProd = environment === ImmichEnvironment.PRODUCTION;
const buildFolder = dto.IMMICH_BUILD_DATA || '/build';
const folders = {
// eslint-disable-next-line unicorn/prefer-module
dist: resolve(`${__dirname}/..`),
geodata: join(buildFolder, 'geodata'),
web: join(buildFolder, 'www'),
};
const databaseUrl = dto.DB_URL;
let redisConfig = {
host: dto.REDIS_HOSTNAME || 'redis',
port: dto.REDIS_PORT || 6379,
@@ -191,30 +184,16 @@ const getEnv = (): EnvData => {
}
}
const parts = {
connectionType: 'parts',
host: dto.DB_HOSTNAME || 'database',
port: dto.DB_PORT || 5432,
username: dto.DB_USERNAME || 'postgres',
password: dto.DB_PASSWORD || 'postgres',
database: dto.DB_DATABASE_NAME || 'immich',
} as const;
let parsedOptions: PostgresConnectionConfig = parts;
if (dto.DB_URL) {
const parsed = parse(dto.DB_URL);
if (!isValidSsl(parsed.ssl)) {
throw new Error(`Invalid ssl option: ${parsed.ssl}`);
}
parsedOptions = {
...parsed,
ssl: parsed.ssl,
host: parsed.host ?? undefined,
port: parsed.port ? Number(parsed.port) : undefined,
database: parsed.database ?? undefined,
};
}
const databaseConnection: DatabaseConnectionParams = dto.DB_URL
? { connectionType: 'url', url: dto.DB_URL }
: {
connectionType: 'parts',
host: dto.DB_HOSTNAME || 'database',
port: dto.DB_PORT || 5432,
username: dto.DB_USERNAME || 'postgres',
password: dto.DB_PASSWORD || 'postgres',
database: dto.DB_DATABASE_NAME || 'immich',
};
return {
host: dto.IMMICH_HOST,
@@ -269,21 +248,7 @@ const getEnv = (): EnvData => {
},
database: {
config: {
typeorm: {
type: 'postgres',
entities: [],
migrations: [`${folders.dist}/migrations` + '/*.{js,ts}'],
subscribers: [],
migrationsRun: false,
synchronize: false,
connectTimeoutMS: 10_000, // 10 seconds
parseInt8: true,
...(databaseUrl ? { connectionType: 'url', url: databaseUrl } : parts),
},
kysely: parsedOptions,
},
config: databaseConnection,
skipMigrations: dto.DB_SKIP_MIGRATIONS ?? false,
vectorExtension: dto.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS,
},
server/src/repositories/database.repository.ts
+23 -2
View File
@@ -3,7 +3,7 @@ import AsyncLock from 'async-lock';
import { FileMigrationProvider, Kysely, Migrator, sql, Transaction } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { readdir } from 'node:fs/promises';
import { join } from 'node:path';
import { join, resolve } from 'node:path';
import semver from 'semver';
import { EXTENSION_NAMES, POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';
import { DB } from 'src/db';
@@ -205,8 +205,29 @@ export class DatabaseRepository {
const { rows } = await tableExists.execute(this.db);
const hasTypeOrmMigrations = !!rows[0]?.result;
if (hasTypeOrmMigrations) {
// eslint-disable-next-line unicorn/prefer-module
const dist = resolve(`${__dirname}/..`);
this.logger.debug('Running typeorm migrations');
const dataSource = new DataSource(database.config.typeorm);
const dataSource = new DataSource({
type: 'postgres',
entities: [],
subscribers: [],
migrations: [`${dist}/migrations` + '/*.{js,ts}'],
migrationsRun: false,
synchronize: false,
connectTimeoutMS: 10_000, // 10 seconds
parseInt8: true,
...(database.config.connectionType === 'url'
? { url: database.config.url }
: {
host: database.config.host,
port: database.config.port,
username: database.config.username,
password: database.config.password,
database: database.config.database,
}),
});
await dataSource.initialize();
await dataSource.runMigrations(options);
await dataSource.destroy();
server/src/repositories/event.repository.ts
+3
View File
@@ -14,6 +14,7 @@ import { SystemConfig } from 'src/config';
import { EventConfig } from 'src/decorators';
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { NotificationDto } from 'src/dtos/notification.dto';
import { ReleaseNotification, ServerVersionResponseDto } from 'src/dtos/server.dto';
import { ImmichWorker, MetadataKey, QueueName } from 'src/enum';
import { ConfigRepository } from 'src/repositories/config.repository';
@@ -64,6 +65,7 @@ type EventMap = {
'assets.restore': [{ assetIds: string[]; userId: string }];
'job.start': [QueueName, JobItem];
'job.failed': [{ job: JobItem; error: Error | any }];
// session events
'session.delete': [{ sessionId: string }];
@@ -104,6 +106,7 @@ export interface ClientEventMap {
on_server_version: [ServerVersionResponseDto];
on_config_update: [];
on_new_release: [ReleaseNotification];
on_notification: [NotificationDto];
on_session_delete: [string];
}
server/src/repositories/index.ts
+3 -1
View File
@@ -22,6 +22,7 @@ import { MediaRepository } from 'src/repositories/media.repository';
import { MemoryRepository } from 'src/repositories/memory.repository';
import { MetadataRepository } from 'src/repositories/metadata.repository';
import { MoveRepository } from 'src/repositories/move.repository';
import { NotificationRepository } from 'src/repositories/notification.repository';
import { OAuthRepository } from 'src/repositories/oauth.repository';
import { PartnerRepository } from 'src/repositories/partner.repository';
import { PersonRepository } from 'src/repositories/person.repository';
@@ -55,6 +56,7 @@ export const repositories = [
CryptoRepository,
DatabaseRepository,
DownloadRepository,
EmailRepository,
EventRepository,
JobRepository,
LibraryRepository,
@@ -65,7 +67,7 @@ export const repositories = [
MemoryRepository,
MetadataRepository,
MoveRepository,
EmailRepository,
NotificationRepository,
OAuthRepository,
PartnerRepository,
PersonRepository,
server/src/repositories/media.repository.ts
+40 -23
View File
@@ -7,7 +7,7 @@ import { Writable } from 'node:stream';
import sharp from 'sharp';
import { ORIENTATION_TO_SHARP_ROTATION } from 'src/constants';
import { Exif } from 'src/database';
import { Colorspace, LogLevel } from 'src/enum';
import { Colorspace, LogLevel, RawExtractedFormat } from 'src/enum';
import { LoggingRepository } from 'src/repositories/logging.repository';
import {
DecodeToBufferOptions,
@@ -36,34 +36,51 @@ type ProgressEvent = {
percent?: number;
};
export type ExtractResult = {
buffer: Buffer;
format: RawExtractedFormat;
};
@Injectable()
export class MediaRepository {
constructor(private logger: LoggingRepository) {
this.logger.setContext(MediaRepository.name);
}
async extract(input: string, output: string): Promise<boolean> {
/**
*
* @param input file path to the input image
* @returns ExtractResult if succeeded, or null if failed
*/
async extract(input: string): Promise<ExtractResult | null> {
try {
// remove existing output file if it exists
// as exiftool-vendored does not support overwriting via "-w!" flag
// and throws "1 files could not be read" error when the output file exists
await fs.unlink(output).catch(() => null);
await exiftool.extractBinaryTag('JpgFromRaw2', input, output);
} catch {
try {
this.logger.debug('Extracting JPEG from RAW image:', input);
await exiftool.extractJpgFromRaw(input, output);
} catch (error: any) {
this.logger.debug('Could not extract JPEG from image, trying preview', error.message);
try {
await exiftool.extractPreview(input, output);
} catch (error: any) {
this.logger.debug('Could not extract preview from image', error.message);
return false;
}
}
const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw2', input);
return { buffer, format: RawExtractedFormat.JPEG };
} catch (error: any) {
this.logger.debug('Could not extract JpgFromRaw2 buffer from image, trying JPEG from RAW next', error.message);
}
try {
const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw', input);
return { buffer, format: RawExtractedFormat.JPEG };
} catch (error: any) {
this.logger.debug('Could not extract JPEG buffer from image, trying PreviewJXL next', error.message);
}
try {
const buffer = await exiftool.extractBinaryTagToBuffer('PreviewJXL', input);
return { buffer, format: RawExtractedFormat.JXL };
} catch (error: any) {
this.logger.debug('Could not extract PreviewJXL buffer from image, trying PreviewImage next', error.message);
}
try {
const buffer = await exiftool.extractBinaryTagToBuffer('PreviewImage', input);
return { buffer, format: RawExtractedFormat.JPEG };
} catch (error: any) {
this.logger.debug('Could not extract preview buffer from image', error.message);
return null;
}
return true;
}
async writeExif(tags: Partial<Exif>, output: string): Promise<boolean> {
@@ -104,7 +121,7 @@ export class MediaRepository {
}
}
decodeImage(input: string, options: DecodeToBufferOptions) {
decodeImage(input: string | Buffer, options: DecodeToBufferOptions) {
return this.getImageDecodingPipeline(input, options).raw().toBuffer({ resolveWithObject: true });
}
@@ -235,7 +252,7 @@ export class MediaRepository {
});
}
async getImageDimensions(input: string): Promise<ImageDimensions> {
async getImageDimensions(input: string | Buffer): Promise<ImageDimensions> {
const { width = 0, height = 0 } = await sharp(input).metadata();
return { width, height };
}
server/src/repositories/notification.repository.ts
+103
View File
@@ -0,0 +1,103 @@
import { Insertable, Kysely, Updateable } from 'kysely';
import { DateTime } from 'luxon';
import { InjectKysely } from 'nestjs-kysely';
import { columns } from 'src/database';
import { DB, Notifications } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators';
import { NotificationSearchDto } from 'src/dtos/notification.dto';
export class NotificationRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
@GenerateSql({ params: [DummyValue.UUID] })
cleanup() {
return this.db
.deleteFrom('notifications')
.where((eb) =>
eb.or([
// remove soft-deleted notifications
eb.and([eb('deletedAt', 'is not', null), eb('deletedAt', '<', DateTime.now().minus({ days: 3 }).toJSDate())]),
// remove old, read notifications
eb.and([
// keep recently read messages around for a few days
eb('readAt', '>', DateTime.now().minus({ days: 2 }).toJSDate()),
eb('createdAt', '<', DateTime.now().minus({ days: 15 }).toJSDate()),
]),
eb.and([
// remove super old, unread notifications
eb('readAt', '=', null),
eb('createdAt', '<', DateTime.now().minus({ days: 30 }).toJSDate()),
]),
]),
)
.execute();
}
@GenerateSql({ params: [DummyValue.UUID, {}] }, { name: 'unread', params: [DummyValue.UUID, { unread: true }] })
search(userId: string, dto: NotificationSearchDto) {
return this.db
.selectFrom('notifications')
.select(columns.notification)
.where((qb) =>
qb.and({
userId,
id: dto.id,
level: dto.level,
type: dto.type,
readAt: dto.unread ? null : undefined,
}),
)
.where('deletedAt', 'is', null)
.orderBy('createdAt', 'desc')
.execute();
}
create(notification: Insertable<Notifications>) {
return this.db
.insertInto('notifications')
.values(notification)
.returning(columns.notification)
.executeTakeFirstOrThrow();
}
get(id: string) {
return this.db
.selectFrom('notifications')
.select(columns.notification)
.where('id', '=', id)
.where('deletedAt', 'is not', null)
.executeTakeFirst();
}
update(id: string, notification: Updateable<Notifications>) {
return this.db
.updateTable('notifications')
.set(notification)
.where('deletedAt', 'is', null)
.where('id', '=', id)
.returning(columns.notification)
.executeTakeFirstOrThrow();
}
async updateAll(ids: string[], notification: Updateable<Notifications>) {
await this.db.updateTable('notifications').set(notification).where('id', 'in', ids).execute();
}
async delete(id: string) {
await this.db
.updateTable('notifications')
.set({ deletedAt: DateTime.now().toJSDate() })
.where('id', '=', id)
.execute();
}
async deleteAll(ids: string[]) {
await this.db
.updateTable('notifications')
.set({ deletedAt: DateTime.now().toJSDate() })
.where('id', 'in', ids)
.execute();
}
}