chore(server,cli,web): housekeeping and stricter code style (#6751)

* add unicorn to eslint

* fix lint errors for cli

* fix merge

* fix album name extraction

* Update cli/src/commands/upload.command.ts

Co-authored-by: Ben McCann <322311+benmccann@users.noreply.github.com>

* es2k23

* use lowercase os

* return undefined album name

* fix bug in asset response dto

* auto fix issues

* fix server code style

* es2022 and formatting

* fix compilation error

* fix test

* fix config load

* fix last lint errors

* set string type

* bump ts

* start work on web

* web formatting

* Fix UUIDParamDto as UUIDParamDto

* fix library service lint

* fix web errors

* fix errors

* formatting

* wip

* lints fixed

* web can now start

* alphabetical package json

* rename error

* chore: clean up

---------

Co-authored-by: Ben McCann <322311+benmccann@users.noreply.github.com>
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
Jonathan Jogenfors
2024-02-02 04:18:00 +01:00
committed by GitHub
parent e4d0560d49
commit f44fa45aa0
218 changed files with 2471 additions and 1244 deletions

View File

@@ -6,12 +6,13 @@ const urlOrParts = url
? { url }
: {
host: process.env.DB_HOSTNAME || 'localhost',
port: parseInt(process.env.DB_PORT || '5432'),
port: Number.parseInt(process.env.DB_PORT || '5432'),
username: process.env.DB_USERNAME || 'postgres',
password: process.env.DB_PASSWORD || 'postgres',
database: process.env.DB_DATABASE_NAME || 'immich',
};
/* eslint unicorn/prefer-module: "off" -- We can fix this when migrating to ESM*/
export const databaseConfig: PostgresConnectionOptions = {
type: 'postgres',
entities: [__dirname + '/entities/*.entity.{js,ts}'],
@@ -19,7 +20,7 @@ export const databaseConfig: PostgresConnectionOptions = {
migrations: [__dirname + '/migrations/*.{js,ts}'],
subscribers: [__dirname + '/subscribers/*.{js,ts}'],
migrationsRun: false,
connectTimeoutMS: 10000, // 10 seconds
connectTimeoutMS: 10_000, // 10 seconds
parseInt8: true,
...urlOrParts,
};

View File

@@ -15,8 +15,8 @@ function parseRedisConfig(): RedisOptions {
}
return {
host: process.env.REDIS_HOSTNAME || 'immich_redis',
port: parseInt(process.env.REDIS_PORT || '6379'),
db: parseInt(process.env.REDIS_DBINDEX || '0'),
port: Number.parseInt(process.env.REDIS_PORT || '6379'),
db: Number.parseInt(process.env.REDIS_DBINDEX || '0'),
username: process.env.REDIS_USERNAME || undefined,
password: process.env.REDIS_PASSWORD || undefined,
path: process.env.REDIS_SOCKET || undefined,

View File

@@ -27,4 +27,4 @@ export const DummyValue = {
// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching
// by a list of IDs) requires splitting the query into multiple chunks.
// We are rounding down this limit, as queries commonly include other filters and parameters.
export const DATABASE_PARAMETER_CHUNK_SIZE = 65500;
export const DATABASE_PARAMETER_CHUNK_SIZE = 65_500;

View File

@@ -59,21 +59,25 @@ export const isValidInteger = (value: number, options: { min?: number; max?: num
export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator {
return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
const originalMethod = descriptor.value;
const paramIndex = options.paramIndex ?? 0;
descriptor.value = async function (...args: any[]) {
const arg = args[paramIndex];
const parameterIndex = options.paramIndex ?? 0;
descriptor.value = async function (...arguments_: any[]) {
const argument = arguments_[parameterIndex];
// Early return if argument length is less than or equal to the chunk size.
if (
(arg instanceof Array && arg.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
(arg instanceof Set && arg.size <= DATABASE_PARAMETER_CHUNK_SIZE)
(Array.isArray(argument) && argument.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
(argument instanceof Set && argument.size <= DATABASE_PARAMETER_CHUNK_SIZE)
) {
return await originalMethod.apply(this, args);
return await originalMethod.apply(this, arguments_);
}
return Promise.all(
chunks(arg, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
await originalMethod.apply(this, [...args.slice(0, paramIndex), chunk, ...args.slice(paramIndex + 1)]);
chunks(argument, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
await Reflect.apply(originalMethod, this, [
...arguments_.slice(0, parameterIndex),
chunk,
...arguments_.slice(parameterIndex + 1),
]);
}),
).then((results) => (options.mergeFn ? options.mergeFn(results) : results));
};

View File

@@ -24,7 +24,8 @@ export class AddLibraries1688392120838 implements MigrationInterface {
);
// Create default library for each user and assign all assets to it
const userIds: string[] = (await queryRunner.query(`SELECT id FROM "users"`)).map((user: any) => user.id);
const users = await queryRunner.query(`SELECT id FROM "users"`);
const userIds: string[] = users.map((user: any) => user.id);
for (const userId of userIds) {
await queryRunner.query(

View File

@@ -14,7 +14,7 @@ export class UsePgVectors1700713871511 implements MigrationInterface {
const clipModelNameQuery = await queryRunner.query(`SELECT value FROM system_config WHERE key = 'machineLearning.clip.modelName'`);
const clipModelName: string = clipModelNameQuery?.[0]?.['value'] ?? 'ViT-B-32__openai';
const clipDimSize = getCLIPModelInfo(clipModelName.replace(/"/g, '')).dimSize;
const clipDimSize = getCLIPModelInfo(clipModelName.replaceAll('"', '')).dimSize;
await queryRunner.query(`
ALTER TABLE asset_faces

View File

@@ -167,7 +167,7 @@ class AlbumAccess implements IAlbumAccess {
})
.then(
(sharedLinks) =>
new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))),
new Set(sharedLinks.flatMap((sharedLink) => (sharedLink.albumId ? [sharedLink.albumId] : []))),
),
),
).then((results) => setUnion(...results));

View File

@@ -71,7 +71,7 @@ export class AlbumRepository implements IAlbumRepository {
@ChunkedArray()
async getMetadataForIds(ids: string[]): Promise<AlbumAssetCount[]> {
// Guard against running invalid query when ids list is empty.
if (!ids.length) {
if (ids.length === 0) {
return [];
}

View File

@@ -24,7 +24,7 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import _ from 'lodash';
import { DateTime } from 'luxon';
import path from 'path';
import path from 'node:path';
import {
And,
Brackets,
@@ -471,7 +471,7 @@ export class AssetRepository implements IAssetRepository {
let where: FindOptionsWhere<AssetEntity> | FindOptionsWhere<AssetEntity>[] = {};
switch (property) {
case WithoutProperty.THUMBNAIL:
case WithoutProperty.THUMBNAIL: {
where = [
{ resizePath: IsNull(), isVisible: true },
{ resizePath: '', isVisible: true },
@@ -480,15 +480,17 @@ export class AssetRepository implements IAssetRepository {
{ thumbhash: IsNull(), isVisible: true },
];
break;
}
case WithoutProperty.ENCODED_VIDEO:
case WithoutProperty.ENCODED_VIDEO: {
where = [
{ type: AssetType.VIDEO, encodedVideoPath: IsNull() },
{ type: AssetType.VIDEO, encodedVideoPath: '' },
];
break;
}
case WithoutProperty.EXIF:
case WithoutProperty.EXIF: {
relations = {
exifInfo: true,
jobStatus: true,
@@ -500,8 +502,9 @@ export class AssetRepository implements IAssetRepository {
},
};
break;
}
case WithoutProperty.SMART_SEARCH:
case WithoutProperty.SMART_SEARCH: {
relations = {
smartSearch: true,
};
@@ -513,8 +516,9 @@ export class AssetRepository implements IAssetRepository {
},
};
break;
}
case WithoutProperty.OBJECT_TAGS:
case WithoutProperty.OBJECT_TAGS: {
relations = {
smartInfo: true,
};
@@ -526,8 +530,9 @@ export class AssetRepository implements IAssetRepository {
},
};
break;
}
case WithoutProperty.FACES:
case WithoutProperty.FACES: {
relations = {
faces: true,
jobStatus: true,
@@ -544,8 +549,9 @@ export class AssetRepository implements IAssetRepository {
},
};
break;
}
case WithoutProperty.PERSON:
case WithoutProperty.PERSON: {
relations = {
faces: true,
};
@@ -558,16 +564,19 @@ export class AssetRepository implements IAssetRepository {
},
};
break;
}
case WithoutProperty.SIDECAR:
case WithoutProperty.SIDECAR: {
where = [
{ sidecarPath: IsNull(), isVisible: true },
{ sidecarPath: '', isVisible: true },
];
break;
}
default:
default: {
throw new Error(`Invalid getWithout property: ${property}`);
}
}
return paginate(this.repository, pagination, {
@@ -584,18 +593,21 @@ export class AssetRepository implements IAssetRepository {
let where: FindOptionsWhere<AssetEntity> | FindOptionsWhere<AssetEntity>[] = {};
switch (property) {
case WithProperty.SIDECAR:
case WithProperty.SIDECAR: {
where = [{ sidecarPath: Not(IsNull()), isVisible: true }];
break;
case WithProperty.IS_OFFLINE:
}
case WithProperty.IS_OFFLINE: {
if (!libraryId) {
throw new Error('Library id is required when finding offline assets');
}
where = [{ isOffline: true, libraryId: libraryId }];
break;
}
default:
default: {
throw new Error(`Invalid getWith property: ${property}`);
}
}
return paginate(this.repository, pagination, {

View File

@@ -51,13 +51,15 @@ export class CommunicationRepository
on(event: 'connect' | ServerEvent, callback: OnConnectCallback | OnServerEventCallback) {
switch (event) {
case 'connect':
case 'connect': {
this.onConnectCallbacks.push(callback);
break;
}
default:
default: {
this.onServerEventCallbacks[event].push(callback as OnServerEventCallback);
break;
}
}
}

View File

@@ -1,8 +1,8 @@
import { ICryptoRepository } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { compareSync, hash } from 'bcrypt';
import { createHash, randomBytes, randomUUID } from 'crypto';
import { createReadStream } from 'fs';
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createReadStream } from 'node:fs';
@Injectable()
export class CryptoRepository implements ICryptoRepository {
@@ -24,7 +24,7 @@ export class CryptoRepository implements ICryptoRepository {
return new Promise<Buffer>((resolve, reject) => {
const hash = createHash('sha1');
const stream = createReadStream(filepath);
stream.on('error', (err) => reject(err));
stream.on('error', (error) => reject(error));
stream.on('data', (chunk) => hash.update(chunk));
stream.on('end', () => resolve(hash.digest()));
});

View File

@@ -10,10 +10,10 @@ import {
import { ImmichLogger } from '@app/infra/logger';
import archiver from 'archiver';
import chokidar, { WatchOptions } from 'chokidar';
import { constants, createReadStream, existsSync, mkdirSync } from 'fs';
import fs, { copyFile, readdir, rename, writeFile } from 'fs/promises';
import { glob } from 'glob';
import path from 'path';
import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs';
import fs, { copyFile, readdir, rename, writeFile } from 'node:fs/promises';
import path from 'node:path';
export class FilesystemProvider implements IStorageRepository {
private logger = new ImmichLogger(FilesystemProvider.name);
@@ -60,7 +60,7 @@ export class FilesystemProvider implements IStorageRepository {
try {
await fs.access(filepath, mode);
return true;
} catch (_) {
} catch {
return false;
}
}
@@ -68,11 +68,11 @@ export class FilesystemProvider implements IStorageRepository {
async unlink(file: string) {
try {
await fs.unlink(file);
} catch (err) {
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') {
} catch (error) {
if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') {
this.logger.warn(`File ${file} does not exist.`);
} else {
throw err;
throw error;
}
}
}

View File

@@ -15,7 +15,7 @@ import { ModuleRef } from '@nestjs/core';
import { SchedulerRegistry } from '@nestjs/schedule';
import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullmq';
import { CronJob, CronTime } from 'cron';
import { setTimeout } from 'timers/promises';
import { setTimeout } from 'node:timers/promises';
import { bullConfig } from '../infra.config';
@Injectable()
@@ -24,7 +24,7 @@ export class JobRepository implements IJobRepository {
private logger = new ImmichLogger(JobRepository.name);
constructor(
private moduleRef: ModuleRef,
private moduleReference: ModuleRef,
private schedulerReqistry: SchedulerRegistry,
) {}
@@ -118,7 +118,7 @@ export class JobRepository implements IJobRepository {
}
async queueAll(items: JobItem[]): Promise<void> {
if (!items.length) {
if (items.length === 0) {
return;
}
@@ -167,19 +167,23 @@ export class JobRepository implements IJobRepository {
private getJobOptions(item: JobItem): JobsOptions | null {
switch (item.name) {
case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE:
case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: {
return { jobId: item.data.id };
case JobName.GENERATE_PERSON_THUMBNAIL:
}
case JobName.GENERATE_PERSON_THUMBNAIL: {
return { priority: 1 };
case JobName.QUEUE_FACIAL_RECOGNITION:
}
case JobName.QUEUE_FACIAL_RECOGNITION: {
return { jobId: JobName.QUEUE_FACIAL_RECOGNITION };
}
default:
default: {
return null;
}
}
}
private getQueue(queue: QueueName): Queue {
return this.moduleRef.get<Queue>(getQueueToken(queue), { strict: false });
return this.moduleReference.get<Queue>(getQueueToken(queue), { strict: false });
}
}

View File

@@ -10,7 +10,7 @@ import {
VisionModelInput,
} from '@app/domain';
import { Injectable } from '@nestjs/common';
import { readFile } from 'fs/promises';
import { readFile } from 'node:fs/promises';
const errorPrefix = 'Machine learning request';

View File

@@ -2,10 +2,10 @@ import { CropOptions, IMediaRepository, ResizeOptions, TranscodeOptions, VideoIn
import { Colorspace } from '@app/infra/entities';
import { ImmichLogger } from '@app/infra/logger';
import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
import fs from 'fs/promises';
import fs from 'node:fs/promises';
import { Writable } from 'node:stream';
import { promisify } from 'node:util';
import sharp from 'sharp';
import { Writable } from 'stream';
import { promisify } from 'util';
const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
sharp.concurrency(0);
@@ -91,7 +91,7 @@ export class MediaRepository implements IMediaRepository {
}
if (typeof output !== 'string') {
throw new Error('Two-pass transcoding does not support writing to a stream');
throw new TypeError('Two-pass transcoding does not support writing to a stream');
}
// two-pass allows for precise control of bitrate at the cost of running twice
@@ -124,12 +124,12 @@ export class MediaRepository implements IMediaRepository {
.inputOptions(options.inputOptions)
.outputOptions(options.outputOptions)
.output(output)
.on('error', (err, stdout, stderr) => this.logger.error(stderr || err));
.on('error', (error, stdout, stderr) => this.logger.error(stderr || error));
}
chainPath(existing: string, path: string) {
const sep = existing.endsWith(':') ? '' : ':';
return `${existing}${sep}${path}`;
const separator = existing.endsWith(':') ? '' : ':';
return `${existing}${separator}${path}`;
}
async generateThumbhash(imagePath: string): Promise<Buffer> {

View File

@@ -15,11 +15,11 @@ import { ImmichLogger } from '@app/infra/logger';
import { Inject } from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { DefaultReadTaskOptions, exiftool, Tags } from 'exiftool-vendored';
import { createReadStream, existsSync } from 'fs';
import { readFile } from 'fs/promises';
import * as geotz from 'geo-tz';
import { getName } from 'i18n-iso-countries';
import * as readLine from 'readline';
import { createReadStream, existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import * as readLine from 'node:readline';
import { DataSource, DeepPartial, QueryRunner, Repository } from 'typeorm';
type GeoEntity = GeodataPlacesEntity | GeodataAdmin1Entity | GeodataAdmin2Entity;
@@ -69,10 +69,10 @@ export class MetadataRepository implements IMetadataRepository {
await this.loadAdmin2(queryRunner);
await queryRunner.commitTransaction();
} catch (e) {
this.logger.fatal('Error importing geodata', e);
} catch (error) {
this.logger.fatal('Error importing geodata', error);
await queryRunner.rollbackTransaction();
throw e;
throw error;
} finally {
await queryRunner.release();
}
@@ -110,10 +110,10 @@ export class MetadataRepository implements IMetadataRepository {
queryRunner,
(lineSplit: string[]) =>
this.geodataPlacesRepository.create({
id: parseInt(lineSplit[0]),
id: Number.parseInt(lineSplit[0]),
name: lineSplit[1],
latitude: parseFloat(lineSplit[4]),
longitude: parseFloat(lineSplit[5]),
latitude: Number.parseFloat(lineSplit[4]),
longitude: Number.parseFloat(lineSplit[5]),
countryCode: lineSplit[8],
admin1Code: lineSplit[10],
admin2Code: lineSplit[11],
@@ -192,7 +192,8 @@ export class MetadataRepository implements IMetadataRepository {
backfillTimezones: true,
inferTimezoneFromDatestamps: true,
useMWG: true,
numericTags: DefaultReadTaskOptions.numericTags.concat(['FocalLength']),
numericTags: [...DefaultReadTaskOptions.numericTags, 'FocalLength'],
/* eslint unicorn/no-array-callback-reference: off, unicorn/no-array-method-this-argument: off */
geoTz: (lat, lon) => geotz.find(lat, lon)[0],
})
.catch((error) => {

View File

@@ -28,12 +28,7 @@ export class PersonRepository implements IPersonRepository {
.createQueryBuilder()
.update()
.set({ personId: newPersonId })
.where(
_.omitBy(
{ personId: oldPersonId ? oldPersonId : undefined, id: faceIds ? In(faceIds) : undefined },
_.isUndefined,
),
)
.where(_.omitBy({ personId: oldPersonId ?? undefined, id: faceIds ? In(faceIds) : undefined }, _.isUndefined))
.execute();
return result.affected ?? 0;

View File

@@ -31,11 +31,11 @@ export class SmartInfoRepository implements ISmartInfoRepository {
throw new Error(`Invalid CLIP model name: ${modelName}`);
}
const curDimSize = await this.getDimSize();
this.logger.verbose(`Current database CLIP dimension size is ${curDimSize}`);
const currentDimSize = await this.getDimSize();
this.logger.verbose(`Current database CLIP dimension size is ${currentDimSize}`);
if (dimSize != curDimSize) {
this.logger.log(`Dimension size of model ${modelName} is ${dimSize}, but database expects ${curDimSize}.`);
if (dimSize != currentDimSize) {
this.logger.log(`Dimension size of model ${modelName} is ${dimSize}, but database expects ${currentDimSize}.`);
await this.updateDimSize(dimSize);
}
}
@@ -119,7 +119,9 @@ export class SmartInfoRepository implements ISmartInfoRepository {
cte = cte.andWhere('faces."personId" IS NOT NULL');
}
this.faceColumns.forEach((col) => cte.addSelect(`faces.${col}`, col));
for (const col of this.faceColumns) {
cte.addSelect(`faces.${col}`, col);
}
results = await manager
.createQueryBuilder()
@@ -157,8 +159,8 @@ export class SmartInfoRepository implements ISmartInfoRepository {
throw new Error(`Invalid CLIP dimension size: ${dimSize}`);
}
const curDimSize = await this.getDimSize();
if (curDimSize === dimSize) {
const currentDimSize = await this.getDimSize();
if (currentDimSize === dimSize) {
return;
}
@@ -181,7 +183,7 @@ export class SmartInfoRepository implements ISmartInfoRepository {
$$)`);
});
this.logger.log(`Successfully updated database CLIP dimension size from ${curDimSize} to ${dimSize}.`);
this.logger.log(`Successfully updated database CLIP dimension size from ${currentDimSize} to ${dimSize}.`);
}
private async getDimSize(): Promise<number> {

View File

@@ -1,7 +1,7 @@
import { ISystemConfigRepository } from '@app/domain';
import { InjectRepository } from '@nestjs/typeorm';
import axios from 'axios';
import { readFile } from 'fs/promises';
import { readFile } from 'node:fs/promises';
import { In, Repository } from 'typeorm';
import { SystemConfigEntity } from '../entities';
import { DummyValue, GenerateSql } from '../infra.util';
@@ -22,7 +22,7 @@ export class SystemConfigRepository implements ISystemConfigRepository {
}
readFile(filename: string): Promise<string> {
return readFile(filename, { encoding: 'utf-8' });
return readFile(filename, { encoding: 'utf8' });
}
saveAll(items: SystemConfigEntity[]): Promise<SystemConfigEntity[]> {

View File

@@ -74,11 +74,7 @@ export class UserRepository implements IUserRepository {
}
async delete(user: UserEntity, hard?: boolean): Promise<UserEntity> {
if (hard) {
return this.userRepository.remove(user);
} else {
return this.userRepository.softRemove(user);
}
return hard ? this.userRepository.remove(user) : this.userRepository.softRemove(user);
}
async restore(user: UserEntity): Promise<UserEntity> {

View File

@@ -1,10 +1,11 @@
#!/usr/bin/env node
import { ISystemConfigRepository } from '@app/domain';
import { INestApplication } from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import { Test } from '@nestjs/testing';
import { TypeOrmModule } from '@nestjs/typeorm';
import { mkdir, rm, writeFile } from 'fs/promises';
import { join } from 'path';
import { mkdir, rm, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { databaseConfig } from '../database.config';
import { databaseEntities } from '../entities';
import { GENERATE_SQL_KEY, GenerateSqlQueries } from '../infra.util';
@@ -157,7 +158,7 @@ class SqlGenerator {
private async write() {
for (const [repoName, data] of Object.entries(this.results)) {
const filename = repoName.replace(/[A-Z]/g, (letter) => `.${letter.toLowerCase()}`).replace('.', '');
const filename = repoName.replaceAll(/[A-Z]/g, (letter) => `.${letter.toLowerCase()}`).replace('.', '');
const file = join(this.options.targetDir, `${filename}.sql`);
await writeFile(file, data.join('\n\n') + '\n');
}

View File

@@ -1,8 +1,6 @@
import { format } from 'sql-formatter';
import { Logger } from 'typeorm';
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { format } = require('sql-formatter');
export class SqlLogger implements Logger {
queries: string[] = [];
errors: Array<{ error: string | Error; query: string }> = [];

View File

@@ -16,21 +16,23 @@ export class AuditSubscriber implements EntitySubscriberInterface<AssetEntity |
private getAudit(entityName: string, entity: any): Partial<AuditEntity> | null {
switch (entityName) {
case AssetEntity.name:
case AssetEntity.name: {
const asset = entity as AssetEntity;
return {
entityType: EntityType.ASSET,
entityId: asset.id,
ownerId: asset.ownerId,
};
}
case AlbumEntity.name:
case AlbumEntity.name: {
const album = entity as AlbumEntity;
return {
entityType: EntityType.ALBUM,
entityId: album.id,
ownerId: album.ownerId,
};
}
}
return null;