feat(server): OpenTelemetry integration (#7356)

* wip

* span class decorator

fix typing

* improvements

* noisy postgres logs

formatting

* add source

* strict string comparison

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* remove debug code

* execution time histogram

* remove prometheus stuff

remove prometheus data

* disable by default

disable nestjs-otel stuff by default

update imports

* re-add postgres instrumentation

formatting

formatting

* refactor: execution time histogram

* decorator alias

* formatting

* keep original method order in filesystem repo

* linting

* enable otel sdk in e2e

* actually enable otel sdk in e2e

* share exclude paths

* formatting

* fix rebase

* more buckets

* add example setup

* add envs

fix

actual fix

* linting

* update comments

* update docker env

* use more specific env

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

server/src/infra/repositories/access.repository.ts

@@ -14,6 +14,7 @@ import {
 } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { ChunkedSet } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
 type IActivityAccess = IAccessRepository['activity'];
 type IAlbumAccess = IAccessRepository['album'];
@@ -24,6 +25,7 @@ type ITimelineAccess = IAccessRepository['timeline'];
 type IPersonAccess = IAccessRepository['person'];
 type IPartnerAccess = IAccessRepository['partner'];
 
+@Instrumentation()
 class ActivityAccess implements IActivityAccess {
   constructor(
     private activityRepository: Repository<ActivityEntity>,

server/src/infra/repositories/activity.repository.ts

@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { IsNull, Repository } from 'typeorm';
 import { ActivityEntity } from '../entities/activity.entity';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
 export interface ActivitySearch {
   albumId?: string;
@@ -12,6 +13,7 @@ export interface ActivitySearch {
   isLiked?: boolean;
 }
 
+@Instrumentation()
 @Injectable()
 export class ActivityRepository implements IActivityRepository {
   constructor(@InjectRepository(ActivityEntity) private repository: Repository<ActivityEntity>) {}

server/src/infra/repositories/album.repository.ts

@@ -8,7 +8,9 @@ import { dataSource } from '../database.config';
 import { AlbumEntity, AssetEntity } from '../entities';
 import { DATABASE_PARAMETER_CHUNK_SIZE, DummyValue, GenerateSql } from '../infra.util';
 import { Chunked, ChunkedArray } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class AlbumRepository implements IAlbumRepository {
   constructor(

server/src/infra/repositories/api-key.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { APIKeyEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class ApiKeyRepository implements IKeyRepository {
   constructor(@InjectRepository(APIKeyEntity) private repository: Repository<APIKeyEntity>) {}

server/src/infra/repositories/asset-stack.repository.ts

@@ -3,7 +3,9 @@ import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { AssetStackEntity } from '../entities';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class AssetStackRepository implements IAssetStackRepository {
   constructor(@InjectRepository(AssetStackEntity) private repository: Repository<AssetStackEntity>) {}

server/src/infra/repositories/asset.repository.ts

@@ -39,6 +39,7 @@ import {
 import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { Chunked, ChunkedArray, OptionalBetween, paginate, paginatedBuilder, searchAssetBuilder } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
 const truncateMap: Record<TimeBucketSize, string> = {
   [TimeBucketSize.DAY]: 'day',
@@ -50,6 +51,7 @@ const dateTrunc = (options: TimeBucketOptions) =>
     truncateMap[options.size]
   }', (asset."localDateTime" at time zone 'UTC')) at time zone 'UTC')::timestamptz`;
 
+@Instrumentation()
 @Injectable()
 export class AssetRepository implements IAssetRepository {
   constructor(

server/src/infra/repositories/audit.repository.ts

@@ -2,7 +2,9 @@ import { AuditSearch, IAuditRepository } from '@app/domain';
 import { InjectRepository } from '@nestjs/typeorm';
 import { LessThan, MoreThan, Repository } from 'typeorm';
 import { AuditEntity } from '../entities';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class AuditRepository implements IAuditRepository {
   constructor(@InjectRepository(AuditEntity) private repository: Repository<AuditEntity>) {}
 

server/src/infra/repositories/communication.repository.ts

@@ -15,7 +15,9 @@ import {
   WebSocketServer,
 } from '@nestjs/websockets';
 import { Server, Socket } from 'socket.io';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @WebSocketGateway({
   cors: true,
   path: '/api/socket.io',

server/src/infra/repositories/crypto.repository.ts

@@ -3,14 +3,26 @@ import { Injectable } from '@nestjs/common';
 import { compareSync, hash } from 'bcrypt';
 import { createHash, randomBytes, randomUUID } from 'node:crypto';
 import { createReadStream } from 'node:fs';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class CryptoRepository implements ICryptoRepository {
-  randomUUID = randomUUID;
-  randomBytes = randomBytes;
+  randomUUID() {
+    return randomUUID();
+  }
 
-  hashBcrypt = hash;
-  compareBcrypt = compareSync;
+  randomBytes(size: number) {
+    return randomBytes(size);
+  }
+
+  hashBcrypt(data: string | Buffer, saltOrRounds: string | number) {
+    return hash(data, saltOrRounds);
+  }
+
+  compareBcrypt(data: string | Buffer, encrypted: string) {
+    return compareSync(data, encrypted);
+  }
 
   hashSha256(value: string) {
     return createHash('sha256').update(value).digest('base64');

server/src/infra/repositories/database.repository.ts

@@ -15,8 +15,10 @@ import { InjectDataSource } from '@nestjs/typeorm';
 import AsyncLock from 'async-lock';
 import { DataSource, EntityManager, QueryRunner } from 'typeorm';
 import { isValidInteger } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 import { ImmichLogger } from '../logger';
 
+@Instrumentation()
 @Injectable()
 export class DatabaseRepository implements IDatabaseRepository {
   private logger = new ImmichLogger(DatabaseRepository.name);

server/src/infra/repositories/filesystem.provider.ts

@@ -13,23 +13,37 @@ import archiver from 'archiver';
 import chokidar, { WatchOptions } from 'chokidar';
 import { glob } from 'fast-glob';
 import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs';
-import fs, { copyFile, readdir, rename, stat, utimes, writeFile } from 'node:fs/promises';
+import fs from 'node:fs/promises';
 import path from 'node:path';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class FilesystemProvider implements IStorageRepository {
   private logger = new ImmichLogger(FilesystemProvider.name);
 
-  readdir = readdir;
+  readdir(folder: string): Promise<string[]> {
+    return fs.readdir(folder);
+  }
 
-  copyFile = copyFile;
+  copyFile(source: string, target: string) {
+    return fs.copyFile(source, target);
+  }
 
-  stat = stat;
+  stat(filepath: string) {
+    return fs.stat(filepath);
+  }
 
-  writeFile = writeFile;
+  writeFile(filepath: string, buffer: Buffer) {
+    return fs.writeFile(filepath, buffer);
+  }
 
-  rename = rename;
+  rename(source: string, target: string) {
+    return fs.rename(source, target);
+  }
 
-  utimes = utimes;
+  utimes(filepath: string, atime: Date, mtime: Date) {
+    return fs.utimes(filepath, atime, mtime);
+  }
 
   createZipStream(): ImmichZipStream {
     const archive = archiver('zip', { store: true });

server/src/infra/repositories/job.repository.ts

@@ -17,7 +17,9 @@ import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullm
 import { CronJob, CronTime } from 'cron';
 import { setTimeout } from 'node:timers/promises';
 import { bullConfig } from '../infra.config';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class JobRepository implements IJobRepository {
   private workers: Partial<Record<QueueName, Worker>> = {};

server/src/infra/repositories/library.repository.ts

@@ -5,7 +5,9 @@ import { IsNull, Not } from 'typeorm';
 import { Repository } from 'typeorm/repository/Repository.js';
 import { LibraryEntity, LibraryType } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class LibraryRepository implements ILibraryRepository {
   constructor(@InjectRepository(LibraryEntity) private repository: Repository<LibraryEntity>) {}

server/src/infra/repositories/machine-learning.repository.ts

@@ -11,9 +11,11 @@ import {
 } from '@app/domain';
 import { Injectable } from '@nestjs/common';
 import { readFile } from 'node:fs/promises';
+import { Instrumentation } from '../instrumentation';
 
 const errorPrefix = 'Machine learning request';
 
+@Instrumentation()
 @Injectable()
 export class MachineLearningRepository implements IMachineLearningRepository {
   private async predict<T>(url: string, input: TextModelInput | VisionModelInput, config: ModelConfig): Promise<T> {
@@ -50,7 +52,7 @@ export class MachineLearningRepository implements IMachineLearningRepository {
     } as CLIPConfig);
   }
 
-  async getFormData(input: TextModelInput | VisionModelInput, config: ModelConfig): Promise<FormData> {
+  private async getFormData(input: TextModelInput | VisionModelInput, config: ModelConfig): Promise<FormData> {
     const formData = new FormData();
     const { enabled, modelName, modelType, ...options } = config;
     if (!enabled) {

server/src/infra/repositories/media.repository.ts

@@ -13,11 +13,13 @@ import fs from 'node:fs/promises';
 import { Writable } from 'node:stream';
 import { promisify } from 'node:util';
 import sharp from 'sharp';
+import { Instrumentation } from '../instrumentation';
 
 const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
 sharp.concurrency(0);
 sharp.cache({ files: 0 });
 
+@Instrumentation()
 export class MediaRepository implements IMediaRepository {
   private logger = new ImmichLogger(MediaRepository.name);
 
@@ -115,19 +117,6 @@ export class MediaRepository implements IMediaRepository {
     });
   }
 
-  configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
-    return ffmpeg(input, { niceness: 10 })
-      .inputOptions(options.inputOptions)
-      .outputOptions(options.outputOptions)
-      .output(output)
-      .on('error', (error, stdout, stderr) => this.logger.error(stderr || error));
-  }
-
-  chainPath(existing: string, path: string) {
-    const separator = existing.endsWith(':') ? '' : ':';
-    return `${existing}${separator}${path}`;
-  }
-
   async generateThumbhash(imagePath: string): Promise<Buffer> {
     const maxSize = 100;
 
@@ -140,4 +129,17 @@ export class MediaRepository implements IMediaRepository {
     const thumbhash = await import('thumbhash');
     return Buffer.from(thumbhash.rgbaToThumbHash(info.width, info.height, data));
   }
+
+  private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
+    return ffmpeg(input, { niceness: 10 })
+      .inputOptions(options.inputOptions)
+      .outputOptions(options.outputOptions)
+      .output(output)
+      .on('error', (error, stdout, stderr) => this.logger.error(stderr || error));
+  }
+
+  private chainPath(existing: string, path: string) {
+    const separator = existing.endsWith(':') ? '' : ':';
+    return `${existing}${separator}${path}`;
+  }
 }

server/src/infra/repositories/metadata.repository.ts

@@ -23,7 +23,9 @@ import * as readLine from 'node:readline';
 import { DataSource, QueryRunner, Repository } from 'typeorm';
 import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class MetadataRepository implements IMetadataRepository {
   constructor(
     @InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,

server/src/infra/repositories/move.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { MoveEntity, PathType } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class MoveRepository implements IMoveRepository {
   constructor(@InjectRepository(MoveEntity) private repository: Repository<MoveEntity>) {}

server/src/infra/repositories/partner.repository.ts

@@ -3,7 +3,9 @@ import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { DeepPartial, Repository } from 'typeorm';
 import { PartnerEntity } from '../entities';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class PartnerRepository implements IPartnerRepository {
   constructor(@InjectRepository(PartnerEntity) private readonly repository: Repository<PartnerEntity>) {}

server/src/infra/repositories/person.repository.ts

@@ -15,7 +15,9 @@ import { FindManyOptions, FindOptionsRelations, FindOptionsSelect, In, Repositor
 import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { ChunkedArray, asVector, paginate } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class PersonRepository implements IPersonRepository {
   constructor(
     @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,

server/src/infra/repositories/search.repository.ts

@@ -26,7 +26,9 @@ import { Repository, SelectQueryBuilder } from 'typeorm';
 import { vectorExt } from '../database.config';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { asVector, isValidInteger, paginatedBuilder, searchAssetBuilder } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class SearchRepository implements ISearchRepository {
   private logger = new ImmichLogger(SearchRepository.name);

server/src/infra/repositories/server-info.repository.ts

@@ -1,6 +1,8 @@
 import { GitHubRelease, IServerInfoRepository } from '@app/domain';
 import { Injectable } from '@nestjs/common';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class ServerInfoRepository implements IServerInfoRepository {
   async getGitHubRelease(): Promise<GitHubRelease> {

server/src/infra/repositories/shared-link.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { SharedLinkEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class SharedLinkRepository implements ISharedLinkRepository {
   constructor(@InjectRepository(SharedLinkEntity) private repository: Repository<SharedLinkEntity>) {}

server/src/infra/repositories/system-config.repository.ts

@@ -5,12 +5,15 @@ import { In, Repository } from 'typeorm';
 import { SystemConfigEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
 import { Chunked } from '../infra.utils';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class SystemConfigRepository implements ISystemConfigRepository {
   constructor(
     @InjectRepository(SystemConfigEntity)
     private repository: Repository<SystemConfigEntity>,
   ) {}
+
   async fetchStyle(url: string) {
     try {
       const response = await fetch(url);

server/src/infra/repositories/system-metadata.repository.ts

@@ -2,12 +2,15 @@ import { ISystemMetadataRepository } from '@app/domain/repositories/system-metad
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { SystemMetadata, SystemMetadataEntity } from '../entities';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 export class SystemMetadataRepository implements ISystemMetadataRepository {
   constructor(
     @InjectRepository(SystemMetadataEntity)
     private repository: Repository<SystemMetadataEntity>,
   ) {}
+
   async get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null> {
     const metadata = await this.repository.findOne({ where: { key } });
     if (!metadata) {

server/src/infra/repositories/tag.repository.ts

@@ -3,7 +3,9 @@ import { AssetEntity, TagEntity } from '@app/infra/entities';
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class TagRepository implements ITagRepository {
   constructor(

server/src/infra/repositories/user-token.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { UserTokenEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class UserTokenRepository implements IUserTokenRepository {
   constructor(@InjectRepository(UserTokenEntity) private repository: Repository<UserTokenEntity>) {}

server/src/infra/repositories/user.repository.ts

@@ -4,7 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { IsNull, Not, Repository } from 'typeorm';
 import { AssetEntity, UserEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
+import { Instrumentation } from '../instrumentation';
 
+@Instrumentation()
 @Injectable()
 export class UserRepository implements IUserRepository {
   constructor(