From b7e4e554895b7766743aafe6326a9df26e1a058d Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Wed, 24 Sep 2025 13:56:46 -0400 Subject: [PATCH] wip --- server/Dockerfile | 18 + server/Dockerfile.dev | 15 + server/src/app.module.ts | 3 +- server/src/constants.ts | 5 + .../controllers/asset-upload.controller.ts | 82 +++++ server/src/controllers/index.ts | 2 + server/src/dtos/upload.dto.ts | 133 ++++++++ server/src/enum.ts | 2 + server/src/repositories/storage.repository.ts | 21 +- server/src/services/asset-upload.service.ts | 318 ++++++++++++++++++ server/src/services/index.ts | 2 + 11 files changed, 595 insertions(+), 6 deletions(-) create mode 100644 server/src/controllers/asset-upload.controller.ts create mode 100644 server/src/dtos/upload.dto.ts create mode 100644 server/src/services/asset-upload.service.ts diff --git a/server/Dockerfile b/server/Dockerfile index 6702b338c5..449e5b484e 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,4 +1,21 @@ FROM ghcr.io/immich-app/base-server-dev:202509210934@sha256:b5ce2d7eaf379d4cf15efd4bab180d8afc8a80d20b36c9800f4091aca6ae267e AS builder + +ARG TUSD_RELEASE=v2.8.0 +ARG TUSD_AMD64_SHA256=33aaa15e14c7b025772d28605c6e0733ec2e8c1d6b646fa6b83af0049022a9ce +ARG TUSD_ARM64_SHA256=07f78ee2fd552296c40f3a7a5a06325a06b53f1dd1ba996a60ecc52696906481 + +# TODO: move this to base image +RUN apt-get update && apt-get install -y curl && \ + if [ $(arch) = "x86_64" ]; then \ + curl -fsSL -o /tmp/tusd.deb "https://github.com/tus/tusd/releases/download/${TUSD_RELEASE}/tusd_snapshot_amd64.deb" && \ + echo "${TUSD_AMD64_SHA256} /tmp/tusd.deb" | sha256sum -c; \ + else \ + curl -fsSL -o /tmp/tusd.deb "https://github.com/tus/tusd/releases/download/${TUSD_RELEASE}/tusd_snapshot_arm64.deb" && \ + echo "${TUSD_ARM64_SHA256} /tmp/tusd.deb" | sha256sum -c; \ + fi && \ + dpkg -i /tmp/tusd.deb && \ + rm -f /tmp/tusd.deb + ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ CI=1 \ COREPACK_HOME=/tmp @@ -40,6 +57,7 @@ ENV NODE_ENV=production \ NVIDIA_DRIVER_CAPABILITIES=all \ NVIDIA_VISIBLE_DEVICES=all +COPY --from=builder /usr/bin/tusd /usr/bin/tusd COPY --from=server /output/server-pruned ./server COPY --from=web /usr/src/app/web/build /build/www COPY --from=cli /output/cli-pruned ./cli diff --git a/server/Dockerfile.dev b/server/Dockerfile.dev index dd2e931745..c1fa5c777d 100644 --- a/server/Dockerfile.dev +++ b/server/Dockerfile.dev @@ -1,6 +1,21 @@ # dev build FROM ghcr.io/immich-app/base-server-dev:202509210934@sha256:b5ce2d7eaf379d4cf15efd4bab180d8afc8a80d20b36c9800f4091aca6ae267e AS dev +ARG TUSD_RELEASE=v2.8.0 +ARG TUSD_AMD64_SHA256=33aaa15e14c7b025772d28605c6e0733ec2e8c1d6b646fa6b83af0049022a9ce +ARG TUSD_ARM64_SHA256=07f78ee2fd552296c40f3a7a5a06325a06b53f1dd1ba996a60ecc52696906481 + +# TODO: move this to base image +RUN if [ $(arch) = "x86_64" ]; then \ + curl -fsSL -o /tmp/tusd.deb "https://github.com/tus/tusd/releases/download/${TUSD_RELEASE}/tusd_snapshot_amd64.deb" && \ + echo "${TUSD_AMD64_SHA256} /tmp/tusd.deb" | sha256sum -c; \ + else \ + curl -fsSL -o /tmp/tusd.deb "https://github.com/tus/tusd/releases/download/${TUSD_RELEASE}/tusd_snapshot_arm64.deb" && \ + echo "${TUSD_ARM64_SHA256} /tmp/tusd.deb" | sha256sum -c; \ + fi && \ + dpkg -i /tmp/tusd.deb && \ + rm -f /tmp/tusd.deb + ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ CI=1 \ COREPACK_HOME=/tmp diff --git a/server/src/app.module.ts b/server/src/app.module.ts index 8d261463e7..4bf403de02 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -28,7 +28,6 @@ import { getKyselyConfig } from 'src/utils/database'; const common = [...repositories, ...services, GlobalExceptionFilter]; export const middleware = [ - FileUploadInterceptor, { provide: APP_FILTER, useClass: GlobalExceptionFilter }, { provide: APP_PIPE, useValue: new ValidationPipe({ transform: true, whitelist: true }) }, { provide: APP_INTERCEPTOR, useClass: LoggingInterceptor }, @@ -85,7 +84,7 @@ class BaseModule implements OnModuleInit, OnModuleDestroy { @Module({ imports: [...imports, ScheduleModule.forRoot()], controllers: [...controllers], - providers: [...common, ...middleware, { provide: IWorker, useValue: ImmichWorker.Api }], + providers: [...common, ...middleware, FileUploadInterceptor, { provide: IWorker, useValue: ImmichWorker.Api }], }) export class ApiModule extends BaseModule {} diff --git a/server/src/constants.ts b/server/src/constants.ts index 1bae521a9f..d691c4bdd5 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -16,6 +16,11 @@ export const ADDED_IN_PREFIX = 'This property was added in '; export const JOBS_ASSET_PAGINATION_SIZE = 1000; export const JOBS_LIBRARY_PAGINATION_SIZE = 10_000; +export const UPLOAD_TUSD_SOCKET_PATH = '/tmp/immich-tusd.sock'; +export const UPLOAD_CHUNK_DIRECTORY = '/tmp/immich-chunked-uploads'; +export const UPLOAD_TUSD_CONNECT_TIMEOUT_MS = 1000; +export const UPLOAD_TUSD_CONNECT_BACKOFF_MS = 100; +export const UPLOAD_TUSD_CONNECT_RETRIES = 30; export const EXTENSION_NAMES: Record = { cube: 'cube', diff --git a/server/src/controllers/asset-upload.controller.ts b/server/src/controllers/asset-upload.controller.ts new file mode 100644 index 0000000000..6edbaef51d --- /dev/null +++ b/server/src/controllers/asset-upload.controller.ts @@ -0,0 +1,82 @@ +import { All, BadRequestException, Body, Controller, HttpException, Post, Req, Res } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; +import { plainToInstance } from 'class-transformer'; +import { validateSync } from 'class-validator'; +import { Request, Response } from 'express'; +import { + TusdHookRequestDto, + TusdHookRequestType, + TusdHookResponseDto, + TusdPreCreateEventDto, + TusdPreFinishEventDto, +} from 'src/dtos/upload.dto'; +import { Permission } from 'src/enum'; +import { LoggingRepository } from 'src/repositories/logging.repository'; +import { AssetUploadService } from 'src/services/asset-upload.service'; +import { AuthService } from 'src/services/auth.service'; + +@ApiTags('Upload') +@Controller('upload') +export class AssetUploadController { + constructor( + private authService: AuthService, + private uploadService: AssetUploadService, + private logger: LoggingRepository, + ) { + logger.setContext(AssetUploadController.name); + } + + // Proxies chunked upload requests to the tusd server. + // Auth is handled in the pre-create and pre-finish hooks from tusd. + @All('asset{/*all}') + handleChunks(@Req() request: Request, @Res() response: Response): Promise { + return this.uploadService.proxyChunks(request, response); + } + + // This controller handles webhooks from the tusd server. + // See https://tus.github.io/tusd/advanced-topics/hooks/ for more information. + @Post('hook') + async processHook(@Body() payload: TusdHookRequestDto): Promise { + const request = payload.Event.HTTPRequest; + const lowerCaseHeaders: Record = {}; + for (const [key, [value]] of Object.entries(request.Header)) { + lowerCaseHeaders[key.toLowerCase()] = value; + } + + try { + const auth = await this.authService.authenticate({ + headers: lowerCaseHeaders, + queryParams: {}, + metadata: { adminRoute: false, sharedLinkRoute: false, permission: Permission.AssetUpload, uri: request.URI }, + }); + + switch (payload.Type) { + case TusdHookRequestType.PreCreate: { + const dto = plainToInstance(TusdPreCreateEventDto, payload.Event); + const errors = validateSync(dto, { whitelist: true }); + if (errors.length > 0) { + throw new BadRequestException('Invalid payload'); + } + return await this.uploadService.handlePreCreate(auth, dto, lowerCaseHeaders); + } + case TusdHookRequestType.PreFinish: { + const dto = plainToInstance(TusdPreFinishEventDto, payload.Event); + const errors = validateSync(dto, { whitelist: true }); + if (errors.length > 0) { + throw new BadRequestException('Invalid payload'); + } + return await this.uploadService.handlePreFinish(auth, dto); + } + default: { + throw new Error(`Unhandled hook type: ${payload.Type}`); + } + } + } catch (error: any) { + if (error instanceof HttpException) { + return { RejectUpload: true, HTTPResponse: { StatusCode: error.getStatus(), Body: error.message } }; + } + this.logger.error('Error processing upload hook', error); + return { RejectUpload: true, HTTPResponse: { StatusCode: 500 } }; + } + } +} diff --git a/server/src/controllers/index.ts b/server/src/controllers/index.ts index e3661ec794..18280c45ec 100644 --- a/server/src/controllers/index.ts +++ b/server/src/controllers/index.ts @@ -3,6 +3,7 @@ import { AlbumController } from 'src/controllers/album.controller'; import { ApiKeyController } from 'src/controllers/api-key.controller'; import { AppController } from 'src/controllers/app.controller'; import { AssetMediaController } from 'src/controllers/asset-media.controller'; +import { AssetUploadController } from 'src/controllers/asset-upload.controller'; import { AssetController } from 'src/controllers/asset.controller'; import { AuthAdminController } from 'src/controllers/auth-admin.controller'; import { AuthController } from 'src/controllers/auth.controller'; @@ -40,6 +41,7 @@ export const controllers = [ AppController, AssetController, AssetMediaController, + AssetUploadController, AuthController, AuthAdminController, DownloadController, diff --git a/server/src/dtos/upload.dto.ts b/server/src/dtos/upload.dto.ts new file mode 100644 index 0000000000..d90ceeeb18 --- /dev/null +++ b/server/src/dtos/upload.dto.ts @@ -0,0 +1,133 @@ +import { Type } from 'class-transformer'; +import { IsEnum, IsInt, IsNotEmpty, IsObject, IsString, IsUUID, ValidateNested } from 'class-validator'; +import { AssetMediaCreateDto } from 'src/dtos/asset-media.dto'; + +export enum TusdHookRequestType { + PreCreate = 'pre-create', + PreFinish = 'pre-finish', +} + +export enum TusdHookStorageType { + FileStore = 'filestore', +} + +export class TusdStorageDto { + @IsEnum(TusdHookStorageType) + Type!: string; + + @IsString() + @IsNotEmpty() + Path!: string; + + @IsString() + @IsNotEmpty() + InfoPath!: string; +} + +export class UploadAssetDataDto extends AssetMediaCreateDto { + @IsString() + @IsNotEmpty() + declare filename: string; +} + +export class TusdMetaDataDto { + @IsString() + @IsNotEmpty() + declare AssetData: string; // base64-encoded JSON string of UploadAssetDataDto +} + +export class TusdPreCreateUploadDto { + @IsInt() + Size!: number; +} + +export class TusdPreFinishUploadDto { + @IsUUID() + ID!: string; + + @IsInt() + Size!: number; + + @Type(() => TusdMetaDataDto) + @ValidateNested() + @IsObject() + MetaData!: TusdMetaDataDto; + + @Type(() => TusdStorageDto) + @ValidateNested() + @IsObject() + Storage!: TusdStorageDto; +} + +export class TusdHttpRequestDto { + @IsString() + @IsNotEmpty() + Method!: string; + + @IsString() + @IsNotEmpty() + URI!: string; + + @IsObject() + Header!: Record; +} + +export class TusdPreCreateEventDto { + @Type(() => TusdPreCreateUploadDto) + @ValidateNested() + @IsObject() + Upload!: TusdPreCreateUploadDto; + + @Type(() => TusdHttpRequestDto) + @ValidateNested() + @IsObject() + HTTPRequest!: TusdHttpRequestDto; +} + +export class TusdPreFinishEventDto { + @Type(() => TusdPreFinishUploadDto) + @ValidateNested() + @IsObject() + Upload!: TusdPreFinishUploadDto; + + @Type(() => TusdHttpRequestDto) + @ValidateNested() + @IsObject() + HTTPRequest!: TusdHttpRequestDto; +} + +export class TusdHookRequestDto { + @IsEnum(TusdHookRequestType) + Type!: TusdHookRequestType; + + @IsObject() + Event!: TusdPreCreateEventDto | TusdPreFinishEventDto; +} + +export class TusdHttpResponseDto { + StatusCode!: number; + + Body?: string; + + Header?: Record; +} + +export class TusdChangeFileInfoStorageDto { + Path?: string; +} + +export class TusdChangeFileInfoDto { + ID?: string; + + MetaData?: TusdMetaDataDto; + + Storage?: TusdChangeFileInfoStorageDto; +} + +export class TusdHookResponseDto { + HTTPResponse?: TusdHttpResponseDto; + + RejectUpload?: boolean; + + ChangeFileInfo?: TusdChangeFileInfoDto; +} diff --git a/server/src/enum.ts b/server/src/enum.ts index 646138b060..3baf72db72 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -20,6 +20,7 @@ export enum ImmichHeader { SharedLinkSlug = 'x-immich-share-slug', Checksum = 'x-immich-checksum', Cid = 'x-immich-cid', + AssetData = 'x-immich-asset-data', } export enum ImmichQuery { @@ -493,6 +494,7 @@ export enum BootstrapEventPriority { JobService = -190, // Initialise config after other bootstrap services, stop other services from using config on bootstrap SystemConfig = 100, + UploadService = 200, } export enum QueueName { diff --git a/server/src/repositories/storage.repository.ts b/server/src/repositories/storage.repository.ts index 7d6b634845..f2b84fdf3d 100644 --- a/server/src/repositories/storage.repository.ts +++ b/server/src/repositories/storage.repository.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import archiver from 'archiver'; import chokidar, { ChokidarOptions } from 'chokidar'; import { escapePath, glob, globStream } from 'fast-glob'; -import { constants, createReadStream, createWriteStream, existsSync, mkdirSync } from 'node:fs'; +import { constants, createReadStream, createWriteStream, existsSync, mkdirSync, unlinkSync } from 'node:fs'; import fs from 'node:fs/promises'; import path from 'node:path'; import { Readable, Writable } from 'node:stream'; @@ -134,6 +134,16 @@ export class StorageRepository { } } + unlinkSync(file: string) { + try { + unlinkSync(file); + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== 'ENOENT') { + throw error; + } + } + } + async unlinkDir(folder: string, options: { recursive?: boolean; force?: boolean }) { await fs.rm(folder, options); } @@ -156,10 +166,13 @@ export class StorageRepository { } } + mkdir(filepath: string): Promise { + return fs.mkdir(filepath, { recursive: true }); + } + mkdirSync(filepath: string): void { - if (!existsSync(filepath)) { - mkdirSync(filepath, { recursive: true }); - } + // does not throw an error if the folder already exists + mkdirSync(filepath, { recursive: true }); } existsSync(filepath: string) { diff --git a/server/src/services/asset-upload.service.ts b/server/src/services/asset-upload.service.ts new file mode 100644 index 0000000000..22ef0f9b54 --- /dev/null +++ b/server/src/services/asset-upload.service.ts @@ -0,0 +1,318 @@ +import { BadRequestException, Injectable, InternalServerErrorException } from '@nestjs/common'; +import { plainToInstance } from 'class-transformer'; +import { validateSync } from 'class-validator'; +import { Request, Response } from 'express'; +import { ChildProcess, spawn } from 'node:child_process'; +import { request as httpRequest } from 'node:http'; +import { createConnection } from 'node:net'; +import { extname, join, parse, resolve } from 'node:path'; +import { + UPLOAD_CHUNK_DIRECTORY, + UPLOAD_TUSD_CONNECT_BACKOFF_MS, + UPLOAD_TUSD_CONNECT_RETRIES, + UPLOAD_TUSD_CONNECT_TIMEOUT_MS, + UPLOAD_TUSD_SOCKET_PATH, +} from 'src/constants'; +import { StorageCore } from 'src/cores/storage.core'; +import { OnEvent } from 'src/decorators'; +import { AssetMediaStatus } from 'src/dtos/asset-media-response.dto'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { + TusdHookResponseDto, + TusdPreCreateEventDto, + TusdPreFinishEventDto, + UploadAssetDataDto, +} from 'src/dtos/upload.dto'; +import { AssetVisibility, ImmichHeader, ImmichWorker, JobName, StorageFolder } from 'src/enum'; +import { BaseService } from 'src/services/base.service'; +import { isAssetChecksumConstraint } from 'src/utils/database'; +import { mimeTypes } from 'src/utils/mime-types'; +import { fromChecksum } from 'src/utils/request'; + +@Injectable() +export class AssetUploadService extends BaseService { + private tusdProcess?: ChildProcess; + + @OnEvent({ name: 'AppBootstrap', workers: [ImmichWorker.Api] }) + async onBootstrap() { + await this.storageRepository.unlinkDir(UPLOAD_CHUNK_DIRECTORY, { recursive: true, force: true }); + await this.storageRepository.mkdir(UPLOAD_CHUNK_DIRECTORY); + const { host, port } = this.configRepository.getEnv(); + + const args = [ + '-unix-sock', + UPLOAD_TUSD_SOCKET_PATH, + '--base-path', + '/api/upload/asset', + '--upload-dir', + UPLOAD_CHUNK_DIRECTORY, + '--hooks-http', + `http://${host ?? 'localhost'}:${port}:/api/upload/hook`, + '--hooks-enabled-events', + 'pre-create,pre-finish', + '--behind-proxy', + '-enable-experimental-protocol', + '-shutdown-timeout', + '5s', + '-hooks-http-forward-headers', + Object.values(ImmichHeader).join(',') + ',authorization', + '-disable-download', + '-disable-termination', + ]; + + this.logger.log(`Starting tusd server with args: ${args.join(' ')}`); + + this.tusdProcess = spawn('tusd', args, { + stdio: ['ignore', 'pipe', 'pipe'], + }); + + this.tusdProcess.stdout?.on('data', (data) => this.logger.verboseFn(() => `tusd: ${data.toString().trim()}`)); + + this.tusdProcess.stderr?.on('data', (data) => this.logger.warn(`tusd: ${data.toString().trim()}`)); + + this.tusdProcess.on('error', (error) => { + this.logger.error(`tusd failed to start: ${error.message}`); + process.exit(1); + }); + + this.tusdProcess.on('exit', async (code) => { + this.tusdProcess = undefined; + try { + await Promise.all([ + this.storageRepository.unlink(UPLOAD_TUSD_SOCKET_PATH), + this.storageRepository.unlinkDir(UPLOAD_CHUNK_DIRECTORY, { recursive: true, force: true }), + ]); + } finally { + if (code) { + this.logger.error(`tusd exited unexpectedly with code ${code}`); + // TODO: more graceful shutdown + process.exit(code); + } + } + }); + + this.logger.log('Waiting for tusd server...'); + await this.waitForTusd(UPLOAD_TUSD_SOCKET_PATH); + this.logger.log('tusd server started successfully'); + } + + @OnEvent({ name: 'AppShutdown' }) + onShutdown() { + this.tusdProcess?.kill('SIGTERM'); + this.tusdProcess = undefined; + } + + async proxyChunks(request: Request, response: Response): Promise { + if (!this.tusdProcess) { + throw new InternalServerErrorException('tusd server is not running'); + } + + delete request.headers.host; + request.headers.connection = 'close'; // not sure why, but it doesn't respond for 60 seconds without this + return new Promise((resolve, reject) => { + const proxyReq = httpRequest( + { + socketPath: UPLOAD_TUSD_SOCKET_PATH, + path: request.url, + method: request.method, + headers: request.headers, + }, + (proxyRes) => { + response.status(proxyRes.statusCode || 200); + for (const [key, value] of Object.entries(proxyRes.headers)) { + if (value !== undefined) { + response.setHeader(key, value); + } + } + + proxyRes.pipe(response); + proxyRes.on('end', resolve); + proxyRes.on('error', reject); + }, + ); + + proxyReq.on('error', (error: any) => { + this.logger.error(`Failed to proxy to tusd: ${error.message}`); + reject(new InternalServerErrorException('Upload service unavailable')); + }); + + if (request.readable) { + request.pipe(proxyReq); + } else { + proxyReq.end(); + } + }); + } + + private async waitForTusd(socketPath: string, maxRetries = UPLOAD_TUSD_CONNECT_RETRIES): Promise { + for (let i = 0; i < maxRetries; i++) { + const ready = await new Promise((resolve) => { + const socket = createConnection(socketPath, () => { + socket.end(); + resolve(true); + }); + + socket.on('error', () => { + resolve(false); + }); + + socket.setTimeout(UPLOAD_TUSD_CONNECT_TIMEOUT_MS); + socket.on('timeout', () => { + socket.destroy(); + resolve(false); + }); + }); + + if (ready) { + return; + } + + await new Promise((r) => setTimeout(r, UPLOAD_TUSD_CONNECT_BACKOFF_MS)); + } + + throw new Error('tusd server failed to start within timeout'); + } + + async handlePreCreate( + auth: AuthDto, + payload: TusdPreCreateEventDto, + headers: Record, + ): Promise { + this.logger.debugFn(() => `PreCreate hook received: ${JSON.stringify(payload)}`); + const checksum = headers[ImmichHeader.Checksum]?.[0]; + if (checksum) { + const existingId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, fromChecksum(checksum)); + if (existingId) { + const body = JSON.stringify({ status: AssetMediaStatus.DUPLICATE, id: existingId }); + return { + RejectUpload: true, + HTTPResponse: { StatusCode: 200, Body: body }, + }; + } + } + + this.requireQuota(auth, payload.Upload.Size); + const encodedAssetData = headers[ImmichHeader.AssetData]; + if (!encodedAssetData) { + throw new BadRequestException(`Missing ${ImmichHeader.AssetData} header`); + } + + const assetData = this.parseMetadata(encodedAssetData); + this.logger.log('assetData: ' + JSON.stringify(assetData)); + + const assetId = this.cryptoRepository.randomUUID(); + const folder = StorageCore.getNestedFolder(StorageFolder.Upload, auth.user.id, assetId); + const extension = extname(assetId); + const path = join(folder, `${assetId}${extension}`); + return { + ChangeFileInfo: { + ID: assetId, + Storage: { Path: path }, + MetaData: { + AssetData: encodedAssetData, + }, + }, + }; + } + + async handlePreFinish(auth: AuthDto, dto: TusdPreFinishEventDto): Promise { + this.logger.debugFn(() => `PreFinish hook received: ${JSON.stringify(dto)}`); + const { + Upload: { + MetaData: { AssetData: encodedAssetData }, + Storage: { Path: path }, + Size: size, + ID: assetId, + }, + } = dto; + + const parsedPath = parse(resolve(path)); + if (!parsedPath.dir.startsWith(StorageCore.getFolderLocation(StorageFolder.Upload, auth.user.id))) { + throw new BadRequestException('Path is not in user folder'); + } + const metadata = this.parseMetadata(encodedAssetData); + + try { + this.requireQuota(auth, size); + const checksum = await this.cryptoRepository.hashFile(path); + await this.storageRepository.utimes(path, new Date(), new Date(metadata.fileModifiedAt)); + try { + await this.assetRepository.create({ + id: assetId, + ownerId: auth.user.id, + libraryId: null, + checksum, + originalPath: path, + deviceAssetId: metadata.deviceAssetId, + deviceId: metadata.deviceId, + fileCreatedAt: metadata.fileCreatedAt, + fileModifiedAt: metadata.fileModifiedAt, + localDateTime: metadata.fileCreatedAt, + type: mimeTypes.assetType(path), + isFavorite: metadata.isFavorite, + duration: metadata.duration || null, + visibility: metadata.visibility || AssetVisibility.Timeline, + originalFileName: metadata.filename || parsedPath.base, + }); + } catch (error: any) { + if (isAssetChecksumConstraint(error)) { + const duplicateId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, checksum); + if (!duplicateId) { + this.logger.error(`Error locating duplicate for checksum constraint`); + throw new InternalServerErrorException(); + } + void this.tryUnlink(path); + const body = JSON.stringify({ id: duplicateId, status: AssetMediaStatus.DUPLICATE }); + return { HTTPResponse: { StatusCode: 200, Body: body } }; + } + } + } catch (error: any) { + void this.tryUnlink(path); + throw error; + } + + await this.userRepository.updateUsage(auth.user.id, size); + await this.assetRepository.upsertExif({ assetId: assetId, fileSizeInByte: size }); + await this.jobRepository.queue({ + name: JobName.AssetExtractMetadata, + data: { id: assetId, source: 'upload' }, + }); + + this.logger.log(`Asset created from chunked upload: ${assetId}`); + const body = JSON.stringify({ id: assetId, status: AssetMediaStatus.CREATED }); + return { HTTPResponse: { StatusCode: 201, Body: body } }; + } + + private async tryUnlink(path: string): Promise { + try { + await this.storageRepository.unlink(path); + } catch { + this.logger.warn(`Failed to remove file at ${path}`); + } + } + + private requireQuota(auth: AuthDto, size: number) { + if (auth.user.quotaSizeInBytes === null) { + return; + } + + if (auth.user.quotaSizeInBytes < auth.user.quotaUsageInBytes + size) { + throw new BadRequestException('Quota has been exceeded!'); + } + } + + private parseMetadata(encodedAssetData: string): UploadAssetDataDto { + let assetData: any; + try { + assetData = JSON.parse(Buffer.from(encodedAssetData, 'base64').toString('utf8')); + } catch { + throw new BadRequestException(`Invalid ${ImmichHeader.AssetData} header`); + } + const dto = plainToInstance(UploadAssetDataDto, assetData); + const assetDataErrors = validateSync(dto, { whitelist: true }); + if (assetDataErrors.length > 0 || !mimeTypes.isAsset(assetData.filename)) { + throw new BadRequestException(`Invalid ${ImmichHeader.AssetData} header`); + } + return dto; + } +} diff --git a/server/src/services/index.ts b/server/src/services/index.ts index cad38ca1f4..6771baf4a3 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -3,6 +3,7 @@ import { AlbumService } from 'src/services/album.service'; import { ApiKeyService } from 'src/services/api-key.service'; import { ApiService } from 'src/services/api.service'; import { AssetMediaService } from 'src/services/asset-media.service'; +import { AssetUploadService } from 'src/services/asset-upload.service'; import { AssetService } from 'src/services/asset.service'; import { AuditService } from 'src/services/audit.service'; import { AuthAdminService } from 'src/services/auth-admin.service'; @@ -47,6 +48,7 @@ export const services = [ AlbumService, ApiService, AssetMediaService, + AssetUploadService, AssetService, AuditService, AuthService,