feat(server, web): quotas (#4471)

* feat: quotas

* chore: open api

* chore: update status box and upload error message

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

server/src/domain/asset/asset.service.spec.ts

@@ -13,6 +13,7 @@ import {
   newPartnerRepositoryMock,
   newStorageRepositoryMock,
   newSystemConfigRepositoryMock,
+  newUserRepositoryMock,
 } from '@test';
 import { when } from 'jest-when';
 import { Readable } from 'stream';
@@ -28,6 +29,7 @@ import {
   IPartnerRepository,
   IStorageRepository,
   ISystemConfigRepository,
+  IUserRepository,
   JobItem,
   TimeBucketSize,
 } from '../repositories';
@@ -67,6 +69,7 @@ const uploadFile = {
       checksum: Buffer.from('checksum', 'utf8'),
       originalPath: 'upload/admin/image.jpeg',
       originalName: 'image.jpeg',
+      size: 1000,
     },
   },
   filename: (fieldName: UploadFieldName, filename: string) => {
@@ -79,6 +82,7 @@ const uploadFile = {
         checksum: Buffer.from('checksum', 'utf8'),
         originalPath: `upload/admin/${filename}`,
         originalName: filename,
+        size: 1000,
       },
     };
   },
@@ -167,6 +171,7 @@ describe(AssetService.name, () => {
   let cryptoMock: jest.Mocked<ICryptoRepository>;
   let jobMock: jest.Mocked<IJobRepository>;
   let storageMock: jest.Mocked<IStorageRepository>;
+  let userMock: jest.Mocked<IUserRepository>;
   let communicationMock: jest.Mocked<ICommunicationRepository>;
   let configMock: jest.Mocked<ISystemConfigRepository>;
   let partnerMock: jest.Mocked<IPartnerRepository>;
@@ -182,6 +187,7 @@ describe(AssetService.name, () => {
     cryptoMock = newCryptoRepositoryMock();
     jobMock = newJobRepositoryMock();
     storageMock = newStorageRepositoryMock();
+    userMock = newUserRepositoryMock();
     configMock = newSystemConfigRepositoryMock();
     partnerMock = newPartnerRepositoryMock();
 
@@ -192,6 +198,7 @@ describe(AssetService.name, () => {
       jobMock,
       configMock,
       storageMock,
+      userMock,
       communicationMock,
       partnerMock,
     );
@@ -836,7 +843,7 @@ describe(AssetService.name, () => {
     });
 
     it('should remove faces', async () => {
-      const assetWithFace = { ...(assetStub.image as AssetEntity), faces: [faceStub.face1, faceStub.mergeFace1] };
+      const assetWithFace = { ...assetStub.image, faces: [faceStub.face1, faceStub.mergeFace1] };
 
       when(assetMock.getById).calledWith(assetWithFace.id).mockResolvedValue(assetWithFace);
 
@@ -863,9 +870,7 @@ describe(AssetService.name, () => {
     });
 
     it('should update stack parent if asset has stack children', async () => {
-      when(assetMock.getById)
-        .calledWith(assetStub.primaryImage.id)
-        .mockResolvedValue(assetStub.primaryImage as AssetEntity);
+      when(assetMock.getById).calledWith(assetStub.primaryImage.id).mockResolvedValue(assetStub.primaryImage);
 
       await sut.handleAssetDeletion({ id: assetStub.primaryImage.id });
 
@@ -878,9 +883,7 @@ describe(AssetService.name, () => {
     });
 
     it('should not schedule delete-files job for readonly assets', async () => {
-      when(assetMock.getById)
-        .calledWith(assetStub.readOnly.id)
-        .mockResolvedValue(assetStub.readOnly as AssetEntity);
+      when(assetMock.getById).calledWith(assetStub.readOnly.id).mockResolvedValue(assetStub.readOnly);
 
       await sut.handleAssetDeletion({ id: assetStub.readOnly.id });
 
@@ -890,21 +893,17 @@ describe(AssetService.name, () => {
     });
 
     it('should not process assets from external library without fromExternal flag', async () => {
-      when(assetMock.getById)
-        .calledWith(assetStub.external.id)
-        .mockResolvedValue(assetStub.external as AssetEntity);
+      when(assetMock.getById).calledWith(assetStub.external.id).mockResolvedValue(assetStub.external);
 
       await sut.handleAssetDeletion({ id: assetStub.external.id });
 
-      expect(jobMock.queue).not.toBeCalled();
-      expect(jobMock.queueAll).not.toBeCalled();
-      expect(assetMock.remove).not.toBeCalled();
+      expect(jobMock.queue).not.toHaveBeenCalled();
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
+      expect(assetMock.remove).not.toHaveBeenCalled();
     });
 
     it('should process assets from external library with fromExternal flag', async () => {
-      when(assetMock.getById)
-        .calledWith(assetStub.external.id)
-        .mockResolvedValue(assetStub.external as AssetEntity);
+      when(assetMock.getById).calledWith(assetStub.external.id).mockResolvedValue(assetStub.external);
 
       await sut.handleAssetDeletion({ id: assetStub.external.id, fromExternal: true });
 
@@ -949,6 +948,13 @@ describe(AssetService.name, () => {
         ],
       ]);
     });
+
+    it('should update usage', async () => {
+      when(assetMock.getById).calledWith(assetStub.image.id).mockResolvedValue(assetStub.image);
+      await sut.handleAssetDeletion({ id: assetStub.image.id });
+
+      expect(userMock.updateUsage).toHaveBeenCalledWith(assetStub.image.ownerId, -5000);
+    });
   });
 
   describe('run', () => {

server/src/domain/asset/asset.service.ts

@@ -20,6 +20,7 @@ import {
   IPartnerRepository,
   IStorageRepository,
   ISystemConfigRepository,
+  IUserRepository,
   ImmichReadStream,
   JobItem,
   TimeBucketOptions,
@@ -75,6 +76,7 @@ export interface UploadFile {
   checksum: Buffer;
   originalPath: string;
   originalName: string;
+  size: number;
 }
 
 export class AssetService {
@@ -89,6 +91,7 @@ export class AssetService {
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
+    @Inject(IUserRepository) private userRepository: IUserRepository,
     @Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository,
     @Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
   ) {
@@ -481,6 +484,7 @@ export class AssetService {
     }
 
     await this.assetRepository.remove(asset);
+    await this.userRepository.updateUsage(asset.ownerId, -(asset.exifInfo?.fileSizeInByte || 0));
     this.communicationRepository.send(ClientEvent.ASSET_DELETE, asset.ownerId, id);
 
     // TODO refactor this to use cascades

server/src/domain/job/job.constants.ts

@@ -37,9 +37,10 @@ export enum JobName {
   METADATA_EXTRACTION = 'metadata-extraction',
   LINK_LIVE_PHOTOS = 'link-live-photos',
 
-  // user deletion
+  // user
   USER_DELETION = 'user-deletion',
   USER_DELETE_CHECK = 'user-delete-check',
+  USER_SYNC_USAGE = 'user-sync-usage',
 
   // asset
   ASSET_DELETION = 'asset-deletion',
@@ -95,6 +96,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.CLEAN_OLD_AUDIT_LOGS]: QueueName.BACKGROUND_TASK,
   [JobName.PERSON_CLEANUP]: QueueName.BACKGROUND_TASK,
   [JobName.PERSON_DELETE]: QueueName.BACKGROUND_TASK,
+  [JobName.USER_SYNC_USAGE]: QueueName.BACKGROUND_TASK,
 
   // conversion
   [JobName.QUEUE_VIDEO_CONVERSION]: QueueName.VIDEO_CONVERSION,

server/src/domain/job/job.service.spec.ts

@@ -60,6 +60,7 @@ describe(JobService.name, () => {
         { name: JobName.PERSON_CLEANUP },
         { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
         { name: JobName.CLEAN_OLD_AUDIT_LOGS },
+        { name: JobName.USER_SYNC_USAGE },
       ]);
     });
   });

server/src/domain/job/job.service.ts

@@ -164,6 +164,7 @@ export class JobService {
       { name: JobName.PERSON_CLEANUP },
       { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
       { name: JobName.CLEAN_OLD_AUDIT_LOGS },
+      { name: JobName.USER_SYNC_USAGE },
     ]);
   }
 

server/src/domain/partner/partner.service.spec.ts

@@ -21,7 +21,9 @@ const responseDto = {
     externalPath: null,
     memoriesEnabled: true,
     avatarColor: UserAvatarColor.PRIMARY,
+    quotaSizeInBytes: null,
     inTimeline: true,
+    quotaUsageInBytes: 0,
   },
   user1: <PartnerResponseDto>{
     email: 'immich@test.com',
@@ -39,6 +41,8 @@ const responseDto = {
     memoriesEnabled: true,
     avatarColor: UserAvatarColor.PRIMARY,
     inTimeline: true,
+    quotaSizeInBytes: null,
+    quotaUsageInBytes: 0,
   },
 };
 

server/src/domain/repositories/job.repository.ts

@@ -39,9 +39,10 @@ export type JobItem =
   | { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IEntityJob }
   | { name: JobName.GENERATE_THUMBHASH_THUMBNAIL; data: IEntityJob }
 
-  // User Deletion
+  // User
   | { name: JobName.USER_DELETE_CHECK; data?: IBaseJob }
   | { name: JobName.USER_DELETION; data: IEntityJob }
+  | { name: JobName.USER_SYNC_USAGE; data?: IBaseJob }
 
   // Storage Template
   | { name: JobName.STORAGE_TEMPLATE_MIGRATION; data?: IBaseJob }

server/src/domain/repositories/user.repository.ts

@@ -10,6 +10,7 @@ export interface UserStatsQueryResponse {
   photos: number;
   videos: number;
   usage: number;
+  quotaSizeInBytes: number | null;
 }
 
 export interface UserFindOptions {
@@ -32,4 +33,6 @@ export interface IUserRepository {
   update(id: string, user: Partial<UserEntity>): Promise<UserEntity>;
   delete(user: UserEntity, hard?: boolean): Promise<UserEntity>;
   restore(user: UserEntity): Promise<UserEntity>;
+  updateUsage(id: string, delta: number): Promise<void>;
+  syncUsage(): Promise<void>;
 }

server/src/domain/server-info/server-info.dto.ts

@@ -45,6 +45,8 @@ export class UsageByUserDto {
   videos!: number;
   @ApiProperty({ type: 'integer', format: 'int64' })
   usage!: number;
+  @ApiProperty({ type: 'integer', format: 'int64' })
+  quotaSizeInBytes!: number | null;
 }
 
 export class ServerStatsResponseDto {

server/src/domain/server-info/server-info.service.spec.ts

@@ -220,6 +220,7 @@ describe(ServerInfoService.name, () => {
           photos: 10,
           videos: 11,
           usage: 12345,
+          quotaSizeInBytes: 0,
         },
         {
           userId: 'user2',
@@ -227,6 +228,7 @@ describe(ServerInfoService.name, () => {
           photos: 10,
           videos: 20,
           usage: 123456,
+          quotaSizeInBytes: 0,
         },
         {
           userId: 'user3',
@@ -234,6 +236,7 @@ describe(ServerInfoService.name, () => {
           photos: 100,
           videos: 0,
           usage: 987654,
+          quotaSizeInBytes: 0,
         },
       ]);
 
@@ -244,6 +247,7 @@ describe(ServerInfoService.name, () => {
         usageByUser: [
           {
             photos: 10,
+            quotaSizeInBytes: 0,
             usage: 12345,
             userName: '1 User',
             userId: 'user1',
@@ -251,6 +255,7 @@ describe(ServerInfoService.name, () => {
           },
           {
             photos: 10,
+            quotaSizeInBytes: 0,
             usage: 123456,
             userName: '2 User',
             userId: 'user2',
@@ -258,6 +263,7 @@ describe(ServerInfoService.name, () => {
           },
           {
             photos: 100,
+            quotaSizeInBytes: 0,
             usage: 987654,
             userName: '3 User',
             userId: 'user3',

server/src/domain/server-info/server-info.service.ts

@@ -118,6 +118,7 @@ export class ServerInfoService {
       usage.photos = user.photos;
       usage.videos = user.videos;
       usage.usage = user.usage;
+      usage.quotaSizeInBytes = user.quotaSizeInBytes;
 
       serverStats.photos += usage.photos;
       serverStats.videos += usage.videos;

server/src/domain/user/dto/create-user.dto.ts

@@ -1,5 +1,6 @@
+import { ApiProperty } from '@nestjs/swagger';
 import { Transform } from 'class-transformer';
-import { IsBoolean, IsEmail, IsNotEmpty, IsString } from 'class-validator';
+import { IsBoolean, IsEmail, IsNotEmpty, IsNumber, IsPositive, IsString } from 'class-validator';
 import { Optional, toEmail, toSanitized } from '../../domain.util';
 
 export class CreateUserDto {
@@ -27,6 +28,12 @@ export class CreateUserDto {
   @Optional()
   @IsBoolean()
   memoriesEnabled?: boolean;
+
+  @Optional({ nullable: true })
+  @IsNumber()
+  @IsPositive()
+  @ApiProperty({ type: 'integer', format: 'int64' })
+  quotaSizeInBytes?: number | null;
 }
 
 export class CreateAdminDto {

server/src/domain/user/dto/update-user.dto.ts

@@ -1,7 +1,7 @@
 import { UserAvatarColor } from '@app/infra/entities';
 import { ApiProperty } from '@nestjs/swagger';
 import { Transform } from 'class-transformer';
-import { IsBoolean, IsEmail, IsEnum, IsNotEmpty, IsString, IsUUID } from 'class-validator';
+import { IsBoolean, IsEmail, IsEnum, IsNotEmpty, IsNumber, IsPositive, IsString, IsUUID } from 'class-validator';
 import { Optional, toEmail, toSanitized } from '../../domain.util';
 
 export class UpdateUserDto {
@@ -50,4 +50,10 @@ export class UpdateUserDto {
   @IsEnum(UserAvatarColor)
   @ApiProperty({ enumName: 'UserAvatarColor', enum: UserAvatarColor })
   avatarColor?: UserAvatarColor;
+
+  @Optional({ nullable: true })
+  @IsNumber()
+  @IsPositive()
+  @ApiProperty({ type: 'integer', format: 'int64' })
+  quotaSizeInBytes?: number | null;
 }

server/src/domain/user/response-dto/user-response.dto.ts

@@ -33,6 +33,10 @@ export class UserResponseDto extends UserDto {
   updatedAt!: Date;
   oauthId!: string;
   memoriesEnabled?: boolean;
+  @ApiProperty({ type: 'integer', format: 'int64' })
+  quotaSizeInBytes!: number | null;
+  @ApiProperty({ type: 'integer', format: 'int64' })
+  quotaUsageInBytes!: number;
 }
 
 export const mapSimpleUser = (entity: UserEntity): UserDto => {
@@ -57,5 +61,7 @@ export function mapUser(entity: UserEntity): UserResponseDto {
     updatedAt: entity.updatedAt,
     oauthId: entity.oauthId,
     memoriesEnabled: entity.memoriesEnabled,
+    quotaSizeInBytes: entity.quotaSizeInBytes,
+    quotaUsageInBytes: entity.quotaUsageInBytes,
   };
 }

server/src/domain/user/user.service.spec.ts

@@ -512,4 +512,11 @@ describe(UserService.name, () => {
       expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/library/admin', options);
     });
   });
+
+  describe('handleUserSyncUsage', () => {
+    it('should sync usage', async () => {
+      await sut.handleUserSyncUsage();
+      expect(userMock.syncUsage).toHaveBeenCalledTimes(1);
+    });
+  });
 });

server/src/domain/user/user.service.ts

@@ -127,6 +127,11 @@ export class UserService {
     return { admin, password, provided: !!providedPassword };
   }
 
+  async handleUserSyncUsage() {
+    await this.userRepository.syncUsage();
+    return true;
+  }
+
   async handleUserDeleteCheck() {
     const users = await this.userRepository.getDeletedUsers();
     await this.jobRepository.queueAll(