chore(server): Store generated files (thumbnails, encoded video) in subdirectories (#4112)

* save thumbnails in subdirectories

* migration job, migrate assets and face thumbnails

* fix tests

* directory depth of two instead of three

* cleanup empty dirs after migration

* clean up empty dirs after migration, migrate people without assetId

* add job card for new migration job

* fix removeEmptyDirs race condition because of missing await

* cleanup empty directories after asset deletion

* move ensurePath to storage core

* rename jobs

* remove unnecessary property of IEntityJob

* use updated person getById, minor refactoring

* ensure that directory cleanup doesn't interfere with migration

* better description for job in ui

* fix remove directories when migration is done

* cleanup empty folders at start of migration

* fix: actually persist concurrency setting

* add comment explaining regex

* chore: cleanup

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

server/src/domain/job/job.constants.ts

@@ -7,6 +7,7 @@ export enum QueueName {
   CLIP_ENCODING = 'clipEncoding',
   BACKGROUND_TASK = 'backgroundTask',
   STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
+  MIGRATION = 'migration',
   SEARCH = 'search',
   SIDECAR = 'sidecar',
   LIBRARY = 'library',
@@ -45,6 +46,11 @@ export enum JobName {
   STORAGE_TEMPLATE_MIGRATION_SINGLE = 'storage-template-migration-single',
   SYSTEM_CONFIG_CHANGE = 'system-config-change',
 
+  // migration
+  QUEUE_MIGRATION = 'queue-migration',
+  MIGRATE_ASSET = 'migrate-asset',
+  MIGRATE_PERSON = 'migrate-person',
+
   // object tagging
   QUEUE_OBJECT_TAGGING = 'queue-object-tagging',
   CLASSIFY_IMAGE = 'classify-image',
@@ -119,6 +125,11 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: QueueName.STORAGE_TEMPLATE_MIGRATION,
   [JobName.SYSTEM_CONFIG_CHANGE]: QueueName.STORAGE_TEMPLATE_MIGRATION,
 
+  // migration
+  [JobName.QUEUE_MIGRATION]: QueueName.MIGRATION,
+  [JobName.MIGRATE_ASSET]: QueueName.MIGRATION,
+  [JobName.MIGRATE_PERSON]: QueueName.MIGRATION,
+
   // object tagging
   [JobName.QUEUE_OBJECT_TAGGING]: QueueName.OBJECT_TAGGING,
   [JobName.CLASSIFY_IMAGE]: QueueName.OBJECT_TAGGING,

server/src/domain/job/job.dto.ts

@@ -68,6 +68,9 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
   @ApiProperty({ type: JobStatusDto })
   [QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobStatusDto;
 
+  @ApiProperty({ type: JobStatusDto })
+  [QueueName.MIGRATION]!: JobStatusDto;
+
   @ApiProperty({ type: JobStatusDto })
   [QueueName.BACKGROUND_TASK]!: JobStatusDto;
 

server/src/domain/job/job.repository.ts

@@ -46,6 +46,11 @@ export type JobItem =
   | { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IEntityJob }
   | { name: JobName.SYSTEM_CONFIG_CHANGE; data?: IBaseJob }
 
+  // Migration
+  | { name: JobName.QUEUE_MIGRATION; data?: IBaseJob }
+  | { name: JobName.MIGRATE_ASSET; data?: IEntityJob }
+  | { name: JobName.MIGRATE_PERSON; data?: IEntityJob }
+
   // Metadata Extraction
   | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob }
   | { name: JobName.METADATA_EXTRACTION; data: IEntityJob }

server/src/domain/job/job.service.spec.ts

@@ -94,6 +94,7 @@ describe(JobService.name, () => {
         [QueueName.OBJECT_TAGGING]: expectedJobStatus,
         [QueueName.SEARCH]: expectedJobStatus,
         [QueueName.STORAGE_TEMPLATE_MIGRATION]: expectedJobStatus,
+        [QueueName.MIGRATION]: expectedJobStatus,
         [QueueName.THUMBNAIL_GENERATION]: expectedJobStatus,
         [QueueName.VIDEO_CONVERSION]: expectedJobStatus,
         [QueueName.RECOGNIZE_FACES]: expectedJobStatus,
@@ -229,6 +230,7 @@ describe(JobService.name, () => {
           [QueueName.SIDECAR]: { concurrency: 10 },
           [QueueName.LIBRARY]: { concurrency: 10 },
           [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
+          [QueueName.MIGRATION]: { concurrency: 10 },
           [QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
           [QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
         },
@@ -242,6 +244,7 @@ describe(JobService.name, () => {
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
+      expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.MIGRATION, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);
     });

server/src/domain/job/job.service.ts

@@ -76,6 +76,9 @@ export class JobService {
       case QueueName.STORAGE_TEMPLATE_MIGRATION:
         return this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
 
+      case QueueName.MIGRATION:
+        return this.jobRepository.queue({ name: JobName.QUEUE_MIGRATION });
+
       case QueueName.OBJECT_TAGGING:
         await this.configCore.requireFeature(FeatureFlag.TAG_IMAGE);
         return this.jobRepository.queue({ name: JobName.QUEUE_OBJECT_TAGGING, data: { force } });