chore(server): Store generated files (thumbnails, encoded video) in subdirectories (#4112)

* save thumbnails in subdirectories

* migration job, migrate assets and face thumbnails

* fix tests

* directory depth of two instead of three

* cleanup empty dirs after migration

* clean up empty dirs after migration, migrate people without assetId

* add job card for new migration job

* fix removeEmptyDirs race condition because of missing await

* cleanup empty directories after asset deletion

* move ensurePath to storage core

* rename jobs

* remove unnecessary property of IEntityJob

* use updated person getById, minor refactoring

* ensure that directory cleanup doesn't interfere with migration

* better description for job in ui

* fix remove directories when migration is done

* cleanup empty folders at start of migration

* fix: actually persist concurrency setting

* add comment explaining regex

* chore: cleanup

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

server/src/domain/storage/storage.core.ts

@@ -1,5 +1,6 @@
 import { join } from 'node:path';
 import { APP_MEDIA_LOCATION } from '../domain.constant';
+import { IStorageRepository } from './storage.repository';
 
 export enum StorageFolder {
   ENCODED_VIDEO = 'encoded-video',
@@ -10,6 +11,8 @@ export enum StorageFolder {
 }
 
 export class StorageCore {
+  constructor(private repository: IStorageRepository) {}
+
   getFolderLocation(
     folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS,
     userId: string,
@@ -24,4 +27,22 @@ export class StorageCore {
   getBaseFolder(folder: StorageFolder) {
     return join(APP_MEDIA_LOCATION, folder);
   }
+
+  ensurePath(
+    folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS,
+    ownerId: string,
+    fileName: string,
+  ): string {
+    const folderPath = join(
+      this.getFolderLocation(folder, ownerId),
+      fileName.substring(0, 2),
+      fileName.substring(2, 4),
+    );
+    this.repository.mkdirSync(folderPath);
+    return join(folderPath, fileName);
+  }
+
+  removeEmptyDirs(folder: StorageFolder) {
+    return this.repository.removeEmptyDirs(this.getBaseFolder(folder));
+  }
 }

server/src/domain/storage/storage.repository.ts

@@ -26,7 +26,7 @@ export interface IStorageRepository {
   createReadStream(filepath: string, mimeType?: string | null): Promise<ImmichReadStream>;
   unlink(filepath: string): Promise<void>;
   unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise<void>;
-  removeEmptyDirs(folder: string): Promise<void>;
+  removeEmptyDirs(folder: string, self?: boolean): Promise<void>;
   moveFile(source: string, target: string): Promise<void>;
   checkFileExists(filepath: string, mode?: number): Promise<boolean>;
   mkdirSync(filepath: string): void;

server/src/domain/storage/storage.service.ts

@@ -6,9 +6,11 @@ import { IStorageRepository } from './storage.repository';
 @Injectable()
 export class StorageService {
   private logger = new Logger(StorageService.name);
-  private storageCore = new StorageCore();
+  private storageCore: StorageCore;
 
-  constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) {}
+  constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) {
+    this.storageCore = new StorageCore(storageRepository);
+  }
 
   init() {
     const libraryBase = this.storageCore.getBaseFolder(StorageFolder.LIBRARY);