feat(server): custom library scanning interval (#4390)

* add automatic library scan config options

* add validation

* open api

* use CronJob instead of cron-validator

* fix tests

* catch potential error of the library scan initialization

* better description for input field

* move library scan job initialization to server app service

* fix tests

* add comments to all parameters of cronjob contructor

* make scan a child of a more general library object

* open api

* chore: cleanup

* move cronjob handling to job repoistory

* web: select for common cron expressions

* fix open api

* fix tests

* put scanning settings in nested accordion

* fix system config validation

* refactor, tests

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

server/src/domain/library/library.service.spec.ts

@@ -1,4 +1,4 @@
-import { AssetType, LibraryType, UserEntity } from '@app/infra/entities';
+import { AssetType, LibraryType, SystemConfig, SystemConfigKey, UserEntity } from '@app/infra/entities';
 import { BadRequestException } from '@nestjs/common';
 
 import {
@@ -12,6 +12,7 @@ import {
   newJobRepositoryMock,
   newLibraryRepositoryMock,
   newStorageRepositoryMock,
+  newSystemConfigRepositoryMock,
   newUserRepositoryMock,
   userStub,
 } from '@test';
@@ -23,8 +24,10 @@ import {
   IJobRepository,
   ILibraryRepository,
   IStorageRepository,
+  ISystemConfigRepository,
   IUserRepository,
 } from '../repositories';
+import { SystemConfigCore } from '../system-config/system-config.core';
 import { LibraryService } from './library.service';
 
 describe(LibraryService.name, () => {
@@ -32,6 +35,7 @@ describe(LibraryService.name, () => {
 
   let accessMock: IAccessRepositoryMock;
   let assetMock: jest.Mocked<IAssetRepository>;
+  let configMock: jest.Mocked<ISystemConfigRepository>;
   let cryptoMock: jest.Mocked<ICryptoRepository>;
   let userMock: jest.Mocked<IUserRepository>;
   let jobMock: jest.Mocked<IJobRepository>;
@@ -40,6 +44,7 @@ describe(LibraryService.name, () => {
 
   beforeEach(() => {
     accessMock = newAccessRepositoryMock();
+    configMock = newSystemConfigRepositoryMock();
     libraryMock = newLibraryRepositoryMock();
     userMock = newUserRepositoryMock();
     assetMock = newAssetRepositoryMock();
@@ -55,13 +60,46 @@ describe(LibraryService.name, () => {
 
     accessMock.library.hasOwnerAccess.mockResolvedValue(true);
 
-    sut = new LibraryService(accessMock, assetMock, cryptoMock, jobMock, libraryMock, storageMock, userMock);
+    sut = new LibraryService(
+      accessMock,
+      assetMock,
+      configMock,
+      cryptoMock,
+      jobMock,
+      libraryMock,
+      storageMock,
+      userMock,
+    );
   });
 
   it('should work', () => {
     expect(sut).toBeDefined();
   });
 
+  describe('init', () => {
+    it('should init cron job and subscribe to config changes', async () => {
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.LIBRARY_SCAN_ENABLED, value: true },
+        { key: SystemConfigKey.LIBRARY_SCAN_CRON_EXPRESSION, value: '0 0 * * *' },
+      ]);
+
+      await sut.init();
+      expect(configMock.load).toHaveBeenCalled();
+      expect(jobMock.addCronJob).toHaveBeenCalled();
+
+      SystemConfigCore.create(newSystemConfigRepositoryMock(false)).config$.next({
+        library: {
+          scan: {
+            enabled: true,
+            cronExpression: '0 1 * * *',
+          },
+        },
+      } as SystemConfig);
+
+      expect(jobMock.updateCronJob).toHaveBeenCalledWith('libraryScan', '0 1 * * *', true);
+    });
+  });
+
   describe('handleQueueAssetRefresh', () => {
     it("should not queue assets outside of user's external path", async () => {
       const mockLibraryJob: ILibraryRefreshJob = {

server/src/domain/library/library.service.ts

@@ -7,7 +7,7 @@ import { basename, parse } from 'path';
 import { AccessCore, Permission } from '../access';
 import { AuthUserDto } from '../auth';
 import { mimeTypes } from '../domain.constant';
-import { usePagination } from '../domain.util';
+import { usePagination, validateCronExpression } from '../domain.util';
 import { IBaseJob, IEntityJob, ILibraryFileJob, ILibraryRefreshJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
 
 import {
@@ -17,9 +17,11 @@ import {
   IJobRepository,
   ILibraryRepository,
   IStorageRepository,
+  ISystemConfigRepository,
   IUserRepository,
   WithProperty,
 } from '../repositories';
+import { SystemConfigCore } from '../system-config';
 import {
   CreateLibraryDto,
   LibraryResponseDto,
@@ -33,10 +35,12 @@ import {
 export class LibraryService {
   readonly logger = new Logger(LibraryService.name);
   private access: AccessCore;
+  private configCore: SystemConfigCore;
 
   constructor(
     @Inject(IAccessRepository) accessRepository: IAccessRepository,
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
     @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(ILibraryRepository) private repository: ILibraryRepository,
@@ -44,6 +48,26 @@ export class LibraryService {
     @Inject(IUserRepository) private userRepository: IUserRepository,
   ) {
     this.access = AccessCore.create(accessRepository);
+    this.configCore = SystemConfigCore.create(configRepository);
+    this.configCore.addValidator((config) => {
+      if (!validateCronExpression(config.library.scan.cronExpression)) {
+        throw new Error(`Invalid cron expression ${config.library.scan.cronExpression}`);
+      }
+    });
+  }
+
+  async init() {
+    const config = await this.configCore.getConfig();
+    this.jobRepository.addCronJob(
+      'libraryScan',
+      config.library.scan.cronExpression,
+      () => this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }),
+      config.library.scan.enabled,
+    );
+
+    this.configCore.config$.subscribe((config) => {
+      this.jobRepository.updateCronJob('libraryScan', config.library.scan.cronExpression, config.library.scan.enabled);
+    });
   }
 
   async getStatistics(authUser: AuthUserDto, id: string): Promise<LibraryStatsResponseDto> {