feat(server): Automatic watching of library folders (#6192)

* feat: initial watch support

* allow offline files

* chore: ignore query errors when resetting e2e db

* revert db query

* add savepoint

* guard the user query

* chore: openapi and db migration

* wip

* support multiple libraries

* fix tests

* wip

* can now cleanup chokidar watchers

* fix unit tests

* add library watch queue

* add missing init from merge

* wip

* can now filter file extensions

* remove watch api from non job client

* Fix e2e test

* watch library with updated import path and exclusion pattern

* add library watch frontend ui

* case sensitive watching extensions

* can auto watch libraries

* move watcher e2e tests to separate file

* don't watch libraries from a queue

* use event emitters

* shorten e2e test timeout

* refactor chokidar code to filesystem provider

* expose chokidar parameters to config file

* fix storage mock

* set default config for library watching

* add fs provider mocks

* cleanup

* add more unit tests for watcher

* chore: fix format + sql

* add more tests

* move unwatch feature back to library service

* add file event unit tests

* chore: formatting

* add documentation

* fix e2e tests

* chore: fix e2e tests

* fix library updating

* test cleanup

* fix typo

* cleanup

* fixing as per pr comments

* reduce library watch config file

* update storage config and mocks

* move negative event tests to unit tests

* fix library watcher e2e

* make watch configuration global

* remove the feature flag

* refactor watcher teardown

* fix microservices init

* centralize asset scan job queue

* improve docs

* add more tests

* chore: open api

* initialize app service

* fix docs

* fix library watch feature flag

* Update docs/docs/features/libraries.md

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* fix: import right app service

* don't be truthy

* fix test speling

* stricter library update tests

* move fs watcher mock to external file

* subscribe to config changes

* docker does not need polling

* make library watch() private

* feat: add configuration ui

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/test/fixtures/library.stub.ts

@@ -30,4 +30,74 @@ export const libraryStub = {
     isVisible: true,
     exclusionPatterns: [],
   }),
+  externalLibrary2: Object.freeze<LibraryEntity>({
+    id: 'library-id2',
+    name: 'test_library2',
+    assets: [],
+    owner: userStub.externalPath1,
+    ownerId: 'user-id',
+    type: LibraryType.EXTERNAL,
+    importPaths: [],
+    createdAt: new Date('2021-01-01'),
+    updatedAt: new Date('2022-01-01'),
+    refreshedAt: null,
+    isVisible: true,
+    exclusionPatterns: [],
+  }),
+  externalLibraryWithImportPaths1: Object.freeze<LibraryEntity>({
+    id: 'library-id-with-paths1',
+    name: 'library-with-import-paths1',
+    assets: [],
+    owner: userStub.externalPath1,
+    ownerId: 'user-id',
+    type: LibraryType.EXTERNAL,
+    importPaths: ['/foo', '/bar'],
+    createdAt: new Date('2023-01-01'),
+    updatedAt: new Date('2023-01-01'),
+    refreshedAt: null,
+    isVisible: true,
+    exclusionPatterns: [],
+  }),
+  externalLibraryWithImportPaths2: Object.freeze<LibraryEntity>({
+    id: 'library-id-with-paths2',
+    name: 'library-with-import-paths2',
+    assets: [],
+    owner: userStub.externalPath1,
+    ownerId: 'user-id',
+    type: LibraryType.EXTERNAL,
+    importPaths: ['/xyz', '/asdf'],
+    createdAt: new Date('2023-01-01'),
+    updatedAt: new Date('2023-01-01'),
+    refreshedAt: null,
+    isVisible: true,
+    exclusionPatterns: [],
+  }),
+  externalLibraryWithExclusionPattern: Object.freeze<LibraryEntity>({
+    id: 'library-id',
+    name: 'test_library',
+    assets: [],
+    owner: userStub.externalPath1,
+    ownerId: 'user-id',
+    type: LibraryType.EXTERNAL,
+    importPaths: [],
+    createdAt: new Date('2023-01-01'),
+    updatedAt: new Date('2023-01-01'),
+    refreshedAt: null,
+    isVisible: true,
+    exclusionPatterns: ['**/dir1/**'],
+  }),
+  patternPath: Object.freeze<LibraryEntity>({
+    id: 'library-id1337',
+    name: 'importpath-exclusion-library1',
+    assets: [],
+    owner: userStub.externalPath1,
+    ownerId: 'user-id',
+    type: LibraryType.EXTERNAL,
+    importPaths: ['/xyz', '/asdf'],
+    createdAt: new Date('2023-01-01'),
+    updatedAt: new Date('2023-01-01'),
+    refreshedAt: null,
+    isVisible: true,
+    exclusionPatterns: ['**/dir1/**'],
+  }),
 };

server/test/fixtures/system-config.stub.ts

@@ -22,4 +22,6 @@ export const systemConfigStub: Record<string, SystemConfigEntity[]> = {
     { key: SystemConfigKey.OAUTH_MOBILE_REDIRECT_URI, value: 'http://mobile-redirect' },
     { key: SystemConfigKey.OAUTH_BUTTON_TEXT, value: 'OAuth' },
   ],
+  libraryWatchEnabled: [{ key: SystemConfigKey.LIBRARY_WATCH_ENABLED, value: true }],
+  libraryWatchDisabled: [{ key: SystemConfigKey.LIBRARY_WATCH_ENABLED, value: false }],
 };

server/test/mocks/fswatcher.mock.ts

@@ -0,0 +1,24 @@
+export const newFSWatcherMock = () => {
+  return {
+    options: {},
+    on: jest.fn(),
+    add: jest.fn(),
+    unwatch: jest.fn(),
+    getWatched: jest.fn(),
+    close: jest.fn(),
+    addListener: jest.fn(),
+    removeListener: jest.fn(),
+    removeAllListeners: jest.fn(),
+    eventNames: jest.fn(),
+    rawListeners: jest.fn(),
+    listeners: jest.fn(),
+    emit: jest.fn(),
+    listenerCount: jest.fn(),
+    off: jest.fn(),
+    once: jest.fn(),
+    prependListener: jest.fn(),
+    prependOnceListener: jest.fn(),
+    setMaxListeners: jest.fn(),
+    getMaxListeners: jest.fn(),
+  };
+};

server/test/mocks/index.ts

@@ -0,0 +1 @@
+export * from './fswatcher.mock';

server/test/repositories/storage.repository.mock.ts

@@ -21,5 +21,6 @@ export const newStorageRepositoryMock = (reset = true): jest.Mocked<IStorageRepo
     crawl: jest.fn(),
     rename: jest.fn(),
     copyFile: jest.fn(),
+    watch: jest.fn(),
   };
 };