Files
immich/server/src/infra/repositories/filesystem.provider.spec.ts
T
Jonathan Jogenfors 068e703e88 feat(server): Automatic watching of library folders (#6192)
* feat: initial watch support

* allow offline files

* chore: ignore query errors when resetting e2e db

* revert db query

* add savepoint

* guard the user query

* chore: openapi and db migration

* wip

* support multiple libraries

* fix tests

* wip

* can now cleanup chokidar watchers

* fix unit tests

* add library watch queue

* add missing init from merge

* wip

* can now filter file extensions

* remove watch api from non job client

* Fix e2e test

* watch library with updated import path and exclusion pattern

* add library watch frontend ui

* case sensitive watching extensions

* can auto watch libraries

* move watcher e2e tests to separate file

* don't watch libraries from a queue

* use event emitters

* shorten e2e test timeout

* refactor chokidar code to filesystem provider

* expose chokidar parameters to config file

* fix storage mock

* set default config for library watching

* add fs provider mocks

* cleanup

* add more unit tests for watcher

* chore: fix format + sql

* add more tests

* move unwatch feature back to library service

* add file event unit tests

* chore: formatting

* add documentation

* fix e2e tests

* chore: fix e2e tests

* fix library updating

* test cleanup

* fix typo

* cleanup

* fixing as per pr comments

* reduce library watch config file

* update storage config and mocks

* move negative event tests to unit tests

* fix library watcher e2e

* make watch configuration global

* remove the feature flag

* refactor watcher teardown

* fix microservices init

* centralize asset scan job queue

* improve docs

* add more tests

* chore: open api

* initialize app service

* fix docs

* fix library watch feature flag

* Update docs/docs/features/libraries.md

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* fix: import right app service

* don't be truthy

* fix test speling

* stricter library update tests

* move fs watcher mock to external file

* subscribe to config changes

* docker does not need polling

* make library watch() private

* feat: add configuration ui

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-31 08:15:54 +00:00

208 lines
4.6 KiB
TypeScript

import { CrawlOptionsDto } from '@app/domain';
import mockfs from 'mock-fs';
import { FilesystemProvider } from './filesystem.provider';
interface Test {
test: string;
options: CrawlOptionsDto;
files: Record<string, boolean>;
}
const cwd = process.cwd();
const tests: Test[] = [
{
test: 'should return empty when crawling an empty path list',
options: {
pathsToCrawl: [],
},
files: {},
},
{
test: 'should crawl a single path',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
},
},
{
test: 'should exclude by file extension',
options: {
pathsToCrawl: ['/photos/'],
exclusionPatterns: ['**/*.tif'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.tif': false,
},
},
{
test: 'should exclude by file extension without case sensitivity',
options: {
pathsToCrawl: ['/photos/'],
exclusionPatterns: ['**/*.TIF'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.tif': false,
},
},
{
test: 'should exclude by folder',
options: {
pathsToCrawl: ['/photos/'],
exclusionPatterns: ['**/raw/**'],
},
files: {
'/photos/image.jpg': true,
'/photos/raw/image.jpg': false,
'/photos/raw2/image.jpg': true,
'/photos/folder/raw/image.jpg': false,
'/photos/crawl/image.jpg': true,
},
},
{
test: 'should crawl multiple paths',
options: {
pathsToCrawl: ['/photos/', '/images/', '/albums/'],
},
files: {
'/photos/image1.jpg': true,
'/images/image2.jpg': true,
'/albums/image3.jpg': true,
},
},
{
test: 'should support globbing paths',
options: {
pathsToCrawl: ['/photos*'],
},
files: {
'/photos1/image1.jpg': true,
'/photos2/image2.jpg': true,
'/images/image3.jpg': false,
},
},
{
test: 'should crawl a single path without trailing slash',
options: {
pathsToCrawl: ['/photos'],
},
files: {
'/photos/image.jpg': true,
},
},
{
test: 'should crawl a single path',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/subfolder/image1.jpg': true,
'/photos/subfolder/image2.jpg': true,
'/image1.jpg': false,
},
},
{
test: 'should filter file extensions',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.txt': false,
'/photos/1': false,
},
},
{
test: 'should include photo and video extensions',
options: {
pathsToCrawl: ['/photos/', '/videos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.jpeg': true,
'/photos/image.heic': true,
'/photos/image.heif': true,
'/photos/image.png': true,
'/photos/image.gif': true,
'/photos/image.tif': true,
'/photos/image.tiff': true,
'/photos/image.webp': true,
'/photos/image.dng': true,
'/photos/image.nef': true,
'/videos/video.mp4': true,
'/videos/video.mov': true,
'/videos/video.webm': true,
},
},
{
test: 'should check file extensions without case sensitivity',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.Jpg': true,
'/photos/image.jpG': true,
'/photos/image.JPG': true,
'/photos/image.jpEg': true,
'/photos/image.TIFF': true,
'/photos/image.tif': true,
'/photos/image.dng': true,
'/photos/image.NEF': true,
},
},
{
test: 'should normalize the path',
options: {
pathsToCrawl: ['/photos/1/../2'],
},
files: {
'/photos/1/image.jpg': false,
'/photos/2/image.jpg': true,
},
},
{
test: 'should return absolute paths',
options: {
pathsToCrawl: ['photos'],
},
files: {
[`${cwd}/photos/1.jpg`]: true,
[`${cwd}/photos/2.jpg`]: true,
[`/photos/3.jpg`]: false,
},
},
];
describe(FilesystemProvider.name, () => {
let sut: FilesystemProvider;
beforeEach(() => {
sut = new FilesystemProvider();
});
afterEach(() => {
mockfs.restore();
});
describe('crawl', () => {
for (const { test, options, files } of tests) {
it(test, async () => {
mockfs(Object.fromEntries(Object.keys(files).map((file) => [file, ''])));
const actual = await sut.crawl(options);
const expected = Object.entries(files)
.filter((entry) => entry[1])
.map(([file]) => file);
expect(actual.sort()).toEqual(expected.sort());
});
}
});
});