068e703e88
* feat: initial watch support * allow offline files * chore: ignore query errors when resetting e2e db * revert db query * add savepoint * guard the user query * chore: openapi and db migration * wip * support multiple libraries * fix tests * wip * can now cleanup chokidar watchers * fix unit tests * add library watch queue * add missing init from merge * wip * can now filter file extensions * remove watch api from non job client * Fix e2e test * watch library with updated import path and exclusion pattern * add library watch frontend ui * case sensitive watching extensions * can auto watch libraries * move watcher e2e tests to separate file * don't watch libraries from a queue * use event emitters * shorten e2e test timeout * refactor chokidar code to filesystem provider * expose chokidar parameters to config file * fix storage mock * set default config for library watching * add fs provider mocks * cleanup * add more unit tests for watcher * chore: fix format + sql * add more tests * move unwatch feature back to library service * add file event unit tests * chore: formatting * add documentation * fix e2e tests * chore: fix e2e tests * fix library updating * test cleanup * fix typo * cleanup * fixing as per pr comments * reduce library watch config file * update storage config and mocks * move negative event tests to unit tests * fix library watcher e2e * make watch configuration global * remove the feature flag * refactor watcher teardown * fix microservices init * centralize asset scan job queue * improve docs * add more tests * chore: open api * initialize app service * fix docs * fix library watch feature flag * Update docs/docs/features/libraries.md Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> * fix: import right app service * don't be truthy * fix test speling * stricter library update tests * move fs watcher mock to external file * subscribe to config changes * docker does not need polling * make library watch() private * feat: add configuration ui --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
208 lines
4.6 KiB
TypeScript
208 lines
4.6 KiB
TypeScript
import { CrawlOptionsDto } from '@app/domain';
|
|
import mockfs from 'mock-fs';
|
|
import { FilesystemProvider } from './filesystem.provider';
|
|
|
|
interface Test {
|
|
test: string;
|
|
options: CrawlOptionsDto;
|
|
files: Record<string, boolean>;
|
|
}
|
|
|
|
const cwd = process.cwd();
|
|
|
|
const tests: Test[] = [
|
|
{
|
|
test: 'should return empty when crawling an empty path list',
|
|
options: {
|
|
pathsToCrawl: [],
|
|
},
|
|
files: {},
|
|
},
|
|
{
|
|
test: 'should crawl a single path',
|
|
options: {
|
|
pathsToCrawl: ['/photos/'],
|
|
},
|
|
files: {
|
|
'/photos/image.jpg': true,
|
|
},
|
|
},
|
|
{
|
|
test: 'should exclude by file extension',
|
|
options: {
|
|
pathsToCrawl: ['/photos/'],
|
|
exclusionPatterns: ['**/*.tif'],
|
|
},
|
|
files: {
|
|
'/photos/image.jpg': true,
|
|
'/photos/image.tif': false,
|
|
},
|
|
},
|
|
{
|
|
test: 'should exclude by file extension without case sensitivity',
|
|
options: {
|
|
pathsToCrawl: ['/photos/'],
|
|
exclusionPatterns: ['**/*.TIF'],
|
|
},
|
|
files: {
|
|
'/photos/image.jpg': true,
|
|
'/photos/image.tif': false,
|
|
},
|
|
},
|
|
{
|
|
test: 'should exclude by folder',
|
|
options: {
|
|
pathsToCrawl: ['/photos/'],
|
|
exclusionPatterns: ['**/raw/**'],
|
|
},
|
|
files: {
|
|
'/photos/image.jpg': true,
|
|
'/photos/raw/image.jpg': false,
|
|
'/photos/raw2/image.jpg': true,
|
|
'/photos/folder/raw/image.jpg': false,
|
|
'/photos/crawl/image.jpg': true,
|
|
},
|
|
},
|
|
{
|
|
test: 'should crawl multiple paths',
|
|
options: {
|
|
pathsToCrawl: ['/photos/', '/images/', '/albums/'],
|
|
},
|
|
files: {
|
|
'/photos/image1.jpg': true,
|
|
'/images/image2.jpg': true,
|
|
'/albums/image3.jpg': true,
|
|
},
|
|
},
|
|
{
|
|
test: 'should support globbing paths',
|
|
options: {
|
|
pathsToCrawl: ['/photos*'],
|
|
},
|
|
files: {
|
|
'/photos1/image1.jpg': true,
|
|
'/photos2/image2.jpg': true,
|
|
'/images/image3.jpg': false,
|
|
},
|
|
},
|
|
{
|
|
test: 'should crawl a single path without trailing slash',
|
|
options: {
|
|
pathsToCrawl: ['/photos'],
|
|
},
|
|
files: {
|
|
'/photos/image.jpg': true,
|
|
},
|
|
},
|
|
{
|
|
test: 'should crawl a single path',
|
|
options: {
|
|
pathsToCrawl: ['/photos/'],
|
|
},
|
|
files: {
|
|
'/photos/image.jpg': true,
|
|
'/photos/subfolder/image1.jpg': true,
|
|
'/photos/subfolder/image2.jpg': true,
|
|
'/image1.jpg': false,
|
|
},
|
|
},
|
|
{
|
|
test: 'should filter file extensions',
|
|
options: {
|
|
pathsToCrawl: ['/photos/'],
|
|
},
|
|
files: {
|
|
'/photos/image.jpg': true,
|
|
'/photos/image.txt': false,
|
|
'/photos/1': false,
|
|
},
|
|
},
|
|
{
|
|
test: 'should include photo and video extensions',
|
|
options: {
|
|
pathsToCrawl: ['/photos/', '/videos/'],
|
|
},
|
|
files: {
|
|
'/photos/image.jpg': true,
|
|
'/photos/image.jpeg': true,
|
|
'/photos/image.heic': true,
|
|
'/photos/image.heif': true,
|
|
'/photos/image.png': true,
|
|
'/photos/image.gif': true,
|
|
'/photos/image.tif': true,
|
|
'/photos/image.tiff': true,
|
|
'/photos/image.webp': true,
|
|
'/photos/image.dng': true,
|
|
'/photos/image.nef': true,
|
|
'/videos/video.mp4': true,
|
|
'/videos/video.mov': true,
|
|
'/videos/video.webm': true,
|
|
},
|
|
},
|
|
{
|
|
test: 'should check file extensions without case sensitivity',
|
|
options: {
|
|
pathsToCrawl: ['/photos/'],
|
|
},
|
|
files: {
|
|
'/photos/image.jpg': true,
|
|
'/photos/image.Jpg': true,
|
|
'/photos/image.jpG': true,
|
|
'/photos/image.JPG': true,
|
|
'/photos/image.jpEg': true,
|
|
'/photos/image.TIFF': true,
|
|
'/photos/image.tif': true,
|
|
'/photos/image.dng': true,
|
|
'/photos/image.NEF': true,
|
|
},
|
|
},
|
|
{
|
|
test: 'should normalize the path',
|
|
options: {
|
|
pathsToCrawl: ['/photos/1/../2'],
|
|
},
|
|
files: {
|
|
'/photos/1/image.jpg': false,
|
|
'/photos/2/image.jpg': true,
|
|
},
|
|
},
|
|
{
|
|
test: 'should return absolute paths',
|
|
options: {
|
|
pathsToCrawl: ['photos'],
|
|
},
|
|
files: {
|
|
[`${cwd}/photos/1.jpg`]: true,
|
|
[`${cwd}/photos/2.jpg`]: true,
|
|
[`/photos/3.jpg`]: false,
|
|
},
|
|
},
|
|
];
|
|
|
|
describe(FilesystemProvider.name, () => {
|
|
let sut: FilesystemProvider;
|
|
|
|
beforeEach(() => {
|
|
sut = new FilesystemProvider();
|
|
});
|
|
|
|
afterEach(() => {
|
|
mockfs.restore();
|
|
});
|
|
|
|
describe('crawl', () => {
|
|
for (const { test, options, files } of tests) {
|
|
it(test, async () => {
|
|
mockfs(Object.fromEntries(Object.keys(files).map((file) => [file, ''])));
|
|
|
|
const actual = await sut.crawl(options);
|
|
const expected = Object.entries(files)
|
|
.filter((entry) => entry[1])
|
|
.map(([file]) => file);
|
|
|
|
expect(actual.sort()).toEqual(expected.sort());
|
|
});
|
|
}
|
|
});
|
|
});
|