cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): Automatic watching of library folders (#6192)

Browse Source 
* feat: initial watch support

* allow offline files

* chore: ignore query errors when resetting e2e db

* revert db query

* add savepoint

* guard the user query

* chore: openapi and db migration

* wip

* support multiple libraries

* fix tests

* wip

* can now cleanup chokidar watchers

* fix unit tests

* add library watch queue

* add missing init from merge

* wip

* can now filter file extensions

* remove watch api from non job client

* Fix e2e test

* watch library with updated import path and exclusion pattern

* add library watch frontend ui

* case sensitive watching extensions

* can auto watch libraries

* move watcher e2e tests to separate file

* don't watch libraries from a queue

* use event emitters

* shorten e2e test timeout

* refactor chokidar code to filesystem provider

* expose chokidar parameters to config file

* fix storage mock

* set default config for library watching

* add fs provider mocks

* cleanup

* add more unit tests for watcher

* chore: fix format + sql

* add more tests

* move unwatch feature back to library service

* add file event unit tests

* chore: formatting

* add documentation

* fix e2e tests

* chore: fix e2e tests

* fix library updating

* test cleanup

* fix typo

* cleanup

* fixing as per pr comments

* reduce library watch config file

* update storage config and mocks

* move negative event tests to unit tests

* fix library watcher e2e

* make watch configuration global

* remove the feature flag

* refactor watcher teardown

* fix microservices init

* centralize asset scan job queue

* improve docs

* add more tests

* chore: open api

* initialize app service

* fix docs

* fix library watch feature flag

* Update docs/docs/features/libraries.md

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* fix: import right app service

* don't be truthy

* fix test speling

* stricter library update tests

* move fs watcher mock to external file

* subscribe to config changes

* docker does not need polling

* make library watch() private

* feat: add configuration ui

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Jonathan Jogenfors
2024-01-31 09:15:54 +01:00
committed by GitHub
parent 4079e92bbf
commit 068e703e88
48 changed files with 1613 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/e2e/api/jest-e2e.json
View File

@@ -5,7 +5,6 @@
"globalSetup": "<rootDir>/e2e/api/setup.ts",
"testEnvironment": "node",
"testMatch": ["**/e2e/api/specs/*.e2e-spec.[tj]s"],
"testTimeout": 60000,
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
},

16
server/e2e/client/library-api.ts
View File

@@ -1,4 +1,10 @@
import { CreateLibraryDto, LibraryResponseDto, LibraryStatsResponseDto, ScanLibraryDto } from '@app/domain';
import {
CreateLibraryDto,
LibraryResponseDto,
LibraryStatsResponseDto,
ScanLibraryDto,
UpdateLibraryDto,
} from '@app/domain';
import request from 'supertest';
export const libraryApi = {
@@ -44,4 +50,12 @@ export const libraryApi = {
expect(status).toBe(200);
return body;
},
update: async (server: any, accessToken: string, id: string, data: UpdateLibraryDto) => {
const { body, status } = await request(server)
.put(`/library/${id}`)
.set('Authorization', `Bearer ${accessToken}`)
.send(data);
expect(status).toBe(200);
return body as LibraryResponseDto;
},
};

17
server/e2e/jobs/config/library-watcher-e2e-config.json Normal file
View File

@@ -0,0 +1,17 @@
{
"reverseGeocoding": {
"enabled": false
},
"machineLearning": {
"enabled": false
},
"logging": {
"enabled": false,
"level": "debug"
},
"library": {
"watch": {
"enabled": true
}
}
}

3
server/e2e/jobs/immich-e2e-config.json
View File

@@ -6,6 +6,7 @@
"enabled": false
},
"logging": {
"enabled": false
"enabled": false,
"level": "debug"
}
}

2
server/e2e/jobs/jest-e2e.json
View File

@@ -5,7 +5,7 @@
"globalSetup": "<rootDir>/e2e/jobs/setup.ts",
"testEnvironment": "node",
"testMatch": ["**/e2e/jobs/specs/*.e2e-spec.[tj]s"],
"testTimeout": 60000,
"testTimeout": 10000,
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
},

235
server/e2e/jobs/specs/library-watcher.e2e-spec.ts Normal file
View File

@@ -0,0 +1,235 @@
import { LibraryResponseDto, LibraryService, LoginResponseDto } from '@app/domain';
import { AssetType, LibraryType } from '@app/infra/entities';
import * as fs from 'fs/promises';
import { api } from '../../client';
import path from 'path';
import {
IMMICH_TEST_ASSET_PATH,
IMMICH_TEST_ASSET_TEMP_PATH,
restoreTempFolder,
testApp,
waitForEvent,
} from '../../../src/test-utils/utils';
describe(`Library watcher (e2e)`, () => {
let server: any;
let admin: LoginResponseDto;
let libraryService: LibraryService;
const configFilePath = process.env.IMMICH_CONFIG_FILE;
beforeAll(async () => {
process.env.IMMICH_CONFIG_FILE = path.normalize(`${__dirname}/../config/library-watcher-e2e-config.json`);
server = (await testApp.create()).getHttpServer();
libraryService = testApp.get(LibraryService);
});
beforeEach(async () => {
await testApp.reset();
await restoreTempFolder();
await api.authApi.adminSignUp(server);
admin = await api.authApi.adminLogin(server);
await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, '/');
});
afterEach(async () => {
await libraryService.unwatchAll();
});
afterAll(async () => {
await testApp.teardown();
await restoreTempFolder();
process.env.IMMICH_CONFIG_FILE = configFilePath;
});
describe('Event handling', () => {
let library: LibraryResponseDto;
describe('Single import path', () => {
beforeEach(async () => {
library = await api.libraryApi.create(server, admin.accessToken, {
type: LibraryType.EXTERNAL,
importPaths: [`${IMMICH_TEST_ASSET_TEMP_PATH}`],
});
});
it('should import a new file', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file.jpg`,
);
await waitForEvent(libraryService, 'add');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets.length).toEqual(1);
});
it('should import new files with case insensitive extensions', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file2.JPG`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file3.Jpg`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file4.jpG`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file5.jPg`,
);
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets.length).toEqual(4);
});
it('should update a changed file', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file.jpg`,
);
await waitForEvent(libraryService, 'add');
const originalAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(originalAssets.length).toEqual(1);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/prairie_falcon.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file.jpg`,
);
await waitForEvent(libraryService, 'change');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets).toEqual([
expect.objectContaining({
// Make sure we keep the original asset id
id: originalAssets[0].id,
type: AssetType.IMAGE,
exifInfo: expect.objectContaining({
make: 'Canon',
model: 'Canon EOS R5',
exifImageWidth: 800,
exifImageHeight: 533,
exposureTime: '1/4000',
}),
}),
]);
});
});
describe('Multiple import paths', () => {
beforeEach(async () => {
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`, { recursive: true });
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`, { recursive: true });
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`, { recursive: true });
library = await api.libraryApi.create(server, admin.accessToken, {
type: LibraryType.EXTERNAL,
importPaths: [
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`,
],
});
});
it('should add new files in multiple import paths', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1/file2.jpg`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/polemonium_reptans.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2/file3.jpg`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/tanners_ridge.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3/file4.jpg`,
);
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(assets.length).toEqual(3);
});
it('should offline a removed file', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/polemonium_reptans.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1/file.jpg`,
);
await waitForEvent(libraryService, 'add');
const addedAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(addedAssets.length).toEqual(1);
await fs.unlink(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1/file.jpg`);
await waitForEvent(libraryService, 'unlink');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets[0].isOffline).toEqual(true);
});
});
});
describe('Configuration', () => {
let library: LibraryResponseDto;
beforeEach(async () => {
library = await api.libraryApi.create(server, admin.accessToken, {
type: LibraryType.EXTERNAL,
importPaths: [
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`,
],
});
await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, '/');
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`, { recursive: true });
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`, { recursive: true });
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`, { recursive: true });
});
it('should use an updated import paths', async () => {
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir4`, { recursive: true });
await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir4`,
]);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/polemonium_reptans.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir4/file.jpg`,
);
await waitForEvent(libraryService, 'add');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets.length).toEqual(1);
});
});
});

10
server/e2e/jobs/specs/library.e2e-spec.ts
View File

@@ -21,11 +21,6 @@ describe(`${LibraryController.name} (e2e)`, () => {
server = (await testApp.create()).getHttpServer();
});
afterAll(async () => {
await testApp.teardown();
await restoreTempFolder();
});
beforeEach(async () => {
await testApp.reset();
await restoreTempFolder();
@@ -33,6 +28,11 @@ describe(`${LibraryController.name} (e2e)`, () => {
admin = await api.authApi.adminLogin(server);
});
afterAll(async () => {
await testApp.teardown();
await restoreTempFolder();
});
describe('DELETE /library/:id', () => {
it('should delete an external library with assets', async () => {
const library = await api.libraryApi.create(server, admin.accessToken, {

7
server/e2e/jobs/specs/metadata.e2e-spec.ts
View File

@@ -1,6 +1,5 @@
import { AssetResponseDto, LoginResponseDto } from '@app/domain';
import { AssetController } from '@app/immich';
import { INestApplication } from '@nestjs/common';
import { exiftool } from 'exiftool-vendored';
import { readFile, writeFile } from 'fs/promises';
import {
@@ -13,17 +12,15 @@ import {
import { api } from '../../client';
describe(`${AssetController.name} (e2e)`, () => {
let app: INestApplication;
let server: any;
let admin: LoginResponseDto;
beforeAll(async () => {
app = await testApp.create();
server = app.getHttpServer();
server = (await testApp.create()).getHttpServer();
});
beforeEach(async () => {
await db.reset();
await testApp.reset();
await restoreTempFolder();
await api.authApi.adminSignUp(server);
admin = await api.authApi.adminLogin(server);