cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): Automatic watching of library folders (#6192)

Browse Source 
* feat: initial watch support

* allow offline files

* chore: ignore query errors when resetting e2e db

* revert db query

* add savepoint

* guard the user query

* chore: openapi and db migration

* wip

* support multiple libraries

* fix tests

* wip

* can now cleanup chokidar watchers

* fix unit tests

* add library watch queue

* add missing init from merge

* wip

* can now filter file extensions

* remove watch api from non job client

* Fix e2e test

* watch library with updated import path and exclusion pattern

* add library watch frontend ui

* case sensitive watching extensions

* can auto watch libraries

* move watcher e2e tests to separate file

* don't watch libraries from a queue

* use event emitters

* shorten e2e test timeout

* refactor chokidar code to filesystem provider

* expose chokidar parameters to config file

* fix storage mock

* set default config for library watching

* add fs provider mocks

* cleanup

* add more unit tests for watcher

* chore: fix format + sql

* add more tests

* move unwatch feature back to library service

* add file event unit tests

* chore: formatting

* add documentation

* fix e2e tests

* chore: fix e2e tests

* fix library updating

* test cleanup

* fix typo

* cleanup

* fixing as per pr comments

* reduce library watch config file

* update storage config and mocks

* move negative event tests to unit tests

* fix library watcher e2e

* make watch configuration global

* remove the feature flag

* refactor watcher teardown

* fix microservices init

* centralize asset scan job queue

* improve docs

* add more tests

* chore: open api

* initialize app service

* fix docs

* fix library watch feature flag

* Update docs/docs/features/libraries.md

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* fix: import right app service

* don't be truthy

* fix test speling

* stricter library update tests

* move fs watcher mock to external file

* subscribe to config changes

* docker does not need polling

* make library watch() private

* feat: add configuration ui

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Jonathan Jogenfors
2024-01-31 09:15:54 +01:00
committed by GitHub
parent 4079e92bbf
commit 068e703e88
48 changed files with 1613 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
server/package-lock.json generated
View File

@@ -22,6 +22,7 @@
"@nestjs/typeorm": "^10.0.0",
"@nestjs/websockets": "^10.2.2",
"@socket.io/postgres-adapter": "^0.3.1",
"@types/picomatch": "^2.3.3",
"archiver": "^6.0.0",
"async-lock": "^1.4.0",
"axios": "^1.5.0",
@@ -45,6 +46,7 @@
"node-addon-api": "^7.0.0",
"openid-client": "^5.4.3",
"pg": "^8.11.3",
"picomatch": "^3.0.1",
"reflect-metadata": "^0.1.13",
"rxjs": "^7.8.1",
"sanitize-filename": "^1.6.3",
@@ -76,6 +78,7 @@
"@types/ua-parser-js": "^0.7.36",
"@typescript-eslint/eslint-plugin": "^6.4.1",
"@typescript-eslint/parser": "^6.4.1",
"chokidar": "^3.5.3",
"dotenv": "^16.3.1",
"eslint": "^8.48.0",
"eslint-config-prettier": "^9.0.0",
@@ -147,18 +150,6 @@
}
}
},
"node_modules/@angular-devkit/core/node_modules/picomatch": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-3.0.1.tgz",
"integrity": "sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag==",
"dev": true,
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/@angular-devkit/schematics": {
"version": "17.0.9",
"resolved": "https://registry.npmjs.org/@angular-devkit/schematics/-/schematics-17.0.9.tgz",
@@ -3304,6 +3295,11 @@
"node": ">=12"
}
},
"node_modules/@types/picomatch": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-2.3.3.tgz",
"integrity": "sha512-Yll76ZHikRFCyz/pffKGjrCwe/le2CDwOP5F210KQo27kpRE46U2rDnzikNlVn6/ezH3Mhn46bJMTfeVTtcYMg=="
},
"node_modules/@types/qs": {
"version": "6.9.8",
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.8.tgz",
@@ -3994,6 +3990,18 @@
"node": ">= 8"
}
},
"node_modules/anymatch/node_modules/picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"engines": {
"node": ">=8.6"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/app-root-path": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz",
@@ -7924,6 +7932,18 @@
"node": "^14.15.0 || ^16.10.0 || >=18.0.0"
}
},
"node_modules/jest-util/node_modules/picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"engines": {
"node": ">=8.6"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/jest-validate": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz",
@@ -8417,6 +8437,18 @@
"node": ">=8.6"
}
},
"node_modules/micromatch/node_modules/picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"engines": {
"node": ">=8.6"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/mime": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
@@ -9287,12 +9319,11 @@
"dev": true
},
"node_modules/picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-3.0.1.tgz",
"integrity": "sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag==",
"engines": {
"node": ">=8.6"
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
@@ -9768,6 +9799,18 @@
"node": ">=8.10.0"
}
},
"node_modules/readdirp/node_modules/picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"engines": {
"node": ">=8.6"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/rechoir": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
@@ -12248,14 +12291,6 @@
"picomatch": "3.0.1",
"rxjs": "7.8.1",
"source-map": "0.7.4"
},
"dependencies": {
"picomatch": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-3.0.1.tgz",
"integrity": "sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag==",
"dev": true
}
}
},
"@angular-devkit/schematics": {
@@ -14468,6 +14503,11 @@
}
}
},
"@types/picomatch": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-2.3.3.tgz",
"integrity": "sha512-Yll76ZHikRFCyz/pffKGjrCwe/le2CDwOP5F210KQo27kpRE46U2rDnzikNlVn6/ezH3Mhn46bJMTfeVTtcYMg=="
},
"@types/qs": {
"version": "6.9.8",
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.8.tgz",
@@ -15007,6 +15047,14 @@
"requires": {
"normalize-path": "^3.0.0",
"picomatch": "^2.0.4"
},
"dependencies": {
"picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true
}
}
},
"app-root-path": {
@@ -17938,6 +17986,14 @@
"ci-info": "^3.2.0",
"graceful-fs": "^4.2.9",
"picomatch": "^2.2.3"
},
"dependencies": {
"picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true
}
}
},
"jest-validate": {
@@ -18339,6 +18395,14 @@
"requires": {
"braces": "^3.0.2",
"picomatch": "^2.3.1"
},
"dependencies": {
"picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true
}
}
},
"mime": {
@@ -19005,10 +19069,9 @@
"dev": true
},
"picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-3.0.1.tgz",
"integrity": "sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag=="
},
"pirates": {
"version": "4.0.6",
@@ -19345,6 +19408,14 @@
"dev": true,
"requires": {
"picomatch": "^2.2.1"
},
"dependencies": {
"picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true
}
}
},
"rechoir": {