cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'immich-app:main' into FAQ-edit

Browse Source
This commit is contained in:
aviv926
2024-01-03 21:08:20 +02:00
committed by GitHub
parent 43f15ce9b7 40fbe81c7b
commit cc5c2b0461
47 changed files with 1011 additions and 1200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/package-lock.json
Generated
+497 -837
View File
File diff suppressed because it is too large Load Diff
cli/package.json
+2 -2
View File
@@ -1,6 +1,6 @@
{ {
"name": "@immich/cli", "name": "@immich/cli",
"version": "2.0.5", "version": "2.0.6",
"description": "Command Line Interface (CLI) for Immich", "description": "Command Line Interface (CLI) for Immich",
"main": "dist/index.js", "main": "dist/index.js",
"bin": { "bin": {
@@ -38,7 +38,7 @@
"eslint-config-prettier": "^9.0.0", "eslint-config-prettier": "^9.0.0",
"eslint-plugin-jest": "^27.2.2", "eslint-plugin-jest": "^27.2.2",
"eslint-plugin-prettier": "^5.0.0", "eslint-plugin-prettier": "^5.0.0",
"eslint-plugin-unicorn": "^49.0.0", "eslint-plugin-unicorn": "^50.0.0",
"immich": "file:../server", "immich": "file:../server",
"jest": "^29.5.0", "jest": "^29.5.0",
"jest-extended": "^4.0.0", "jest-extended": "^4.0.0",
cli/src/commands/upload.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export default class Upload extends BaseCommand {
for (const asset of assetsToUpload) { for (const asset of assetsToUpload) {
// Compute total size first // Compute total size first
await asset.process(); await asset.prepare();
totalSize += asset.fileSize; totalSize += asset.fileSize;
if (options.albumName) { if (options.albumName) {
cli/src/cores/models/asset.ts
+16 -23
View File
@@ -1,18 +1,17 @@
import * as fs from 'graceful-fs';
import { basename } from 'node:path';
import crypto from 'crypto'; import crypto from 'crypto';
import Os from 'os';
import FormData from 'form-data'; import FormData from 'form-data';
import * as fs from 'graceful-fs';
import { createReadStream } from 'node:fs';
import { basename } from 'node:path';
import Os from 'os';
export class Asset { export class Asset {
readonly path: string; readonly path: string;
readonly deviceId!: string; readonly deviceId!: string;
assetData?: fs.ReadStream;
deviceAssetId?: string; deviceAssetId?: string;
fileCreatedAt?: string; fileCreatedAt?: string;
fileModifiedAt?: string; fileModifiedAt?: string;
sidecarData?: fs.ReadStream;
sidecarPath?: string; sidecarPath?: string;
fileSize!: number; fileSize!: number;
albumName?: string; albumName?: string;
@@ -21,32 +20,30 @@ export class Asset {
this.path = path; this.path = path;
} }
async process() { async prepare() {
const stats = await fs.promises.stat(this.path); const stats = await fs.promises.stat(this.path);
this.deviceAssetId = `${basename(this.path)}-${stats.size}`.replace(/\s+/g, ''); this.deviceAssetId = `${basename(this.path)}-${stats.size}`.replace(/\s+/g, '');
this.fileCreatedAt = stats.mtime.toISOString(); this.fileCreatedAt = stats.mtime.toISOString();
this.fileModifiedAt = stats.mtime.toISOString(); this.fileModifiedAt = stats.mtime.toISOString();
this.fileSize = stats.size; this.fileSize = stats.size;
this.albumName = this.extractAlbumName(); this.albumName = this.extractAlbumName();
this.assetData = this.getReadStream(this.path);
// TODO: doesn't xmp replace the file extension? Will need investigation
const sideCarPath = `${this.path}.xmp`;
try {
fs.accessSync(sideCarPath, fs.constants.R_OK);
this.sidecarData = this.getReadStream(sideCarPath);
} catch (error) {}
} }
getUploadFormData(): FormData { getUploadFormData(): FormData {
if (!this.assetData) throw new Error('Asset data not set');
if (!this.deviceAssetId) throw new Error('Device asset id not set'); if (!this.deviceAssetId) throw new Error('Device asset id not set');
if (!this.fileCreatedAt) throw new Error('File created at not set'); if (!this.fileCreatedAt) throw new Error('File created at not set');
if (!this.fileModifiedAt) throw new Error('File modified at not set'); if (!this.fileModifiedAt) throw new Error('File modified at not set');
// TODO: doesn't xmp replace the file extension? Will need investigation
const sideCarPath = `${this.path}.xmp`;
let sidecarData: fs.ReadStream | undefined = undefined;
try {
fs.accessSync(sideCarPath, fs.constants.R_OK);
sidecarData = createReadStream(sideCarPath);
} catch (error) {}
const data: any = { const data: any = {
assetData: this.assetData as any, assetData: createReadStream(this.path),
deviceAssetId: this.deviceAssetId, deviceAssetId: this.deviceAssetId,
deviceId: 'CLI', deviceId: 'CLI',
fileCreatedAt: this.fileCreatedAt, fileCreatedAt: this.fileCreatedAt,
@@ -59,17 +56,13 @@ export class Asset {
formData.append(prop, data[prop]); formData.append(prop, data[prop]);
} }
if (this.sidecarData) { if (sidecarData) {
formData.append('sidecarData', this.sidecarData); formData.append('sidecarData', sidecarData);
} }
return formData; return formData;
} }
private getReadStream(path: string): fs.ReadStream {
return fs.createReadStream(path);
}
async delete(): Promise<void> { async delete(): Promise<void> {
return fs.promises.unlink(this.path); return fs.promises.unlink(this.path);
} }
docs/docs/features/facial-recognition.md
+1 -1
View File
@@ -25,6 +25,6 @@ Additional actions you can do with a detected person are:
- Merge two or more detected faces into one person - Merge two or more detected faces into one person
- Hide face - Hide face
It can be found from the app bar when you access the detial view of a person It can be found from the app bar when you access the detail view of a person.
<img src={require('./img/facial-recognition-4.png').default} title='Facial Recognition 4' width="70%"/> <img src={require('./img/facial-recognition-4.png').default} title='Facial Recognition 4' width="70%"/>
docs/docs/features/hardware-transcoding.md
+16
View File
@@ -43,12 +43,28 @@ As this is a new feature, it is still experimental and may not work on all syste
## Setup ## Setup
#### Initial Setup
1. If you do not already have it, download the latest [`hwaccel.yml`][hw-file] file and ensure it's in the same folder as the `docker-compose.yml`. 1. If you do not already have it, download the latest [`hwaccel.yml`][hw-file] file and ensure it's in the same folder as the `docker-compose.yml`.
2. Uncomment the lines that apply to your system and desired usage. 2. Uncomment the lines that apply to your system and desired usage.
3. In the `docker-compose.yml` under `immich-microservices`, uncomment the lines relating to the `hwaccel.yml` file. 3. In the `docker-compose.yml` under `immich-microservices`, uncomment the lines relating to the `hwaccel.yml` file.
4. Redeploy the `immich-microservices` container with these updated settings. 4. Redeploy the `immich-microservices` container with these updated settings.
5. In the Admin page under `FFmpeg settings`, change the hardware acceleration setting to the appropriate option and save. 5. In the Admin page under `FFmpeg settings`, change the hardware acceleration setting to the appropriate option and save.
#### All-In-One - Unraid Setup
##### NVENC - NVIDIA GPUs
- If you are using other backends. You will still need to implement [`hwaccel.yml`][hw-file] file into the `immich-microservices` service directly, please see the "Initial Setup" section above on how to do that.
- As of v1.92.0, steps 1 and 2 are no longer necessary. If your version of Immich is below that or missing the environment variables, please follow these steps. Otherwise, skip to step 3.
- Please note that`NVIDIA_DRIVER_CAPABILITIES` is no longer required to enter as a variable.
1. Assuming you already have the Nvidia Driver Plugin installed on your Unraid Server. Please confirm that your Nvida GPU is showing up with its GPU ID in the Nvidia Driver Plugin. The ID will be `GPU-LONG_STRING_OF_CHARACTERS`. Copy the GPU ID.
2. In the Imagegenius/Immich Docker Container app, add two new variables: Key=`NVIDIA_VISIBLE_DEVICES` Value=`GPU-LONG_STRING_OF_CHARACTERS` and Key=`NVIDIA_DRIVER_CAPABILITIES` Value=`all`
3. While you are in the docker container app, change the Container from Basic Mode to Advanced Mode and add the following parameter to the Extra Parameters field: `--runtime=nvidia`
4. Restart the Imagegenius/Immich Docker Container app.
5. In the Admin page under FFmpeg settings, change the hardware acceleration setting to the appropriate option and save.
## Tips ## Tips
- You may want to choose a slower preset than for software transcoding to maintain quality and efficiency - You may want to choose a slower preset than for software transcoding to maintain quality and efficiency
docs/docs/features/libraries.md
+20
View File
@@ -12,6 +12,26 @@ Immich comes preconfigured with an upload library for each user. All assets uplo
External libraries tracks assets stored outside of immich, i.e. in the file system. Immich will only read data from the files, and will not modify them in any way. Therefore, the delete button is disabled for external assets. When the external library is scanned, immich will read the metadata from the file and create an asset in the library for each image or video file. These items will then be shown in the main timeline, and they will look and behave like any other asset, including viewing on the map, adding to albums, etc. External libraries tracks assets stored outside of immich, i.e. in the file system. Immich will only read data from the files, and will not modify them in any way. Therefore, the delete button is disabled for external assets. When the external library is scanned, immich will read the metadata from the file and create an asset in the library for each image or video file. These items will then be shown in the main timeline, and they will look and behave like any other asset, including viewing on the map, adding to albums, etc.
# Step by Step Guide
In order to create an external library, the administrator must:
- Click on 'Administration' (top right).
- Click on 'Edit' next to the user.
- Set the external path - this would be the path that is exposed to the immich microservices service.
- Click 'Confirm'.
- Click into the user profile (top right) then click into 'Libraries'.
- Click 'Create External Library'.
- Next to the new library, click the three dots, then click 'Edit Import Paths'.
- Set up your path relative to the external path you set earlier - e.g., if you set the external path to be `/mnt/media/external` and your Import Paths to `/photos`, then you will end up with `/mnt/media/external/photos`.
- Click 'Save'.
- Click 'Scan All Libraries'.
To confirm it's working, you can open up a Docker shell for the immich microservices, and you should see similar outputs to this:
`[Nest] 7 - 11/24/2023, 1:06:39 AM LOG [MediaService] Successfully generated WEBP image thumbnail for asset b6d37673-5931-4b25-8380-89135021f48b`
# Scanning functionality
If a file is modified outside of Immich, the changes will not be reflected in immich until the library is scanned again. There are different ways to scan a library depending on the use case: If a file is modified outside of Immich, the changes will not be reflected in immich until the library is scanned again. There are different ways to scan a library depending on the use case:
- Scan Library Files: This is the default scan method and also the quickest. It will scan all files in the library and add new files to the library. It will notice if any files are missing (see below) but not check existing assets - Scan Library Files: This is the default scan method and also the quickest. It will scan all files in the library and add new files to the library. It will notice if any files are missing (see below) but not check existing assets
docs/docs/guides/remote-access.md
+1 -1
View File
@@ -42,7 +42,7 @@ If you are unable to open a port on your router for Wireguard or OpenVPN to your
A reverse proxy is a service that sits between web servers and clients. A reverse proxy can either be hosted on the server itself or remotely. Clients can connect to the reverse proxy via https, and the proxy relays data to Immich. This setup makes most sense if you have your own domain and want to access your Immich instance just like any other website, from outside your LAN. You can also use a DDNS provider like DuckDNS or no-ip if you don't have a domain. This configuration allows the Immich Android and iphone apps to connect to your server without a VPN or tailscale app on the client side. A reverse proxy is a service that sits between web servers and clients. A reverse proxy can either be hosted on the server itself or remotely. Clients can connect to the reverse proxy via https, and the proxy relays data to Immich. This setup makes most sense if you have your own domain and want to access your Immich instance just like any other website, from outside your LAN. You can also use a DDNS provider like DuckDNS or no-ip if you don't have a domain. This configuration allows the Immich Android and iphone apps to connect to your server without a VPN or tailscale app on the client side.
If you're hosting your own reverse proxy, [Nginx](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) is a great option. An example configuration for Nginx is provided [here](https://immich.app/docs/administration/reverse-proxy). If you're hosting your own reverse proxy, [Nginx](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) is a great option. An example configuration for Nginx is provided [here](/docs/administration/reverse-proxy.md).
You'll also need your own certificate to authenticate https connections. If you're making Immich publicly accesible, [Let's Encrypt](https://letsencrypt.org/) can provide a free certificate for your domain and is the recommended option. Alternatively, a [self-signed certificate](https://en.wikipedia.org/wiki/Self-signed_certificate) allows you to encrypt your connection to Immich, but it raises a security warning on the client's browser. You'll also need your own certificate to authenticate https connections. If you're making Immich publicly accesible, [Let's Encrypt](https://letsencrypt.org/) can provide a free certificate for your domain and is the recommended option. Alternatively, a [self-signed certificate](https://en.wikipedia.org/wiki/Self-signed_certificate) allows you to encrypt your connection to Immich, but it raises a security warning on the client's browser.
docs/docs/install/config-file.md
+1 -1
View File
@@ -142,4 +142,4 @@ So you can just grab it from there, paste it into a file and you're pretty much
### Step 2 - Specify the file location ### Step 2 - Specify the file location
In your `.env` file, set the variable `IMMICH_CONFIG_FILE` to the path of your config. In your `.env` file, set the variable `IMMICH_CONFIG_FILE` to the path of your config.
For more information, refer to the [Environment Variables](https://docs.immich.app/docs/install/environment-variables) section. For more information, refer to the [Environment Variables](/docs/install/environment-variables.md) section.
mobile/assets/i18n/en-US.json
+7 -7
View File
@@ -93,7 +93,7 @@
"backup_controller_page_status_off": "Automatic foreground backup is off", "backup_controller_page_status_off": "Automatic foreground backup is off",
"backup_controller_page_status_on": "Automatic foreground backup is on", "backup_controller_page_status_on": "Automatic foreground backup is on",
"backup_controller_page_storage_format": "{} of {} used", "backup_controller_page_storage_format": "{} of {} used",
"backup_controller_page_to_backup": "Albums to be backup", "backup_controller_page_to_backup": "Albums to back up",
"backup_controller_page_total": "Total", "backup_controller_page_total": "Total",
"backup_controller_page_total_sub": "All unique photos and videos from selected albums", "backup_controller_page_total_sub": "All unique photos and videos from selected albums",
"backup_controller_page_turn_off": "Turn off foreground backup", "backup_controller_page_turn_off": "Turn off foreground backup",
@@ -103,7 +103,7 @@
"backup_info_card_assets": "assets", "backup_info_card_assets": "assets",
"backup_manual_cancelled": "Cancelled", "backup_manual_cancelled": "Cancelled",
"backup_manual_failed": "Failed", "backup_manual_failed": "Failed",
"backup_manual_in_progress": "Upload already in progress. Try after sometime", "backup_manual_in_progress": "Upload already in progress. Try again after some time",
"backup_manual_success": "Success", "backup_manual_success": "Success",
"backup_manual_title": "Upload status", "backup_manual_title": "Upload status",
"cache_settings_album_thumbnails": "Library page thumbnails ({} assets)", "cache_settings_album_thumbnails": "Library page thumbnails ({} assets)",
@@ -190,7 +190,7 @@
"home_page_delete_err_partner": "Can not delete partner assets, skipping", "home_page_delete_err_partner": "Can not delete partner assets, skipping",
"home_page_favorite_err_local": "Can not favorite local assets yet, skipping", "home_page_favorite_err_local": "Can not favorite local assets yet, skipping",
"home_page_favorite_err_partner": "Can not favorite partner assets yet, skipping", "home_page_favorite_err_partner": "Can not favorite partner assets yet, skipping",
"home_page_first_time_notice": "If this is your first time using the app, please make sure to choose a backup album(s) so that the timeline can populate photos and videos in the album(s).", "home_page_first_time_notice": "If this is your first time using the app, please make sure to choose backup album(s) so that the timeline can populate photos and videos in the album(s).",
"home_page_share_err_local": "Can not share local assets via link, skipping", "home_page_share_err_local": "Can not share local assets via link, skipping",
"home_page_upload_err_limit": "Can only upload a maximum of 30 assets at a time, skipping", "home_page_upload_err_limit": "Can only upload a maximum of 30 assets at a time, skipping",
"image_viewer_page_state_provider_download_error": "Download Error", "image_viewer_page_state_provider_download_error": "Download Error",
@@ -230,7 +230,7 @@
"login_form_failed_get_oauth_server_config": "Error logging using OAuth, check server URL", "login_form_failed_get_oauth_server_config": "Error logging using OAuth, check server URL",
"login_form_failed_get_oauth_server_disable": "OAuth feature is not available on this server", "login_form_failed_get_oauth_server_disable": "OAuth feature is not available on this server",
"login_form_failed_login": "Error logging you in, check server URL, email and password", "login_form_failed_login": "Error logging you in, check server URL, email and password",
"login_form_handshake_exception": "There was an Handshake Exception with the server. Enable self-signed certificate support in the settings if you are using a self-signed certificate.", "login_form_handshake_exception": "There was a Handshake Exception with the server. Enable self-signed certificate support in the settings if you are using a self-signed certificate.",
"login_form_label_email": "Email", "login_form_label_email": "Email",
"login_form_label_password": "Password", "login_form_label_password": "Password",
"login_form_next_button": "Next", "login_form_next_button": "Next",
@@ -379,8 +379,8 @@
"shared_link_create_error": "Error while creating shared link", "shared_link_create_error": "Error while creating shared link",
"shared_link_create_info": "Let anyone with the link see the selected photo(s)", "shared_link_create_info": "Let anyone with the link see the selected photo(s)",
"shared_link_create_submit_button": "Create link", "shared_link_create_submit_button": "Create link",
"shared_link_edit_allow_download": "Allow public user to download", "shared_link_edit_allow_download": "Allow public users to download",
"shared_link_edit_allow_upload": "Allow public user to upload", "shared_link_edit_allow_upload": "Allow public users to upload",
"shared_link_edit_app_bar_title": "Edit link", "shared_link_edit_app_bar_title": "Edit link",
"shared_link_edit_change_expiry": "Change expiration time", "shared_link_edit_change_expiry": "Change expiration time",
"shared_link_edit_description": "Description", "shared_link_edit_description": "Description",
@@ -430,7 +430,7 @@
"theme_setting_dark_mode_switch": "Dark mode", "theme_setting_dark_mode_switch": "Dark mode",
"theme_setting_image_viewer_quality_subtitle": "Adjust the quality of the detail image viewer", "theme_setting_image_viewer_quality_subtitle": "Adjust the quality of the detail image viewer",
"theme_setting_image_viewer_quality_title": "Image viewer quality", "theme_setting_image_viewer_quality_title": "Image viewer quality",
"theme_setting_system_theme_switch": "Automatic (Follow system setting)", "theme_setting_system_theme_switch": "Automatic (follow system settings)",
"theme_setting_theme_subtitle": "Choose the app's theme setting", "theme_setting_theme_subtitle": "Choose the app's theme setting",
"theme_setting_theme_title": "Theme", "theme_setting_theme_title": "Theme",
"theme_setting_three_stage_loading_subtitle": "Three-stage loading might increase the loading performance but causes significantly higher network load", "theme_setting_three_stage_loading_subtitle": "Three-stage loading might increase the loading performance but causes significantly higher network load",
mobile/lib/modules/backup/services/backup.service.dart
+1 -4
View File
@@ -44,10 +44,7 @@ class BackupService {
final String deviceId = Store.get(StoreKey.deviceId); final String deviceId = Store.get(StoreKey.deviceId);
try { try {
return await _apiService.assetApi.getUserAssetsByDeviceId(deviceId); return await _apiService.assetApi.getAllUserAssetsByDeviceId(deviceId);
// TODO! Start using this in 1.92.0
// return await _apiService.assetApi.getAllUserAssetsByDeviceId(deviceId);
} catch (e) { } catch (e) {
debugPrint('Error [getDeviceBackupAsset] ${e.toString()}'); debugPrint('Error [getDeviceBackupAsset] ${e.toString()}');
return null; return null;
renovate.json
+7 -3
View File
@@ -33,6 +33,12 @@
"matchPackagePrefixes": ["exiftool"], "matchPackagePrefixes": ["exiftool"],
"schedule": "on tuesday" "schedule": "on tuesday"
}, },
{
"groupName": "svelte",
"matchUpdateTypes": ["major"],
"matchPackagePrefixes": ["@sveltejs"],
"schedule": "on tuesday"
},
{ {
"matchFileNames": ["web/**"], "matchFileNames": ["web/**"],
"groupName": "web", "groupName": "web",
@@ -62,9 +68,7 @@
"versioning": "node" "versioning": "node"
} }
], ],
"ignorePaths": [ "ignorePaths": ["mobile/openapi/pubspec.yaml"],
"mobile/openapi/pubspec.yaml"
],
"ignoreDeps": [ "ignoreDeps": [
"http", "http",
"latlong2", "latlong2",
server/package-lock.json
Generated
+56 -55
View File
@@ -17,7 +17,7 @@
"@nestjs/core": "^10.2.2", "@nestjs/core": "^10.2.2",
"@nestjs/platform-express": "^10.2.2", "@nestjs/platform-express": "^10.2.2",
"@nestjs/platform-socket.io": "^10.2.2", "@nestjs/platform-socket.io": "^10.2.2",
"@nestjs/schedule": "^3.0.3", "@nestjs/schedule": "^4.0.0",
"@nestjs/swagger": "^7.1.8", "@nestjs/swagger": "^7.1.8",
"@nestjs/typeorm": "^10.0.0", "@nestjs/typeorm": "^10.0.0",
"@nestjs/websockets": "^10.2.2", "@nestjs/websockets": "^10.2.2",
@@ -73,7 +73,7 @@
"@types/multer": "^1.4.7", "@types/multer": "^1.4.7",
"@types/node": "^20.5.7", "@types/node": "^20.5.7",
"@types/sharp": "^0.31.1", "@types/sharp": "^0.31.1",
"@types/supertest": "^2.0.12", "@types/supertest": "^6.0.0",
"@types/ua-parser-js": "^0.7.36", "@types/ua-parser-js": "^0.7.36",
"@typescript-eslint/eslint-plugin": "^6.4.1", "@typescript-eslint/eslint-plugin": "^6.4.1",
"@typescript-eslint/parser": "^6.4.1", "@typescript-eslint/parser": "^6.4.1",
@@ -2466,11 +2466,11 @@
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
}, },
"node_modules/@nestjs/schedule": { "node_modules/@nestjs/schedule": {
"version": "3.0.4", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-3.0.4.tgz", "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.0.0.tgz",
"integrity": "sha512-uFJpuZsXfpvgx2y7/KrIZW9e1L68TLiwRodZ6+Gc8xqQiHSUzAVn+9F4YMxWFlHITZvvkjWziUFgRNCitDcTZQ==", "integrity": "sha512-zz4h54m/F/1qyQKvMJCRphmuwGqJltDAkFxUXCVqJBXEs5kbPt93Pza3heCQOcMH22MZNhGlc9DmDMLXVHmgVQ==",
"dependencies": { "dependencies": {
"cron": "2.4.3", "cron": "3.1.3",
"uuid": "9.0.1" "uuid": "9.0.1"
}, },
"peerDependencies": { "peerDependencies": {
@@ -3336,9 +3336,9 @@
} }
}, },
"node_modules/@types/cookiejar": { "node_modules/@types/cookiejar": {
"version": "2.1.2", "version": "2.1.5",
"resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.2.tgz", "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz",
"integrity": "sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==", "integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==",
"dev": true "dev": true
}, },
"node_modules/@types/cors": { "node_modules/@types/cors": {
@@ -3522,6 +3522,12 @@
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.5.tgz", "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.5.tgz",
"integrity": "sha512-1cyf6Ge/94zlaWIZA2ei1pE6SZ8xpad2hXaYa5JEFiaUH0YS494CZwyi4MXNpXD9oEuv6ZH0Bmh0e7F9sPhmZA==" "integrity": "sha512-1cyf6Ge/94zlaWIZA2ei1pE6SZ8xpad2hXaYa5JEFiaUH0YS494CZwyi4MXNpXD9oEuv6ZH0Bmh0e7F9sPhmZA=="
}, },
"node_modules/@types/methods": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz",
"integrity": "sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==",
"dev": true
},
"node_modules/@types/mime": { "node_modules/@types/mime": {
"version": "1.3.3", "version": "1.3.3",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz",
@@ -3699,22 +3705,24 @@
"dev": true "dev": true
}, },
"node_modules/@types/superagent": { "node_modules/@types/superagent": {
"version": "4.1.19", "version": "8.1.1",
"resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.19.tgz", "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-8.1.1.tgz",
"integrity": "sha512-McM1mlc7PBZpCaw0fw/36uFqo0YeA6m8JqoyE4OfqXsZCIg0hPP2xdE6FM7r6fdprDZHlJwDpydUj1R++93hCA==", "integrity": "sha512-YQyEXA4PgCl7EVOoSAS3o0fyPFU6erv5mMixztQYe1bqbWmmn8c+IrqoxjQeZe4MgwXikgcaZPiI/DsbmOVlzA==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"@types/cookiejar": "*", "@types/cookiejar": "^2.1.5",
"@types/methods": "^1.1.4",
"@types/node": "*" "@types/node": "*"
} }
}, },
"node_modules/@types/supertest": { "node_modules/@types/supertest": {
"version": "2.0.16", "version": "6.0.2",
"resolved": "https://registry.npmjs.org/@types/supertest/-/supertest-2.0.16.tgz", "resolved": "https://registry.npmjs.org/@types/supertest/-/supertest-6.0.2.tgz",
"integrity": "sha512-6c2ogktZ06tr2ENoZivgm7YnprnhYE4ZoXGMY+oA7IuAf17M8FWvujXZGmxLv8y0PTyts4x5A+erSwVUFA8XSg==", "integrity": "sha512-137ypx2lk/wTQbW6An6safu9hXmajAifU/s7szAHLN/FeIm5w7yR0Wkl9fdJMRSHwOn4HLAI0DaB2TOORuhPDg==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"@types/superagent": "*" "@types/methods": "^1.1.4",
"@types/superagent": "^8.1.0"
} }
}, },
"node_modules/@types/through": { "node_modules/@types/through": {
@@ -5610,12 +5618,12 @@
"devOptional": true "devOptional": true
}, },
"node_modules/cron": { "node_modules/cron": {
"version": "2.4.3", "version": "3.1.3",
"resolved": "https://registry.npmjs.org/cron/-/cron-2.4.3.tgz", "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.3.tgz",
"integrity": "sha512-YBvExkQYF7w0PxyeFLRyr817YVDhGxaCi5/uRRMqa4aWD3IFKRd+uNbpW1VWMdqQy8PZ7CElc+accXJcauPKzQ==", "integrity": "sha512-KVxeKTKYj2eNzN4ElnT6nRSbjbfhyxR92O/Jdp6SH3pc05CDJws59jBrZWEMQlxevCiE6QUTrXy+Im3vC3oD3A==",
"dependencies": { "dependencies": {
"@types/luxon": "~3.3.0", "@types/luxon": "~3.3.0",
"luxon": "~3.3.0" "luxon": "~3.4.0"
} }
}, },
"node_modules/cron-parser": { "node_modules/cron-parser": {
@@ -5629,14 +5637,6 @@
"node": ">=12.0.0" "node": ">=12.0.0"
} }
}, },
"node_modules/cron/node_modules/luxon": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz",
"integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==",
"engines": {
"node": ">=12"
}
},
"node_modules/cross-spawn": { "node_modules/cross-spawn": {
"version": "7.0.3", "version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -14601,11 +14601,11 @@
} }
}, },
"@nestjs/schedule": { "@nestjs/schedule": {
"version": "3.0.4", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-3.0.4.tgz", "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.0.0.tgz",
"integrity": "sha512-uFJpuZsXfpvgx2y7/KrIZW9e1L68TLiwRodZ6+Gc8xqQiHSUzAVn+9F4YMxWFlHITZvvkjWziUFgRNCitDcTZQ==", "integrity": "sha512-zz4h54m/F/1qyQKvMJCRphmuwGqJltDAkFxUXCVqJBXEs5kbPt93Pza3heCQOcMH22MZNhGlc9DmDMLXVHmgVQ==",
"requires": { "requires": {
"cron": "2.4.3", "cron": "3.1.3",
"uuid": "9.0.1" "uuid": "9.0.1"
}, },
"dependencies": { "dependencies": {
@@ -15254,9 +15254,9 @@
} }
}, },
"@types/cookiejar": { "@types/cookiejar": {
"version": "2.1.2", "version": "2.1.5",
"resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.2.tgz", "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz",
"integrity": "sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==", "integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==",
"dev": true "dev": true
}, },
"@types/cors": { "@types/cors": {
@@ -15440,6 +15440,12 @@
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.5.tgz", "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.5.tgz",
"integrity": "sha512-1cyf6Ge/94zlaWIZA2ei1pE6SZ8xpad2hXaYa5JEFiaUH0YS494CZwyi4MXNpXD9oEuv6ZH0Bmh0e7F9sPhmZA==" "integrity": "sha512-1cyf6Ge/94zlaWIZA2ei1pE6SZ8xpad2hXaYa5JEFiaUH0YS494CZwyi4MXNpXD9oEuv6ZH0Bmh0e7F9sPhmZA=="
}, },
"@types/methods": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz",
"integrity": "sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==",
"dev": true
},
"@types/mime": { "@types/mime": {
"version": "1.3.3", "version": "1.3.3",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz",
@@ -15604,22 +15610,24 @@
"dev": true "dev": true
}, },
"@types/superagent": { "@types/superagent": {
"version": "4.1.19", "version": "8.1.1",
"resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.19.tgz", "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-8.1.1.tgz",
"integrity": "sha512-McM1mlc7PBZpCaw0fw/36uFqo0YeA6m8JqoyE4OfqXsZCIg0hPP2xdE6FM7r6fdprDZHlJwDpydUj1R++93hCA==", "integrity": "sha512-YQyEXA4PgCl7EVOoSAS3o0fyPFU6erv5mMixztQYe1bqbWmmn8c+IrqoxjQeZe4MgwXikgcaZPiI/DsbmOVlzA==",
"dev": true, "dev": true,
"requires": { "requires": {
"@types/cookiejar": "*", "@types/cookiejar": "^2.1.5",
"@types/methods": "^1.1.4",
"@types/node": "*" "@types/node": "*"
} }
}, },
"@types/supertest": { "@types/supertest": {
"version": "2.0.16", "version": "6.0.2",
"resolved": "https://registry.npmjs.org/@types/supertest/-/supertest-2.0.16.tgz", "resolved": "https://registry.npmjs.org/@types/supertest/-/supertest-6.0.2.tgz",
"integrity": "sha512-6c2ogktZ06tr2ENoZivgm7YnprnhYE4ZoXGMY+oA7IuAf17M8FWvujXZGmxLv8y0PTyts4x5A+erSwVUFA8XSg==", "integrity": "sha512-137ypx2lk/wTQbW6An6safu9hXmajAifU/s7szAHLN/FeIm5w7yR0Wkl9fdJMRSHwOn4HLAI0DaB2TOORuhPDg==",
"dev": true, "dev": true,
"requires": { "requires": {
"@types/superagent": "*" "@types/methods": "^1.1.4",
"@types/superagent": "^8.1.0"
} }
}, },
"@types/through": { "@types/through": {
@@ -17029,19 +17037,12 @@
"devOptional": true "devOptional": true
}, },
"cron": { "cron": {
"version": "2.4.3", "version": "3.1.3",
"resolved": "https://registry.npmjs.org/cron/-/cron-2.4.3.tgz", "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.3.tgz",
"integrity": "sha512-YBvExkQYF7w0PxyeFLRyr817YVDhGxaCi5/uRRMqa4aWD3IFKRd+uNbpW1VWMdqQy8PZ7CElc+accXJcauPKzQ==", "integrity": "sha512-KVxeKTKYj2eNzN4ElnT6nRSbjbfhyxR92O/Jdp6SH3pc05CDJws59jBrZWEMQlxevCiE6QUTrXy+Im3vC3oD3A==",
"requires": { "requires": {
"@types/luxon": "~3.3.0", "@types/luxon": "~3.3.0",
"luxon": "~3.3.0" "luxon": "~3.4.0"
},
"dependencies": {
"luxon": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz",
"integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg=="
}
} }
}, },
"cron-parser": { "cron-parser": {
server/package.json
+2 -2
View File
@@ -44,7 +44,7 @@
"@nestjs/core": "^10.2.2", "@nestjs/core": "^10.2.2",
"@nestjs/platform-express": "^10.2.2", "@nestjs/platform-express": "^10.2.2",
"@nestjs/platform-socket.io": "^10.2.2", "@nestjs/platform-socket.io": "^10.2.2",
"@nestjs/schedule": "^3.0.3", "@nestjs/schedule": "^4.0.0",
"@nestjs/swagger": "^7.1.8", "@nestjs/swagger": "^7.1.8",
"@nestjs/typeorm": "^10.0.0", "@nestjs/typeorm": "^10.0.0",
"@nestjs/websockets": "^10.2.2", "@nestjs/websockets": "^10.2.2",
@@ -100,7 +100,7 @@
"@types/multer": "^1.4.7", "@types/multer": "^1.4.7",
"@types/node": "^20.5.7", "@types/node": "^20.5.7",
"@types/sharp": "^0.31.1", "@types/sharp": "^0.31.1",
"@types/supertest": "^2.0.12", "@types/supertest": "^6.0.0",
"@types/ua-parser-js": "^0.7.36", "@types/ua-parser-js": "^0.7.36",
"@typescript-eslint/eslint-plugin": "^6.4.1", "@typescript-eslint/eslint-plugin": "^6.4.1",
"@typescript-eslint/parser": "^6.4.1", "@typescript-eslint/parser": "^6.4.1",
server/src/domain/asset/asset.service.spec.ts
+9 -6
View File
@@ -784,9 +784,9 @@ describe(AssetService.name, () => {
await sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'], force: true }); await sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'], force: true });
expect(jobMock.queue.mock.calls).toEqual([ expect(jobMock.queueAll).toHaveBeenCalledWith([
[{ name: JobName.ASSET_DELETION, data: { id: 'asset1' } }], { name: JobName.ASSET_DELETION, data: { id: 'asset1' } },
[{ name: JobName.ASSET_DELETION, data: { id: 'asset2' } }], { name: JobName.ASSET_DELETION, data: { id: 'asset2' } },
]); ]);
}); });
@@ -895,6 +895,7 @@ describe(AssetService.name, () => {
await sut.handleAssetDeletion({ id: assetStub.external.id }); await sut.handleAssetDeletion({ id: assetStub.external.id });
expect(jobMock.queue).not.toBeCalled(); expect(jobMock.queue).not.toBeCalled();
expect(jobMock.queueAll).not.toBeCalled();
expect(assetMock.remove).not.toBeCalled(); expect(assetMock.remove).not.toBeCalled();
}); });
@@ -952,19 +953,21 @@ describe(AssetService.name, () => {
it('should run the refresh metadata job', async () => { it('should run the refresh metadata job', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.REFRESH_METADATA }), await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.REFRESH_METADATA }),
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.METADATA_EXTRACTION, data: { id: 'asset-1' } }); expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.METADATA_EXTRACTION, data: { id: 'asset-1' } }]);
}); });
it('should run the refresh thumbnails job', async () => { it('should run the refresh thumbnails job', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.REGENERATE_THUMBNAIL }), await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.REGENERATE_THUMBNAIL }),
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } }); expect(jobMock.queueAll).toHaveBeenCalledWith([
{ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } },
]);
}); });
it('should run the transcode video', async () => { it('should run the transcode video', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.TRANSCODE_VIDEO }), await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.TRANSCODE_VIDEO }),
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.VIDEO_CONVERSION, data: { id: 'asset-1' } }); expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.VIDEO_CONVERSION, data: { id: 'asset-1' } }]);
}); });
}); });
server/src/domain/asset/asset.service.ts
+15 -12
View File
@@ -21,6 +21,7 @@ import {
IStorageRepository, IStorageRepository,
ISystemConfigRepository, ISystemConfigRepository,
ImmichReadStream, ImmichReadStream,
JobItem,
TimeBucketOptions, TimeBucketOptions,
} from '../repositories'; } from '../repositories';
import { StorageCore, StorageFolder } from '../storage'; import { StorageCore, StorageFolder } from '../storage';
@@ -449,9 +450,9 @@ export class AssetService {
); );
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
for (const asset of assets) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id: asset.id } }); assets.map((asset) => ({ name: JobName.ASSET_DELETION, data: { id: asset.id } })),
} );
} }
return true; return true;
@@ -504,9 +505,7 @@ export class AssetService {
await this.access.requirePermission(auth, Permission.ASSET_DELETE, ids); await this.access.requirePermission(auth, Permission.ASSET_DELETE, ids);
if (force) { if (force) {
for (const id of ids) { await this.jobRepository.queueAll(ids.map((id) => ({ name: JobName.ASSET_DELETION, data: { id } })));
await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id } });
}
} else { } else {
await this.assetRepository.softDeleteAll(ids); await this.assetRepository.softDeleteAll(ids);
this.communicationRepository.send(ClientEvent.ASSET_TRASH, auth.user.id, ids); this.communicationRepository.send(ClientEvent.ASSET_TRASH, auth.user.id, ids);
@@ -529,9 +528,9 @@ export class AssetService {
if (action == TrashAction.EMPTY_ALL) { if (action == TrashAction.EMPTY_ALL) {
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
for (const asset of assets) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id: asset.id } }); assets.map((asset) => ({ name: JobName.ASSET_DELETION, data: { id: asset.id } })),
} );
} }
return; return;
} }
@@ -566,21 +565,25 @@ export class AssetService {
async run(auth: AuthDto, dto: AssetJobsDto) { async run(auth: AuthDto, dto: AssetJobsDto) {
await this.access.requirePermission(auth, Permission.ASSET_UPDATE, dto.assetIds); await this.access.requirePermission(auth, Permission.ASSET_UPDATE, dto.assetIds);
const jobs: JobItem[] = [];
for (const id of dto.assetIds) { for (const id of dto.assetIds) {
switch (dto.name) { switch (dto.name) {
case AssetJobName.REFRESH_METADATA: case AssetJobName.REFRESH_METADATA:
await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id } }); jobs.push({ name: JobName.METADATA_EXTRACTION, data: { id } });
break; break;
case AssetJobName.REGENERATE_THUMBNAIL: case AssetJobName.REGENERATE_THUMBNAIL:
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id } }); jobs.push({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id } });
break; break;
case AssetJobName.TRANSCODE_VIDEO: case AssetJobName.TRANSCODE_VIDEO:
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { id } }); jobs.push({ name: JobName.VIDEO_CONVERSION, data: { id } });
break; break;
} }
} }
await this.jobRepository.queueAll(jobs);
} }
private async updateMetadata(dto: ISidecarWriteJob) { private async updateMetadata(dto: ISidecarWriteJob) {
server/src/domain/job/job.service.spec.ts
+21 -13
View File
@@ -55,12 +55,12 @@ describe(JobService.name, () => {
it('should run the scheduled jobs', async () => { it('should run the scheduled jobs', async () => {
await sut.handleNightlyJobs(); await sut.handleNightlyJobs();
expect(jobMock.queue.mock.calls).toEqual([ expect(jobMock.queueAll).toHaveBeenCalledWith([
[{ name: JobName.ASSET_DELETION_CHECK }], { name: JobName.ASSET_DELETION_CHECK },
[{ name: JobName.USER_DELETE_CHECK }], { name: JobName.USER_DELETE_CHECK },
[{ name: JobName.PERSON_CLEANUP }], { name: JobName.PERSON_CLEANUP },
[{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }], { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
[{ name: JobName.CLEAN_OLD_AUDIT_LOGS }], { name: JobName.CLEAN_OLD_AUDIT_LOGS },
]); ]);
}); });
}); });
@@ -138,6 +138,7 @@ describe(JobService.name, () => {
).rejects.toBeInstanceOf(BadRequestException); ).rejects.toBeInstanceOf(BadRequestException);
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
}); });
it('should handle a start video conversion command', async () => { it('should handle a start video conversion command', async () => {
@@ -204,6 +205,7 @@ describe(JobService.name, () => {
).rejects.toBeInstanceOf(BadRequestException); ).rejects.toBeInstanceOf(BadRequestException);
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
}); });
}); });
@@ -276,18 +278,18 @@ describe(JobService.name, () => {
item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } }, item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } },
jobs: [ jobs: [
JobName.GENERATE_WEBP_THUMBNAIL, JobName.GENERATE_WEBP_THUMBNAIL,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
JobName.ENCODE_CLIP, JobName.ENCODE_CLIP,
JobName.RECOGNIZE_FACES, JobName.RECOGNIZE_FACES,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
], ],
}, },
{ {
item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1', source: 'upload' } }, item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1', source: 'upload' } },
jobs: [ jobs: [
JobName.GENERATE_WEBP_THUMBNAIL, JobName.GENERATE_WEBP_THUMBNAIL,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
JobName.ENCODE_CLIP, JobName.ENCODE_CLIP,
JobName.RECOGNIZE_FACES, JobName.RECOGNIZE_FACES,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
JobName.VIDEO_CONVERSION, JobName.VIDEO_CONVERSION,
], ],
}, },
@@ -295,9 +297,9 @@ describe(JobService.name, () => {
item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-live-image', source: 'upload' } }, item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-live-image', source: 'upload' } },
jobs: [ jobs: [
JobName.GENERATE_WEBP_THUMBNAIL, JobName.GENERATE_WEBP_THUMBNAIL,
JobName.RECOGNIZE_FACES,
JobName.GENERATE_THUMBHASH_THUMBNAIL, JobName.GENERATE_THUMBHASH_THUMBNAIL,
JobName.ENCODE_CLIP, JobName.ENCODE_CLIP,
JobName.RECOGNIZE_FACES,
JobName.VIDEO_CONVERSION, JobName.VIDEO_CONVERSION,
], ],
}, },
@@ -327,9 +329,15 @@ describe(JobService.name, () => {
await jobMock.addHandler.mock.calls[0][2](item); await jobMock.addHandler.mock.calls[0][2](item);
await asyncTick(3); await asyncTick(3);
expect(jobMock.queue).toHaveBeenCalledTimes(jobs.length); if (jobs.length > 1) {
for (const jobName of jobs) { expect(jobMock.queueAll).toHaveBeenCalledWith(
expect(jobMock.queue).toHaveBeenCalledWith({ name: jobName, data: expect.anything() }); jobs.map((jobName) => ({ name: jobName, data: expect.anything() })),
);
} else {
expect(jobMock.queue).toHaveBeenCalledTimes(jobs.length);
for (const jobName of jobs) {
expect(jobMock.queue).toHaveBeenCalledWith({ name: jobName, data: expect.anything() });
}
} }
}); });
@@ -338,7 +346,7 @@ describe(JobService.name, () => {
await jobMock.addHandler.mock.calls[0][2](item); await jobMock.addHandler.mock.calls[0][2](item);
await asyncTick(3); await asyncTick(3);
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queueAll).not.toHaveBeenCalled();
}); });
} }
server/src/domain/job/job.service.ts
+17 -11
View File
@@ -158,11 +158,13 @@ export class JobService {
} }
async handleNightlyJobs() { async handleNightlyJobs() {
await this.jobRepository.queue({ name: JobName.ASSET_DELETION_CHECK }); await this.jobRepository.queueAll([
await this.jobRepository.queue({ name: JobName.USER_DELETE_CHECK }); { name: JobName.ASSET_DELETION_CHECK },
await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP }); { name: JobName.USER_DELETE_CHECK },
await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }); { name: JobName.PERSON_CLEANUP },
await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS }); { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
{ name: JobName.CLEAN_OLD_AUDIT_LOGS },
]);
} }
/** /**
@@ -210,19 +212,23 @@ export class JobService {
break; break;
case JobName.GENERATE_JPEG_THUMBNAIL: { case JobName.GENERATE_JPEG_THUMBNAIL: {
await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: item.data }); const jobs: JobItem[] = [
await this.jobRepository.queue({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: item.data }); { name: JobName.GENERATE_WEBP_THUMBNAIL, data: item.data },
await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: item.data }); { name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: item.data },
await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: item.data }); { name: JobName.ENCODE_CLIP, data: item.data },
{ name: JobName.RECOGNIZE_FACES, data: item.data },
];
const [asset] = await this.assetRepository.getByIds([item.data.id]); const [asset] = await this.assetRepository.getByIds([item.data.id]);
if (asset) { if (asset) {
if (asset.type === AssetType.VIDEO) { if (asset.type === AssetType.VIDEO) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: item.data }); jobs.push({ name: JobName.VIDEO_CONVERSION, data: item.data });
} else if (asset.livePhotoVideoId) { } else if (asset.livePhotoVideoId) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { id: asset.livePhotoVideoId } }); jobs.push({ name: JobName.VIDEO_CONVERSION, data: { id: asset.livePhotoVideoId } });
} }
} }
await this.jobRepository.queueAll(jobs);
break; break;
} }
server/src/domain/library/library.service.spec.ts
+38 -39
View File
@@ -135,18 +135,16 @@ describe(LibraryService.name, () => {
await sut.handleQueueAssetRefresh(mockLibraryJob); await sut.handleQueueAssetRefresh(mockLibraryJob);
expect(jobMock.queue.mock.calls).toEqual([ expect(jobMock.queueAll).toHaveBeenCalledWith([
[ {
{ name: JobName.LIBRARY_SCAN_ASSET,
name: JobName.LIBRARY_SCAN_ASSET, data: {
data: { id: libraryStub.externalLibrary1.id,
id: libraryStub.externalLibrary1.id, ownerId: libraryStub.externalLibrary1.owner.id,
ownerId: libraryStub.externalLibrary1.owner.id, assetPath: '/data/user1/photo.jpg',
assetPath: '/data/user1/photo.jpg', force: false,
force: false,
},
}, },
], },
]); ]);
}); });
@@ -420,6 +418,7 @@ describe(LibraryService.name, () => {
await expect(sut.handleAssetRefresh(mockLibraryJob)).resolves.toBe(true); await expect(sut.handleAssetRefresh(mockLibraryJob)).resolves.toBe(true);
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
}); });
it('should import an asset when mtime differs from db asset', async () => { it('should import an asset when mtime differs from db asset', async () => {
@@ -468,6 +467,7 @@ describe(LibraryService.name, () => {
expect(assetMock.save).toHaveBeenCalledWith({ id: assetStub.image.id, isOffline: true }); expect(assetMock.save).toHaveBeenCalledWith({ id: assetStub.image.id, isOffline: true });
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
}); });
it('should online a previously-offline asset', async () => { it('should online a previously-offline asset', async () => {
@@ -607,6 +607,7 @@ describe(LibraryService.name, () => {
); );
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
expect(libraryMock.softDelete).not.toHaveBeenCalled(); expect(libraryMock.softDelete).not.toHaveBeenCalled();
}); });
@@ -953,9 +954,9 @@ describe(LibraryService.name, () => {
libraryMock.getAllDeleted.mockResolvedValue([libraryStub.uploadLibrary1, libraryStub.externalLibrary1]); libraryMock.getAllDeleted.mockResolvedValue([libraryStub.uploadLibrary1, libraryStub.externalLibrary1]);
await expect(sut.handleQueueCleanup()).resolves.toBe(true); await expect(sut.handleQueueCleanup()).resolves.toBe(true);
expect(jobMock.queue.mock.calls).toEqual([ expect(jobMock.queueAll).toHaveBeenCalledWith([
[{ name: JobName.LIBRARY_DELETE, data: { id: libraryStub.uploadLibrary1.id } }], { name: JobName.LIBRARY_DELETE, data: { id: libraryStub.uploadLibrary1.id } },
[{ name: JobName.LIBRARY_DELETE, data: { id: libraryStub.externalLibrary1.id } }], { name: JobName.LIBRARY_DELETE, data: { id: libraryStub.externalLibrary1.id } },
]); ]);
}); });
}); });
@@ -1101,16 +1102,16 @@ describe(LibraryService.name, () => {
data: {}, data: {},
}, },
], ],
[ ]);
{ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.LIBRARY_SCAN, {
data: { name: JobName.LIBRARY_SCAN,
id: libraryStub.externalLibrary1.id, data: {
refreshModifiedFiles: true, id: libraryStub.externalLibrary1.id,
refreshAllFiles: false, refreshModifiedFiles: true,
}, refreshAllFiles: false,
}, },
], },
]); ]);
}); });
@@ -1126,16 +1127,16 @@ describe(LibraryService.name, () => {
data: {}, data: {},
}, },
], ],
[ ]);
{ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.LIBRARY_SCAN, {
data: { name: JobName.LIBRARY_SCAN,
id: libraryStub.externalLibrary1.id, data: {
refreshModifiedFiles: false, id: libraryStub.externalLibrary1.id,
refreshAllFiles: true, refreshModifiedFiles: false,
}, refreshAllFiles: true,
}, },
], },
]); ]);
}); });
}); });
@@ -1147,13 +1148,11 @@ describe(LibraryService.name, () => {
await expect(sut.handleOfflineRemoval({ id: libraryStub.externalLibrary1.id })).resolves.toBe(true); await expect(sut.handleOfflineRemoval({ id: libraryStub.externalLibrary1.id })).resolves.toBe(true);
expect(jobMock.queue.mock.calls).toEqual([ expect(jobMock.queueAll).toHaveBeenCalledWith([
[ {
{ name: JobName.ASSET_DELETION,
name: JobName.ASSET_DELETION, data: { id: assetStub.image1.id, fromExternal: true },
data: { id: assetStub.image1.id, fromExternal: true }, },
},
],
]); ]);
}); });
}); });
server/src/domain/library/library.service.ts
+24 -23
View File
@@ -94,9 +94,9 @@ export class LibraryService {
async handleQueueCleanup(): Promise<boolean> { async handleQueueCleanup(): Promise<boolean> {
this.logger.debug('Cleaning up any pending library deletions'); this.logger.debug('Cleaning up any pending library deletions');
const pendingDeletion = await this.repository.getAllDeleted(); const pendingDeletion = await this.repository.getAllDeleted();
for (const libraryToDelete of pendingDeletion) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.LIBRARY_DELETE, data: { id: libraryToDelete.id } }); pendingDeletion.map((libraryToDelete) => ({ name: JobName.LIBRARY_DELETE, data: { id: libraryToDelete.id } })),
} );
return true; return true;
} }
@@ -160,9 +160,9 @@ export class LibraryService {
// TODO use pagination // TODO use pagination
const assetIds = await this.repository.getAssetIds(job.id, true); const assetIds = await this.repository.getAssetIds(job.id, true);
this.logger.debug(`Will delete ${assetIds.length} asset(s) in library ${job.id}`); this.logger.debug(`Will delete ${assetIds.length} asset(s) in library ${job.id}`);
for (const assetId of assetIds) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id: assetId, fromExternal: true } }); assetIds.map((assetId) => ({ name: JobName.ASSET_DELETION, data: { id: assetId, fromExternal: true } })),
} );
if (assetIds.length === 0) { if (assetIds.length === 0) {
this.logger.log(`Deleting library ${job.id}`); this.logger.log(`Deleting library ${job.id}`);
@@ -333,16 +333,16 @@ export class LibraryService {
// Queue all library refresh // Queue all library refresh
const libraries = await this.repository.getAll(true, LibraryType.EXTERNAL); const libraries = await this.repository.getAll(true, LibraryType.EXTERNAL);
for (const library of libraries) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ libraries.map((library) => ({
name: JobName.LIBRARY_SCAN, name: JobName.LIBRARY_SCAN,
data: { data: {
id: library.id, id: library.id,
refreshModifiedFiles: !job.force, refreshModifiedFiles: !job.force,
refreshAllFiles: job.force ?? false, refreshAllFiles: job.force ?? false,
}, },
}); })),
} );
return true; return true;
} }
@@ -353,9 +353,9 @@ export class LibraryService {
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
this.logger.debug(`Removing ${assets.length} offline assets`); this.logger.debug(`Removing ${assets.length} offline assets`);
for (const asset of assets) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id: asset.id, fromExternal: true } }); assets.map((asset) => ({ name: JobName.ASSET_DELETION, data: { id: asset.id, fromExternal: true } })),
} );
} }
return true; return true;
@@ -411,16 +411,17 @@ export class LibraryService {
this.logger.debug(`Will import ${filteredPaths.length} new asset(s)`); this.logger.debug(`Will import ${filteredPaths.length} new asset(s)`);
} }
for (const assetPath of filteredPaths) { await this.jobRepository.queueAll(
const libraryJobData: ILibraryFileJob = { filteredPaths.map((assetPath) => ({
id: job.id, name: JobName.LIBRARY_SCAN_ASSET,
assetPath: path.normalize(assetPath), data: {
ownerId: library.ownerId, id: job.id,
force: job.refreshAllFiles ?? false, assetPath: path.normalize(assetPath),
}; ownerId: library.ownerId,
force: job.refreshAllFiles ?? false,
await this.jobRepository.queue({ name: JobName.LIBRARY_SCAN_ASSET, data: libraryJobData }); },
} })),
);
} }
await this.repository.update({ id: job.id, refreshedAt: new Date() }); await this.repository.update({ id: job.id, refreshedAt: new Date() });
server/src/domain/media/media.service.spec.ts
+49 -33
View File
@@ -77,17 +77,21 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).toHaveBeenCalled(); expect(assetMock.getAll).toHaveBeenCalled();
expect(assetMock.getWithout).not.toHaveBeenCalled(); expect(assetMock.getWithout).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.GENERATE_JPEG_THUMBNAIL, {
data: { id: assetStub.image.id }, name: JobName.GENERATE_JPEG_THUMBNAIL,
}); data: { id: assetStub.image.id },
},
]);
expect(personMock.getAll).toHaveBeenCalled(); expect(personMock.getAll).toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).not.toHaveBeenCalled(); expect(personMock.getAllWithoutThumbnail).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.GENERATE_PERSON_THUMBNAIL, {
data: { id: personStub.newThumbnail.id }, name: JobName.GENERATE_PERSON_THUMBNAIL,
}); data: { id: personStub.newThumbnail.id },
},
]);
}); });
it('should queue all people with missing thumbnail path', async () => { it('should queue all people with missing thumbnail path', async () => {
@@ -106,12 +110,14 @@ describe(MediaService.name, () => {
expect(personMock.getAll).not.toHaveBeenCalled(); expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled(); expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
expect(personMock.getRandomFace).toHaveBeenCalled(); expect(personMock.getRandomFace).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.GENERATE_PERSON_THUMBNAIL, {
data: { name: JobName.GENERATE_PERSON_THUMBNAIL,
id: personStub.newThumbnail.id, data: {
id: personStub.newThumbnail.id,
},
}, },
}); ]);
}); });
it('should queue all assets with missing resize path', async () => { it('should queue all assets with missing resize path', async () => {
@@ -125,10 +131,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled(); expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL); expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.GENERATE_JPEG_THUMBNAIL, {
data: { id: assetStub.image.id }, name: JobName.GENERATE_JPEG_THUMBNAIL,
}); data: { id: assetStub.image.id },
},
]);
expect(personMock.getAll).not.toHaveBeenCalled(); expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled(); expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
@@ -145,10 +153,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled(); expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL); expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.GENERATE_WEBP_THUMBNAIL, {
data: { id: assetStub.image.id }, name: JobName.GENERATE_WEBP_THUMBNAIL,
}); data: { id: assetStub.image.id },
},
]);
expect(personMock.getAll).not.toHaveBeenCalled(); expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled(); expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
@@ -165,10 +175,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled(); expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL); expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.GENERATE_THUMBHASH_THUMBNAIL, {
data: { id: assetStub.image.id }, name: JobName.GENERATE_THUMBHASH_THUMBNAIL,
}); data: { id: assetStub.image.id },
},
]);
expect(personMock.getAll).not.toHaveBeenCalled(); expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled(); expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
@@ -388,10 +400,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { type: AssetType.VIDEO }); expect(assetMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { type: AssetType.VIDEO });
expect(assetMock.getWithout).not.toHaveBeenCalled(); expect(assetMock.getWithout).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.VIDEO_CONVERSION, {
data: { id: assetStub.video.id }, name: JobName.VIDEO_CONVERSION,
}); data: { id: assetStub.video.id },
},
]);
}); });
it('should queue all video assets without encoded videos', async () => { it('should queue all video assets without encoded videos', async () => {
@@ -404,10 +418,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled(); expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.ENCODED_VIDEO); expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.ENCODED_VIDEO);
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.VIDEO_CONVERSION, {
data: { id: assetStub.video.id }, name: JobName.VIDEO_CONVERSION,
}); data: { id: assetStub.video.id },
},
]);
}); });
}); });
server/src/domain/media/media.service.ts
+21 -13
View File
@@ -21,6 +21,7 @@ import {
IPersonRepository, IPersonRepository,
IStorageRepository, IStorageRepository,
ISystemConfigRepository, ISystemConfigRepository,
JobItem,
VideoCodecHWConfig, VideoCodecHWConfig,
VideoStreamInfo, VideoStreamInfo,
WithoutProperty, WithoutProperty,
@@ -74,22 +75,27 @@ export class MediaService {
}); });
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
const jobs: JobItem[] = [];
for (const asset of assets) { for (const asset of assets) {
if (!asset.resizePath || force) { if (!asset.resizePath || force) {
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } }); jobs.push({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } });
continue; continue;
} }
if (!asset.webpPath) { if (!asset.webpPath) {
await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { id: asset.id } }); jobs.push({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { id: asset.id } });
} }
if (!asset.thumbhash) { if (!asset.thumbhash) {
await this.jobRepository.queue({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: { id: asset.id } }); jobs.push({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: { id: asset.id } });
} }
} }
await this.jobRepository.queueAll(jobs);
} }
const people = force ? await this.personRepository.getAll() : await this.personRepository.getAllWithoutThumbnail(); const people = force ? await this.personRepository.getAll() : await this.personRepository.getAllWithoutThumbnail();
const jobs: JobItem[] = [];
for (const person of people) { for (const person of people) {
if (!person.faceAssetId) { if (!person.faceAssetId) {
const face = await this.personRepository.getRandomFace(person.id); const face = await this.personRepository.getRandomFace(person.id);
@@ -100,9 +106,11 @@ export class MediaService {
await this.personRepository.update({ id: person.id, faceAssetId: face.assetId }); await this.personRepository.update({ id: person.id, faceAssetId: face.assetId });
} }
await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } }); jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } });
} }
await this.jobRepository.queueAll(jobs);
return true; return true;
} }
@@ -118,15 +126,15 @@ export class MediaService {
} }
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
for (const asset of assets) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.MIGRATE_ASSET, data: { id: asset.id } }); assets.map((asset) => ({ name: JobName.MIGRATE_ASSET, data: { id: asset.id } })),
} );
} }
const people = await this.personRepository.getAll(); const people = await this.personRepository.getAll();
for (const person of people) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.MIGRATE_PERSON, data: { id: person.id } }); people.map((person) => ({ name: JobName.MIGRATE_PERSON, data: { id: person.id } })),
} );
return true; return true;
} }
@@ -224,9 +232,9 @@ export class MediaService {
}); });
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
for (const asset of assets) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { id: asset.id } }); assets.map((asset) => ({ name: JobName.VIDEO_CONVERSION, data: { id: asset.id } })),
} );
} }
return true; return true;
server/src/domain/metadata/metadata.service.spec.ts
+25 -16
View File
@@ -208,10 +208,12 @@ describe(MetadataService.name, () => {
await expect(sut.handleQueueMetadataExtraction({ force: false })).resolves.toBe(true); await expect(sut.handleQueueMetadataExtraction({ force: false })).resolves.toBe(true);
expect(assetMock.getWithout).toHaveBeenCalled(); expect(assetMock.getWithout).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.METADATA_EXTRACTION, {
data: { id: assetStub.image.id }, name: JobName.METADATA_EXTRACTION,
}); data: { id: assetStub.image.id },
},
]);
}); });
it('should queue metadata extraction for all assets', async () => { it('should queue metadata extraction for all assets', async () => {
@@ -219,10 +221,12 @@ describe(MetadataService.name, () => {
await expect(sut.handleQueueMetadataExtraction({ force: true })).resolves.toBe(true); await expect(sut.handleQueueMetadataExtraction({ force: true })).resolves.toBe(true);
expect(assetMock.getAll).toHaveBeenCalled(); expect(assetMock.getAll).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.METADATA_EXTRACTION, {
data: { id: assetStub.image.id }, name: JobName.METADATA_EXTRACTION,
}); data: { id: assetStub.image.id },
},
]);
}); });
}); });
@@ -320,6 +324,7 @@ describe(MetadataService.name, () => {
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
expect(storageMock.writeFile).not.toHaveBeenCalled(); expect(storageMock.writeFile).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
expect(assetMock.save).not.toHaveBeenCalledWith( expect(assetMock.save).not.toHaveBeenCalledWith(
expect.objectContaining({ assetType: AssetType.VIDEO, isVisible: false }), expect.objectContaining({ assetType: AssetType.VIDEO, isVisible: false }),
); );
@@ -512,10 +517,12 @@ describe(MetadataService.name, () => {
expect(assetMock.getWith).toHaveBeenCalledWith({ take: 1000, skip: 0 }, WithProperty.SIDECAR); expect(assetMock.getWith).toHaveBeenCalledWith({ take: 1000, skip: 0 }, WithProperty.SIDECAR);
expect(assetMock.getWithout).not.toHaveBeenCalled(); expect(assetMock.getWithout).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.SIDECAR_SYNC, {
data: { id: assetStub.sidecar.id }, name: JobName.SIDECAR_SYNC,
}); data: { id: assetStub.sidecar.id },
},
]);
}); });
it('should queue assets without sidecar files', async () => { it('should queue assets without sidecar files', async () => {
@@ -525,10 +532,12 @@ describe(MetadataService.name, () => {
expect(assetMock.getWithout).toHaveBeenCalledWith({ take: 1000, skip: 0 }, WithoutProperty.SIDECAR); expect(assetMock.getWithout).toHaveBeenCalledWith({ take: 1000, skip: 0 }, WithoutProperty.SIDECAR);
expect(assetMock.getWith).not.toHaveBeenCalled(); expect(assetMock.getWith).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.SIDECAR_DISCOVERY, {
data: { id: assetStub.image.id }, name: JobName.SIDECAR_DISCOVERY,
}); data: { id: assetStub.image.id },
},
]);
}); });
}); });
server/src/domain/metadata/metadata.service.ts
+9 -7
View File
@@ -196,9 +196,9 @@ export class MetadataService {
}); });
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
for (const asset of assets) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id } }); assets.map((asset) => ({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id } })),
} );
} }
return true; return true;
@@ -264,10 +264,12 @@ export class MetadataService {
}); });
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
for (const asset of assets) { await this.jobRepository.queueAll(
const name = force ? JobName.SIDECAR_SYNC : JobName.SIDECAR_DISCOVERY; assets.map((asset) => ({
await this.jobRepository.queue({ name, data: { id: asset.id } }); name: force ? JobName.SIDECAR_SYNC : JobName.SIDECAR_DISCOVERY,
} data: { id: asset.id },
})),
);
} }
return true; return true;
server/src/domain/person/person.service.spec.ts
+39 -21
View File
@@ -286,6 +286,7 @@ describe(PersonService.name, () => {
expect(personMock.getById).toHaveBeenCalledWith('person-1'); expect(personMock.getById).toHaveBeenCalledWith('person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', birthDate: new Date('1976-06-30') }); expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', birthDate: new Date('1976-06-30') });
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1'])); expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1']));
}); });
@@ -403,6 +404,7 @@ describe(PersonService.name, () => {
}), }),
).rejects.toBeInstanceOf(BadRequestException); ).rejects.toBeInstanceOf(BadRequestException);
expect(jobMock.queue).not.toHaveBeenCalledWith(); expect(jobMock.queue).not.toHaveBeenCalledWith();
expect(jobMock.queueAll).not.toHaveBeenCalledWith();
}); });
it('should reassign a face', async () => { it('should reassign a face', async () => {
accessMock.person.checkOwnerAccess.mockResolvedValue(new Set([personStub.withName.id])); accessMock.person.checkOwnerAccess.mockResolvedValue(new Set([personStub.withName.id]));
@@ -417,10 +419,12 @@ describe(PersonService.name, () => {
}), }),
).resolves.toEqual([personStub.noName]); ).resolves.toEqual([personStub.noName]);
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.GENERATE_PERSON_THUMBNAIL, {
data: { id: personStub.newThumbnail.id }, name: JobName.GENERATE_PERSON_THUMBNAIL,
}); data: { id: personStub.newThumbnail.id },
},
]);
}); });
}); });
@@ -452,10 +456,12 @@ describe(PersonService.name, () => {
it('should change person feature photo', async () => { it('should change person feature photo', async () => {
personMock.getRandomFace.mockResolvedValue(faceStub.primaryFace1); personMock.getRandomFace.mockResolvedValue(faceStub.primaryFace1);
await sut.createNewFeaturePhoto([personStub.newThumbnail.id]); await sut.createNewFeaturePhoto([personStub.newThumbnail.id]);
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.GENERATE_PERSON_THUMBNAIL, {
data: { id: personStub.newThumbnail.id }, name: JobName.GENERATE_PERSON_THUMBNAIL,
}); data: { id: personStub.newThumbnail.id },
},
]);
}); });
}); });
@@ -480,6 +486,7 @@ describe(PersonService.name, () => {
}); });
expect(jobMock.queue).not.toHaveBeenCalledWith(); expect(jobMock.queue).not.toHaveBeenCalledWith();
expect(jobMock.queueAll).not.toHaveBeenCalledWith();
}); });
it('should fail if user has not the correct permissions on the asset', async () => { it('should fail if user has not the correct permissions on the asset', async () => {
@@ -495,6 +502,7 @@ describe(PersonService.name, () => {
).rejects.toBeInstanceOf(BadRequestException); ).rejects.toBeInstanceOf(BadRequestException);
expect(jobMock.queue).not.toHaveBeenCalledWith(); expect(jobMock.queue).not.toHaveBeenCalledWith();
expect(jobMock.queueAll).not.toHaveBeenCalledWith();
}); });
}); });
@@ -542,7 +550,9 @@ describe(PersonService.name, () => {
await sut.handlePersonCleanup(); await sut.handlePersonCleanup();
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.PERSON_DELETE, data: { id: personStub.noName.id } }); expect(jobMock.queueAll).toHaveBeenCalledWith([
{ name: JobName.PERSON_DELETE, data: { id: personStub.noName.id } },
]);
}); });
}); });
@@ -552,6 +562,7 @@ describe(PersonService.name, () => {
await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(true); await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(true);
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
expect(configMock.load).toHaveBeenCalled(); expect(configMock.load).toHaveBeenCalled();
}); });
@@ -563,10 +574,12 @@ describe(PersonService.name, () => {
await sut.handleQueueRecognizeFaces({}); await sut.handleQueueRecognizeFaces({});
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.FACES); expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.FACES);
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.RECOGNIZE_FACES, {
data: { id: assetStub.image.id }, name: JobName.RECOGNIZE_FACES,
}); data: { id: assetStub.image.id },
},
]);
}); });
it('should queue all assets', async () => { it('should queue all assets', async () => {
@@ -580,14 +593,18 @@ describe(PersonService.name, () => {
await sut.handleQueueRecognizeFaces({ force: true }); await sut.handleQueueRecognizeFaces({ force: true });
expect(assetMock.getAll).toHaveBeenCalled(); expect(assetMock.getAll).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ expect(jobMock.queueAll).toHaveBeenCalledWith([
name: JobName.RECOGNIZE_FACES, {
data: { id: assetStub.image.id }, name: JobName.RECOGNIZE_FACES,
}); data: { id: assetStub.image.id },
expect(jobMock.queue).toHaveBeenCalledWith({ },
name: JobName.PERSON_DELETE, ]);
data: { id: personStub.withName.id }, expect(jobMock.queueAll).toHaveBeenCalledWith([
}); {
name: JobName.PERSON_DELETE,
data: { id: personStub.withName.id },
},
]);
}); });
}); });
@@ -644,6 +661,7 @@ describe(PersonService.name, () => {
); );
expect(personMock.createFace).not.toHaveBeenCalled(); expect(personMock.createFace).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
expect(assetMock.upsertJobStatus).toHaveBeenCalledWith({ expect(assetMock.upsertJobStatus).toHaveBeenCalledWith({
assetId: assetStub.image.id, assetId: assetStub.image.id,
server/src/domain/person/person.service.ts
+16 -15
View File
@@ -22,6 +22,7 @@ import {
ISmartInfoRepository, ISmartInfoRepository,
IStorageRepository, IStorageRepository,
ISystemConfigRepository, ISystemConfigRepository,
JobItem,
UpdateFacesData, UpdateFacesData,
WithoutProperty, WithoutProperty,
} from '../repositories'; } from '../repositories';
@@ -153,6 +154,8 @@ export class PersonService {
this.logger.debug( this.logger.debug(
`Changing feature photos for ${changeFeaturePhoto.length} ${changeFeaturePhoto.length > 1 ? 'people' : 'person'}`, `Changing feature photos for ${changeFeaturePhoto.length} ${changeFeaturePhoto.length > 1 ? 'people' : 'person'}`,
); );
const jobs: JobItem[] = [];
for (const personId of changeFeaturePhoto) { for (const personId of changeFeaturePhoto) {
const assetFace = await this.repository.getRandomFace(personId); const assetFace = await this.repository.getRandomFace(personId);
@@ -161,15 +164,11 @@ export class PersonService {
id: personId, id: personId,
faceAssetId: assetFace.id, faceAssetId: assetFace.id,
}); });
jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: personId } });
await this.jobRepository.queue({
name: JobName.GENERATE_PERSON_THUMBNAIL,
data: {
id: personId,
},
});
} }
} }
await this.jobRepository.queueAll(jobs);
} }
async getById(auth: AuthDto, id: string): Promise<PersonResponseDto> { async getById(auth: AuthDto, id: string): Promise<PersonResponseDto> {
@@ -270,8 +269,10 @@ export class PersonService {
const people = await this.repository.getAllWithoutFaces(); const people = await this.repository.getAllWithoutFaces();
for (const person of people) { for (const person of people) {
this.logger.debug(`Person ${person.name || person.id} no longer has any faces, deleting.`); this.logger.debug(`Person ${person.name || person.id} no longer has any faces, deleting.`);
await this.jobRepository.queue({ name: JobName.PERSON_DELETE, data: { id: person.id } });
} }
await this.jobRepository.queueAll(
people.map((person) => ({ name: JobName.PERSON_DELETE, data: { id: person.id } })),
);
return true; return true;
} }
@@ -290,16 +291,16 @@ export class PersonService {
if (force) { if (force) {
const people = await this.repository.getAll(); const people = await this.repository.getAll();
for (const person of people) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.PERSON_DELETE, data: { id: person.id } }); people.map((person) => ({ name: JobName.PERSON_DELETE, data: { id: person.id } })),
} );
this.logger.debug(`Deleted ${people.length} people`); this.logger.debug(`Deleted ${people.length} people`);
} }
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
for (const asset of assets) { await this.jobRepository.queueAll(
await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: { id: asset.id } }); assets.map((asset) => ({ name: JobName.RECOGNIZE_FACES, data: { id: asset.id } })),
} );
} }
return true; return true;
@@ -333,7 +334,7 @@ export class PersonService {
for (const { embedding, ...rest } of faces) { for (const { embedding, ...rest } of faces) {
const matches = await this.smartInfoRepository.searchFaces({ const matches = await this.smartInfoRepository.searchFaces({
ownerId: asset.ownerId, userIds: [asset.ownerId],
embedding, embedding,
numResults: 1, numResults: 1,
maxDistance: machineLearning.facialRecognition.maxDistance, maxDistance: machineLearning.facialRecognition.maxDistance,
server/src/domain/repositories/asset.repository.ts
+1 -1
View File
@@ -199,5 +199,5 @@ export interface IAssetRepository {
search(options: AssetSearchOptions): Promise<AssetEntity[]>; search(options: AssetSearchOptions): Promise<AssetEntity[]>;
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>; getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>; getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
searchMetadata(query: string, userId: string, options: MetadataSearchOptions): Promise<AssetEntity[]>; searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise<AssetEntity[]>;
} }
server/src/domain/repositories/job.repository.ts
+1
View File
@@ -103,6 +103,7 @@ export interface IJobRepository {
deleteCronJob(name: string): void; deleteCronJob(name: string): void;
setConcurrency(queueName: QueueName, concurrency: number): void; setConcurrency(queueName: QueueName, concurrency: number): void;
queue(item: JobItem): Promise<void>; queue(item: JobItem): Promise<void>;
queueAll(items: JobItem[]): Promise<void>;
pause(name: QueueName): Promise<void>; pause(name: QueueName): Promise<void>;
resume(name: QueueName): Promise<void>; resume(name: QueueName): Promise<void>;
empty(name: QueueName): Promise<void>; empty(name: QueueName): Promise<void>;
server/src/domain/repositories/smart-info.repository.ts
+1 -1
View File
@@ -5,7 +5,7 @@ export const ISmartInfoRepository = 'ISmartInfoRepository';
export type Embedding = number[]; export type Embedding = number[];
export interface EmbeddingSearch { export interface EmbeddingSearch {
ownerId: string; userIds: string[];
embedding: Embedding; embedding: Embedding;
numResults: number; numResults: number;
maxDistance?: number; maxDistance?: number;
server/src/domain/search/search.service.spec.ts
+9 -3
View File
@@ -4,6 +4,7 @@ import {
authStub, authStub,
newAssetRepositoryMock, newAssetRepositoryMock,
newMachineLearningRepositoryMock, newMachineLearningRepositoryMock,
newPartnerRepositoryMock,
newPersonRepositoryMock, newPersonRepositoryMock,
newSmartInfoRepositoryMock, newSmartInfoRepositoryMock,
newSystemConfigRepositoryMock, newSystemConfigRepositoryMock,
@@ -13,6 +14,7 @@ import { mapAsset } from '../asset';
import { import {
IAssetRepository, IAssetRepository,
IMachineLearningRepository, IMachineLearningRepository,
IPartnerRepository,
IPersonRepository, IPersonRepository,
ISmartInfoRepository, ISmartInfoRepository,
ISystemConfigRepository, ISystemConfigRepository,
@@ -29,6 +31,7 @@ describe(SearchService.name, () => {
let machineMock: jest.Mocked<IMachineLearningRepository>; let machineMock: jest.Mocked<IMachineLearningRepository>;
let personMock: jest.Mocked<IPersonRepository>; let personMock: jest.Mocked<IPersonRepository>;
let smartInfoMock: jest.Mocked<ISmartInfoRepository>; let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
let partnerMock: jest.Mocked<IPartnerRepository>;
beforeEach(() => { beforeEach(() => {
assetMock = newAssetRepositoryMock(); assetMock = newAssetRepositoryMock();
@@ -36,7 +39,8 @@ describe(SearchService.name, () => {
machineMock = newMachineLearningRepositoryMock(); machineMock = newMachineLearningRepositoryMock();
personMock = newPersonRepositoryMock(); personMock = newPersonRepositoryMock();
smartInfoMock = newSmartInfoRepositoryMock(); smartInfoMock = newSmartInfoRepositoryMock();
sut = new SearchService(configMock, machineMock, personMock, smartInfoMock, assetMock); partnerMock = newPartnerRepositoryMock();
sut = new SearchService(configMock, machineMock, personMock, smartInfoMock, assetMock, partnerMock);
}); });
it('should work', () => { it('should work', () => {
@@ -87,6 +91,7 @@ describe(SearchService.name, () => {
it('should search by metadata if `clip` option is false', async () => { it('should search by metadata if `clip` option is false', async () => {
const dto: SearchDto = { q: 'test query', clip: false }; const dto: SearchDto = { q: 'test query', clip: false };
assetMock.searchMetadata.mockResolvedValueOnce([assetStub.image]); assetMock.searchMetadata.mockResolvedValueOnce([assetStub.image]);
partnerMock.getAll.mockResolvedValueOnce([]);
const expectedResponse = { const expectedResponse = {
albums: { albums: {
total: 0, total: 0,
@@ -105,7 +110,7 @@ describe(SearchService.name, () => {
const result = await sut.search(authStub.user1, dto); const result = await sut.search(authStub.user1, dto);
expect(result).toEqual(expectedResponse); expect(result).toEqual(expectedResponse);
expect(assetMock.searchMetadata).toHaveBeenCalledWith(dto.q, authStub.user1.user.id, { numResults: 250 }); expect(assetMock.searchMetadata).toHaveBeenCalledWith(dto.q, [authStub.user1.user.id], { numResults: 250 });
expect(smartInfoMock.searchCLIP).not.toHaveBeenCalled(); expect(smartInfoMock.searchCLIP).not.toHaveBeenCalled();
}); });
@@ -114,6 +119,7 @@ describe(SearchService.name, () => {
const embedding = [1, 2, 3]; const embedding = [1, 2, 3];
smartInfoMock.searchCLIP.mockResolvedValueOnce([assetStub.image]); smartInfoMock.searchCLIP.mockResolvedValueOnce([assetStub.image]);
machineMock.encodeText.mockResolvedValueOnce(embedding); machineMock.encodeText.mockResolvedValueOnce(embedding);
partnerMock.getAll.mockResolvedValueOnce([]);
const expectedResponse = { const expectedResponse = {
albums: { albums: {
total: 0, total: 0,
@@ -133,7 +139,7 @@ describe(SearchService.name, () => {
expect(result).toEqual(expectedResponse); expect(result).toEqual(expectedResponse);
expect(smartInfoMock.searchCLIP).toHaveBeenCalledWith({ expect(smartInfoMock.searchCLIP).toHaveBeenCalledWith({
ownerId: authStub.user1.user.id, userIds: [authStub.user1.user.id],
embedding, embedding,
numResults: 100, numResults: 100,
}); });
server/src/domain/search/search.service.ts
+15 -2
View File
@@ -7,6 +7,7 @@ import { PersonResponseDto } from '../person';
import { import {
IAssetRepository, IAssetRepository,
IMachineLearningRepository, IMachineLearningRepository,
IPartnerRepository,
IPersonRepository, IPersonRepository,
ISmartInfoRepository, ISmartInfoRepository,
ISystemConfigRepository, ISystemConfigRepository,
@@ -28,6 +29,7 @@ export class SearchService {
@Inject(IPersonRepository) private personRepository: IPersonRepository, @Inject(IPersonRepository) private personRepository: IPersonRepository,
@Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository, @Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository, @Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
) { ) {
this.configCore = SystemConfigCore.create(configRepository); this.configCore = SystemConfigCore.create(configRepository);
} }
@@ -64,6 +66,7 @@ export class SearchService {
throw new Error('CLIP is not enabled'); throw new Error('CLIP is not enabled');
} }
const strategy = dto.clip ? SearchStrategy.CLIP : SearchStrategy.TEXT; const strategy = dto.clip ? SearchStrategy.CLIP : SearchStrategy.TEXT;
const userIds = await this.getUserIdsToSearch(auth);
let assets: AssetEntity[] = []; let assets: AssetEntity[] = [];
@@ -74,10 +77,10 @@ export class SearchService {
{ text: query }, { text: query },
machineLearning.clip, machineLearning.clip,
); );
assets = await this.smartInfoRepository.searchCLIP({ ownerId: auth.user.id, embedding, numResults: 100 }); assets = await this.smartInfoRepository.searchCLIP({ userIds: userIds, embedding, numResults: 100 });
break; break;
case SearchStrategy.TEXT: case SearchStrategy.TEXT:
assets = await this.assetRepository.searchMetadata(query, auth.user.id, { numResults: 250 }); assets = await this.assetRepository.searchMetadata(query, userIds, { numResults: 250 });
default: default:
break; break;
} }
@@ -97,4 +100,14 @@ export class SearchService {
}, },
}; };
} }
private async getUserIdsToSearch(auth: AuthDto): Promise<string[]> {
const userIds: string[] = [auth.user.id];
const partners = await this.partnerRepository.getAll(auth.user.id);
const partnersIds = partners
.filter((partner) => partner.sharedBy && partner.inTimeline)
.map((partner) => partner.sharedById);
userIds.push(...partnersIds);
return userIds;
}
} }
server/src/domain/smart-info/smart-info.service.spec.ts
+2 -2
View File
@@ -69,7 +69,7 @@ describe(SmartInfoService.name, () => {
await sut.handleQueueEncodeClip({ force: false }); await sut.handleQueueEncodeClip({ force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.ENCODE_CLIP, data: { id: assetStub.image.id } }); expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.ENCODE_CLIP, data: { id: assetStub.image.id } }]);
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.CLIP_ENCODING); expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.CLIP_ENCODING);
}); });
@@ -81,7 +81,7 @@ describe(SmartInfoService.name, () => {
await sut.handleQueueEncodeClip({ force: true }); await sut.handleQueueEncodeClip({ force: true });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.ENCODE_CLIP, data: { id: assetStub.image.id } }); expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.ENCODE_CLIP, data: { id: assetStub.image.id } }]);
expect(assetMock.getAll).toHaveBeenCalled(); expect(assetMock.getAll).toHaveBeenCalled();
}); });
}); });
server/src/domain/smart-info/smart-info.service.ts
+1 -3
View File
@@ -64,9 +64,7 @@ export class SmartInfoService {
}); });
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
for (const asset of assets) { await this.jobRepository.queueAll(assets.map((asset) => ({ name: JobName.ENCODE_CLIP, data: { id: asset.id } })));
await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: { id: asset.id } });
}
} }
return true; return true;
server/src/domain/user/user.core.ts
+1 -1
View File
@@ -97,7 +97,7 @@ export class UserCore {
payload.password = await this.cryptoRepository.hashBcrypt(payload.password, SALT_ROUNDS); payload.password = await this.cryptoRepository.hashBcrypt(payload.password, SALT_ROUNDS);
} }
if (payload.storageLabel) { if (payload.storageLabel) {
payload.storageLabel = sanitize(payload.storageLabel); payload.storageLabel = sanitize(payload.storageLabel.replace(/\./g, ''));
} }
const userEntity = await this.userRepository.create(payload); const userEntity = await this.userRepository.create(payload);
await this.libraryRepository.create({ await this.libraryRepository.create({
server/src/domain/user/user.service.spec.ts
+6 -3
View File
@@ -342,7 +342,7 @@ describe(UserService.name, () => {
userMock.update.mockResolvedValue({ ...userStub.admin, profileImagePath: file.path }); userMock.update.mockResolvedValue({ ...userStub.admin, profileImagePath: file.path });
await sut.createProfileImage(authStub.admin, file); await sut.createProfileImage(authStub.admin, file);
await expect(jobMock.queue.mock.calls).toEqual([[{ name: JobName.DELETE_FILES, data: { files } }]]); expect(jobMock.queue.mock.calls).toEqual([[{ name: JobName.DELETE_FILES, data: { files } }]]);
}); });
it('should not delete the profile image if it has not been set', async () => { it('should not delete the profile image if it has not been set', async () => {
@@ -352,6 +352,7 @@ describe(UserService.name, () => {
await sut.createProfileImage(authStub.admin, file); await sut.createProfileImage(authStub.admin, file);
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
}); });
}); });
@@ -361,6 +362,7 @@ describe(UserService.name, () => {
await expect(sut.deleteProfileImage(authStub.admin)).rejects.toBeInstanceOf(BadRequestException); await expect(sut.deleteProfileImage(authStub.admin)).rejects.toBeInstanceOf(BadRequestException);
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
}); });
it('should delete the profile image if user has one', async () => { it('should delete the profile image if user has one', async () => {
@@ -368,7 +370,7 @@ describe(UserService.name, () => {
const files = [userStub.profilePath.profileImagePath]; const files = [userStub.profilePath.profileImagePath];
await sut.deleteProfileImage(authStub.admin); await sut.deleteProfileImage(authStub.admin);
await expect(jobMock.queue.mock.calls).toEqual([[{ name: JobName.DELETE_FILES, data: { files } }]]); expect(jobMock.queue.mock.calls).toEqual([[{ name: JobName.DELETE_FILES, data: { files } }]]);
}); });
}); });
@@ -456,6 +458,7 @@ describe(UserService.name, () => {
expect(userMock.getDeletedUsers).toHaveBeenCalled(); expect(userMock.getDeletedUsers).toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled(); expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).toHaveBeenCalledWith([]);
}); });
it('should queue user ready for deletion', async () => { it('should queue user ready for deletion', async () => {
@@ -465,7 +468,7 @@ describe(UserService.name, () => {
await sut.handleUserDeleteCheck(); await sut.handleUserDeleteCheck();
expect(userMock.getDeletedUsers).toHaveBeenCalled(); expect(userMock.getDeletedUsers).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.USER_DELETION, data: { id: user.id } }); expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.USER_DELETION, data: { id: user.id } }]);
}); });
}); });
server/src/domain/user/user.service.ts
+5 -6
View File
@@ -129,12 +129,11 @@ export class UserService {
async handleUserDeleteCheck() { async handleUserDeleteCheck() {
const users = await this.userRepository.getDeletedUsers(); const users = await this.userRepository.getDeletedUsers();
for (const user of users) { await this.jobRepository.queueAll(
if (this.isReadyForDeletion(user)) { users.flatMap((user) =>
await this.jobRepository.queue({ name: JobName.USER_DELETION, data: { id: user.id } }); this.isReadyForDeletion(user) ? [{ name: JobName.USER_DELETION, data: { id: user.id } }] : [],
} ),
} );
return true; return true;
} }
server/src/infra/repositories/asset.repository.ts
+7 -3
View File
@@ -804,10 +804,14 @@ export class AssetRepository implements IAssetRepository {
return builder; return builder;
} }
@GenerateSql({ params: [DummyValue.STRING, DummyValue.UUID, { numResults: 250 }] }) @GenerateSql({ params: [DummyValue.STRING, [DummyValue.UUID], { numResults: 250 }] })
async searchMetadata(query: string, ownerId: string, { numResults }: MetadataSearchOptions): Promise<AssetEntity[]> { async searchMetadata(
query: string,
userIds: string[],
{ numResults }: MetadataSearchOptions,
): Promise<AssetEntity[]> {
const rows = await this.getBuilder({ const rows = await this.getBuilder({
userIds: [ownerId], userIds: userIds,
exifInfo: false, exifInfo: false,
isArchived: false, isArchived: false,
}) })
server/src/infra/repositories/job.repository.ts
+25 -6
View File
@@ -34,7 +34,7 @@ export class JobRepository implements IJobRepository {
} }
addCronJob(name: string, expression: string, onTick: () => void, start = true): void { addCronJob(name: string, expression: string, onTick: () => void, start = true): void {
const job = new CronJob( const job = new CronJob<null, null>(
expression, expression,
onTick, onTick,
// function to run onComplete // function to run onComplete
@@ -116,12 +116,31 @@ export class JobRepository implements IJobRepository {
) as unknown as Promise<JobCounts>; ) as unknown as Promise<JobCounts>;
} }
async queue(item: JobItem): Promise<void> { async queueAll(items: JobItem[]): Promise<void> {
const jobName = item.name; if (!items.length) {
const jobData = (item as { data?: any })?.data || {}; return;
const jobOptions = this.getJobOptions(item) || undefined; }
await this.getQueue(JOBS_TO_QUEUE[jobName]).add(jobName, jobData, jobOptions); const itemsByQueue = items.reduce<Record<string, JobItem[]>>((acc, item) => {
const queueName = JOBS_TO_QUEUE[item.name];
acc[queueName] = acc[queueName] || [];
acc[queueName].push(item);
return acc;
}, {});
for (const [queueName, items] of Object.entries(itemsByQueue)) {
const queue = this.getQueue(queueName as QueueName);
const jobs = items.map((item) => ({
name: item.name,
data: (item as { data?: any })?.data || {},
options: this.getJobOptions(item) || undefined,
}));
await queue.addBulk(jobs);
}
}
async queue(item: JobItem): Promise<void> {
await this.queueAll([item]);
} }
private getJobOptions(item: JobItem): JobsOptions | null { private getJobOptions(item: JobItem): JobsOptions | null {
server/src/infra/repositories/smart-info.repository.ts
+8 -8
View File
@@ -41,9 +41,9 @@ export class SmartInfoRepository implements ISmartInfoRepository {
} }
@GenerateSql({ @GenerateSql({
params: [{ ownerId: DummyValue.UUID, embedding: Array.from({ length: 512 }, Math.random), numResults: 100 }], params: [{ userIds: [DummyValue.UUID], embedding: Array.from({ length: 512 }, Math.random), numResults: 100 }],
}) })
async searchCLIP({ ownerId, embedding, numResults }: EmbeddingSearch): Promise<AssetEntity[]> { async searchCLIP({ userIds, embedding, numResults }: EmbeddingSearch): Promise<AssetEntity[]> {
if (!isValidInteger(numResults, { min: 1 })) { if (!isValidInteger(numResults, { min: 1 })) {
throw new Error(`Invalid value for 'numResults': ${numResults}`); throw new Error(`Invalid value for 'numResults': ${numResults}`);
} }
@@ -55,13 +55,13 @@ export class SmartInfoRepository implements ISmartInfoRepository {
results = await manager results = await manager
.createQueryBuilder(AssetEntity, 'a') .createQueryBuilder(AssetEntity, 'a')
.innerJoin('a.smartSearch', 's') .innerJoin('a.smartSearch', 's')
.where('a.ownerId = :ownerId') .where('a.ownerId IN (:...userIds )')
.andWhere('a.isVisible = true') .andWhere('a.isVisible = true')
.andWhere('a.isArchived = false') .andWhere('a.isArchived = false')
.andWhere('a.fileCreatedAt < NOW()') .andWhere('a.fileCreatedAt < NOW()')
.leftJoinAndSelect('a.exifInfo', 'e') .leftJoinAndSelect('a.exifInfo', 'e')
.orderBy('s.embedding <=> :embedding') .orderBy('s.embedding <=> :embedding')
.setParameters({ ownerId, embedding: asVector(embedding) }) .setParameters({ userIds, embedding: asVector(embedding) })
.limit(numResults) .limit(numResults)
.getMany(); .getMany();
}); });
@@ -72,14 +72,14 @@ export class SmartInfoRepository implements ISmartInfoRepository {
@GenerateSql({ @GenerateSql({
params: [ params: [
{ {
ownerId: DummyValue.UUID, userIds: [DummyValue.UUID],
embedding: Array.from({ length: 512 }, Math.random), embedding: Array.from({ length: 512 }, Math.random),
numResults: 100, numResults: 100,
maxDistance: 0.6, maxDistance: 0.6,
}, },
], ],
}) })
async searchFaces({ ownerId, embedding, numResults, maxDistance }: EmbeddingSearch): Promise<AssetFaceEntity[]> { async searchFaces({ userIds, embedding, numResults, maxDistance }: EmbeddingSearch): Promise<AssetFaceEntity[]> {
if (!isValidInteger(numResults, { min: 1 })) { if (!isValidInteger(numResults, { min: 1 })) {
throw new Error(`Invalid value for 'numResults': ${numResults}`); throw new Error(`Invalid value for 'numResults': ${numResults}`);
} }
@@ -91,9 +91,9 @@ export class SmartInfoRepository implements ISmartInfoRepository {
.createQueryBuilder(AssetFaceEntity, 'faces') .createQueryBuilder(AssetFaceEntity, 'faces')
.select('1 + (faces.embedding <=> :embedding)', 'distance') .select('1 + (faces.embedding <=> :embedding)', 'distance')
.innerJoin('faces.asset', 'asset') .innerJoin('faces.asset', 'asset')
.where('asset.ownerId = :ownerId') .where('asset.ownerId IN (:...userIds )')
.orderBy('1 + (faces.embedding <=> :embedding)') .orderBy('1 + (faces.embedding <=> :embedding)')
.setParameters({ ownerId, embedding: asVector(embedding) }) .setParameters({ userIds, embedding: asVector(embedding) })
.limit(numResults); .limit(numResults);
this.faceColumns.forEach((col) => cte.addSelect(`faces.${col}`, col)); this.faceColumns.forEach((col) => cte.addSelect(`faces.${col}`, col));
server/src/infra/sql/smart.info.repository.sql
+2 -2
View File
@@ -69,7 +69,7 @@ FROM
LEFT JOIN "exif" "e" ON "e"."assetId" = "a"."id" LEFT JOIN "exif" "e" ON "e"."assetId" = "a"."id"
WHERE WHERE
( (
"a"."ownerId" = $1 "a"."ownerId" IN ($1)
AND "a"."isVisible" = true AND "a"."isVisible" = true
AND "a"."isArchived" = false AND "a"."isArchived" = false
AND "a"."fileCreatedAt" < NOW() AND "a"."fileCreatedAt" < NOW()
@@ -103,7 +103,7 @@ WITH
INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId" INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId"
AND ("asset"."deletedAt" IS NULL) AND ("asset"."deletedAt" IS NULL)
WHERE WHERE
"asset"."ownerId" = $2 "asset"."ownerId" IN ($2)
ORDER BY ORDER BY
1 + ("faces"."embedding" <= > $3) ASC 1 + ("faces"."embedding" <= > $3) ASC
LIMIT LIMIT
server/test/repositories/job.repository.mock.ts
+1
View File
@@ -11,6 +11,7 @@ export const newJobRepositoryMock = (): jest.Mocked<IJobRepository> => {
pause: jest.fn(), pause: jest.fn(),
resume: jest.fn(), resume: jest.fn(),
queue: jest.fn().mockImplementation(() => Promise.resolve()), queue: jest.fn().mockImplementation(() => Promise.resolve()),
queueAll: jest.fn().mockImplementation(() => Promise.resolve()),
getQueueStatus: jest.fn(), getQueueStatus: jest.fn(),
getJobCounts: jest.fn(), getJobCounts: jest.fn(),
clear: jest.fn(), clear: jest.fn(),
server/test/test-utils.ts
+1
View File
@@ -77,6 +77,7 @@ export const testApp = {
deleteCronJob: jest.fn(), deleteCronJob: jest.fn(),
validateCronExpression: jest.fn(), validateCronExpression: jest.fn(),
queue: (item: JobItem) => jobs && _handler(item), queue: (item: JobItem) => jobs && _handler(item),
queueAll: (items: JobItem[]) => jobs && Promise.all(items.map(_handler)).then(() => Promise.resolve()),
resume: jest.fn(), resume: jest.fn(),
empty: jest.fn(), empty: jest.fn(),
setConcurrency: jest.fn(), setConcurrency: jest.fn(),
web/package-lock.json
Generated
-6
View File
@@ -21,7 +21,6 @@
"luxon": "^3.2.1", "luxon": "^3.2.1",
"maplibre-gl": "^3.6.0", "maplibre-gl": "^3.6.0",
"socket.io-client": "^4.6.1", "socket.io-client": "^4.6.1",
"svelte-loading-spinners": "^0.3.4",
"svelte-local-storage-store": "^0.6.0", "svelte-local-storage-store": "^0.6.0",
"svelte-maplibre": "^0.7.0", "svelte-maplibre": "^0.7.0",
"thumbhash": "^0.1.1" "thumbhash": "^0.1.1"
@@ -6862,11 +6861,6 @@
"svelte": "^3.19.0 || ^4.0.0" "svelte": "^3.19.0 || ^4.0.0"
} }
}, },
"node_modules/svelte-loading-spinners": {
"version": "0.3.4",
"resolved": "https://registry.npmjs.org/svelte-loading-spinners/-/svelte-loading-spinners-0.3.4.tgz",
"integrity": "sha512-vKaW71QMCBcTNijAGc0mUl8k3DQ66iYmp6MB8BMGCXyWk82bTrcLy8FOnSm9fE+8q6TwzD6PLUoYFHt0II93Xw=="
},
"node_modules/svelte-local-storage-store": { "node_modules/svelte-local-storage-store": {
"version": "0.6.4", "version": "0.6.4",
"resolved": "https://registry.npmjs.org/svelte-local-storage-store/-/svelte-local-storage-store-0.6.4.tgz", "resolved": "https://registry.npmjs.org/svelte-local-storage-store/-/svelte-local-storage-store-0.6.4.tgz",
web/package.json
-1
View File
@@ -67,7 +67,6 @@
"luxon": "^3.2.1", "luxon": "^3.2.1",
"maplibre-gl": "^3.6.0", "maplibre-gl": "^3.6.0",
"socket.io-client": "^4.6.1", "socket.io-client": "^4.6.1",
"svelte-loading-spinners": "^0.3.4",
"svelte-local-storage-store": "^0.6.0", "svelte-local-storage-store": "^0.6.0",
"svelte-maplibre": "^0.7.0", "svelte-maplibre": "^0.7.0",
"thumbhash": "^0.1.1" "thumbhash": "^0.1.1"
web/src/lib/components/shared-components/loading-spinner.svelte
+7 -1
View File
@@ -1,7 +1,13 @@
<script lang="ts">
export let size: string = '24';
</script>
<div> <div>
<svg <svg
role="status" role="status"
class={`h-[24px] w-[24px] animate-spin fill-immich-primary text-gray-400 dark:text-gray-600`} style:height="{size}px"
style:width="{size}px"
class="animate-spin fill-immich-primary text-gray-400 dark:text-gray-600"
viewBox="0 0 100 101" viewBox="0 0 100 101"
fill="none" fill="none"
xmlns="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg"
web/src/lib/components/user-settings-page/library-list.svelte
+2 -2
View File
@@ -7,7 +7,6 @@
import { handleError } from '$lib/utils/handle-error'; import { handleError } from '$lib/utils/handle-error';
import { fade } from 'svelte/transition'; import { fade } from 'svelte/transition';
import Icon from '$lib/components/elements/icon.svelte'; import Icon from '$lib/components/elements/icon.svelte';
import Pulse from 'svelte-loading-spinners/Pulse.svelte';
import { slide } from 'svelte/transition'; import { slide } from 'svelte/transition';
import LibraryImportPathsForm from '../forms/library-import-paths-form.svelte'; import LibraryImportPathsForm from '../forms/library-import-paths-form.svelte';
import LibraryScanSettingsForm from '../forms/library-scan-settings-form.svelte'; import LibraryScanSettingsForm from '../forms/library-scan-settings-form.svelte';
@@ -18,6 +17,7 @@
import MenuOption from '../shared-components/context-menu/menu-option.svelte'; import MenuOption from '../shared-components/context-menu/menu-option.svelte';
import { getContextMenuPosition } from '$lib/utils/context-menu'; import { getContextMenuPosition } from '$lib/utils/context-menu';
import { mdiDatabase, mdiDotsVertical, mdiUpload } from '@mdi/js'; import { mdiDatabase, mdiDotsVertical, mdiUpload } from '@mdi/js';
import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte';
let libraries: LibraryResponseDto[] = []; let libraries: LibraryResponseDto[] = [];
@@ -323,7 +323,7 @@
<td class="w-1/3 text-ellipsis px-4 text-sm">{library.name}</td> <td class="w-1/3 text-ellipsis px-4 text-sm">{library.name}</td>
{#if totalCount[index] == undefined} {#if totalCount[index] == undefined}
<td colspan="2" class="flex w-1/3 items-center justify-center text-ellipsis px-4 text-sm"> <td colspan="2" class="flex w-1/3 items-center justify-center text-ellipsis px-4 text-sm">
<Pulse color="gray" size="40" unit="px" /> <LoadingSpinner size="40" />
</td> </td>
{:else} {:else}
<td class="w-1/6 text-ellipsis px-4 text-sm"> <td class="w-1/6 text-ellipsis px-4 text-sm">
web/src/routes/(user)/share/[key]/+page.svelte
+4 -2
View File
@@ -58,8 +58,10 @@
Please enter the password to view this page. Please enter the password to view this page.
</div> </div>
<div class="mt-4"> <div class="mt-4">
<input type="password" class="immich-form-input mr-2" placeholder="Password" bind:value={password} /> <form novalidate autocomplete="off" on:submit|preventDefault={handlePasswordSubmit}>
<Button on:click={handlePasswordSubmit}>Submit</Button> <input type="password" class="immich-form-input mr-2" placeholder="Password" bind:value={password} />
<Button type="submit">Submit</Button>
</form>
</div> </div>
</div> </div>
</main> </main>