cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): pagination for asset queries in jobs (#2516)

Browse Source 
* feat(server): pagination for asset queries in jobs

* default mock value for getAll

* remove live photo name correction

* order paginated results by createdAt

* change log level

* move usePagination to domain
This commit is contained in:
Michel Heusschen
2023-05-22 20:05:06 +02:00
committed by GitHub
parent feadc45e75
commit f1384fea58
18 changed files with 253 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/libs/domain/src/smart-info/smart-info.service.spec.ts
+18 -6
View File
@@ -38,7 +38,10 @@ describe(SmartInfoService.name, () => {
describe('handleQueueObjectTagging', () => {
it('should queue the assets without tags', async () => {
assetMock.getWithout.mockResolvedValue([assetEntityStub.image]);
assetMock.getWithout.mockResolvedValue({
items: [assetEntityStub.image],
hasNextPage: false,
});
await sut.handleQueueObjectTagging({ force: false });
@@ -46,11 +49,14 @@ describe(SmartInfoService.name, () => {
[{ name: JobName.CLASSIFY_IMAGE, data: { asset: assetEntityStub.image } }],
[{ name: JobName.DETECT_OBJECTS, data: { asset: assetEntityStub.image } }],
]);
expect(assetMock.getWithout).toHaveBeenCalledWith(WithoutProperty.OBJECT_TAGS);
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.OBJECT_TAGS);
});
it('should queue all the assets', async () => {
assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
assetMock.getAll.mockResolvedValue({
items: [assetEntityStub.image],
hasNextPage: false,
});
await sut.handleQueueObjectTagging({ force: true });
@@ -140,16 +146,22 @@ describe(SmartInfoService.name, () => {
describe('handleQueueEncodeClip', () => {
it('should queue the assets without clip embeddings', async () => {
assetMock.getWithout.mockResolvedValue([assetEntityStub.image]);
assetMock.getWithout.mockResolvedValue({
items: [assetEntityStub.image],
hasNextPage: false,
});
await sut.handleQueueEncodeClip({ force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.ENCODE_CLIP, data: { asset: assetEntityStub.image } });
expect(assetMock.getWithout).toHaveBeenCalledWith(WithoutProperty.CLIP_ENCODING);
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.CLIP_ENCODING);
});
it('should queue all the assets', async () => {
assetMock.getAll.mockResolvedValue([assetEntityStub.image]);
assetMock.getAll.mockResolvedValue({
items: [assetEntityStub.image],
hasNextPage: false,
});
await sut.handleQueueEncodeClip({ force: true });
server/libs/domain/src/smart-info/smart-info.service.ts
+21 -12
View File
@@ -1,7 +1,8 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { IAssetRepository, WithoutProperty } from '../asset';
import { MACHINE_LEARNING_ENABLED } from '../domain.constant';
import { IAssetJob, IBaseJob, IJobRepository, JobName } from '../job';
import { usePagination } from '../domain.util';
import { IAssetJob, IBaseJob, IJobRepository, JobName, JOBS_ASSET_PAGINATION_SIZE } from '../job';
import { IMachineLearningRepository } from './machine-learning.interface';
import { ISmartInfoRepository } from './smart-info.repository';
@@ -18,13 +19,17 @@ export class SmartInfoService {
async handleQueueObjectTagging({ force }: IBaseJob) {
try {
const assets = force
? await this.assetRepository.getAll()
: await this.assetRepository.getWithout(WithoutProperty.OBJECT_TAGS);
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
: this.assetRepository.getWithout(pagination, WithoutProperty.OBJECT_TAGS);
});
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: { asset } });
await this.jobRepository.queue({ name: JobName.DETECT_OBJECTS, data: { asset } });
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: { asset } });
await this.jobRepository.queue({ name: JobName.DETECT_OBJECTS, data: { asset } });
}
}
} catch (error: any) {
this.logger.error(`Unable to queue object tagging`, error?.stack);
@@ -69,12 +74,16 @@ export class SmartInfoService {
async handleQueueEncodeClip({ force }: IBaseJob) {
try {
const assets = force
? await this.assetRepository.getAll()
: await this.assetRepository.getWithout(WithoutProperty.CLIP_ENCODING);
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
: this.assetRepository.getWithout(pagination, WithoutProperty.CLIP_ENCODING);
});
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: { asset } });
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: { asset } });
}
}
} catch (error: any) {
this.logger.error(`Unable to queue clip encoding`, error?.stack);