cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server,ml): remove image tagging (#5903)

Browse Source 
* remove image tagging

* updated lock

* fixed tests, improved logging

* be nice

* fixed tests
This commit is contained in:
Mert
2023-12-20 20:47:56 -05:00
committed by GitHub
parent 154292242f
commit 092a23fd7f
65 changed files with 988 additions and 2930 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
server/src/domain/smart-info/dto/model-config.dto.ts
View File

@@ -18,15 +18,6 @@ export class ModelConfig {
modelType?: ModelType;
}
export class ClassificationConfig extends ModelConfig {
@IsNumber()
@Min(0)
@Max(1)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
minScore!: number;
}
export class CLIPConfig extends ModelConfig {
@IsEnum(CLIPMode)
@Optional()

101
server/src/domain/smart-info/smart-info.service.spec.ts
View File

@@ -47,107 +47,6 @@ describe(SmartInfoService.name, () => {
expect(sut).toBeDefined();
});
describe('handleQueueObjectTagging', () => {
beforeEach(async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.MACHINE_LEARNING_CLASSIFICATION_ENABLED, value: true },
]);
});
it('should do nothing if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
await sut.handleQueueObjectTagging({});
expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).not.toHaveBeenCalled();
});
it('should queue the assets without tags', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.MACHINE_LEARNING_CLASSIFICATION_ENABLED, value: true },
]);
assetMock.getWithout.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
await sut.handleQueueObjectTagging({ force: false });
expect(jobMock.queue.mock.calls).toEqual([[{ name: JobName.CLASSIFY_IMAGE, data: { id: assetStub.image.id } }]]);
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.OBJECT_TAGS);
});
it('should queue all the assets', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.MACHINE_LEARNING_CLASSIFICATION_ENABLED, value: true },
]);
assetMock.getAll.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
await sut.handleQueueObjectTagging({ force: true });
expect(jobMock.queue.mock.calls).toEqual([[{ name: JobName.CLASSIFY_IMAGE, data: { id: assetStub.image.id } }]]);
expect(assetMock.getAll).toHaveBeenCalled();
});
});
describe('handleClassifyImage', () => {
it('should do nothing if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
await sut.handleClassifyImage({ id: '123' });
expect(machineMock.classifyImage).not.toHaveBeenCalled();
expect(assetMock.getByIds).not.toHaveBeenCalled();
});
it('should skip assets without a resize path', async () => {
const asset = { resizePath: '' } as AssetEntity;
assetMock.getByIds.mockResolvedValue([asset]);
await sut.handleClassifyImage({ id: asset.id });
expect(smartMock.upsert).not.toHaveBeenCalled();
expect(machineMock.classifyImage).not.toHaveBeenCalled();
});
it('should save the returned tags', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.MACHINE_LEARNING_CLASSIFICATION_ENABLED, value: true },
]);
machineMock.classifyImage.mockResolvedValue(['tag1', 'tag2', 'tag3']);
await sut.handleClassifyImage({ id: asset.id });
expect(machineMock.classifyImage).toHaveBeenCalledWith(
'http://immich-machine-learning:3003',
{
imagePath: 'path/to/resize.ext',
},
{ enabled: true, minScore: 0.9, modelName: 'microsoft/resnet-50' },
);
expect(smartMock.upsert).toHaveBeenCalledWith({
assetId: 'asset-1',
tags: ['tag1', 'tag2', 'tag3'],
});
});
it('should always overwrite old tags', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.MACHINE_LEARNING_CLASSIFICATION_ENABLED, value: true },
]);
machineMock.classifyImage.mockResolvedValue([]);
await sut.handleClassifyImage({ id: asset.id });
expect(machineMock.classifyImage).toHaveBeenCalled();
expect(smartMock.upsert).toHaveBeenCalled();
});
});
describe('handleQueueEncodeClip', () => {
it('should do nothing if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);

42
server/src/domain/smart-info/smart-info.service.ts
View File

@@ -46,48 +46,6 @@ export class SmartInfoService {
await this.jobRepository.resume(QueueName.SMART_SEARCH);
}
async handleQueueObjectTagging({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.classification.enabled) {
return true;
}
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
: this.assetRepository.getWithout(pagination, WithoutProperty.OBJECT_TAGS);
});
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: { id: asset.id } });
}
}
return true;
}
async handleClassifyImage({ id }: IEntityJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.classification.enabled) {
return true;
}
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset.resizePath) {
return false;
}
const tags = await this.machineLearning.classifyImage(
machineLearning.url,
{ imagePath: asset.resizePath },
machineLearning.classification,
);
await this.repository.upsert({ assetId: asset.id, tags });
return true;
}
async handleQueueEncodeClip({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.clip.enabled) {