feat(server, web): smart search filtering and pagination (#6525)

* initial pagination impl

* use limit + offset instead of take + skip

* wip web pagination

* working infinite scroll

* update api

* formatting

* fix rebase

* search refactor

* re-add runtime config for vector search

* fix rebase

* fixes

* useless omitBy

* unnecessary handling

* add sql decorator for `searchAssets`

* fixed search builder

* fixed sql

* remove mock method

* linting

* fixed pagination

* fixed unit tests

* formatting

* fix e2e tests

* re-flatten search builder

* refactor endpoints

* clean up dto

* refinements

* don't break everything just yet

* update openapi spec & sql

* update api

* linting

* update sql

* fixes

* optimize web code

* fix typing

* add page limit

* make limit based on asset count

* increase limit

* simpler import
This commit is contained in:
Mert
2024-02-12 20:50:47 -05:00
committed by GitHub
parent f1e4fdf175
commit e334443919
54 changed files with 3993 additions and 790 deletions
@@ -5,7 +5,7 @@ import {
newDatabaseRepositoryMock,
newJobRepositoryMock,
newMachineLearningRepositoryMock,
newSmartInfoRepositoryMock,
newSearchRepositoryMock,
newSystemConfigRepositoryMock,
} from '@test';
import { JobName } from '../job';
@@ -14,7 +14,7 @@ import {
IDatabaseRepository,
IJobRepository,
IMachineLearningRepository,
ISmartInfoRepository,
ISearchRepository,
ISystemConfigRepository,
WithoutProperty,
} from '../repositories';
@@ -31,18 +31,18 @@ describe(SmartInfoService.name, () => {
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let smartMock: jest.Mocked<ISmartInfoRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
let machineMock: jest.Mocked<IMachineLearningRepository>;
let databaseMock: jest.Mocked<IDatabaseRepository>;
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
smartMock = newSmartInfoRepositoryMock();
searchMock = newSearchRepositoryMock();
jobMock = newJobRepositoryMock();
machineMock = newMachineLearningRepositoryMock();
databaseMock = newDatabaseRepositoryMock();
sut = new SmartInfoService(assetMock, databaseMock, jobMock, machineMock, smartMock, configMock);
sut = new SmartInfoService(assetMock, databaseMock, jobMock, machineMock, searchMock, configMock);
assetMock.getByIds.mockResolvedValue([asset]);
});
@@ -102,12 +102,12 @@ describe(SmartInfoService.name, () => {
await sut.handleEncodeClip({ id: asset.id });
expect(smartMock.upsert).not.toHaveBeenCalled();
expect(searchMock.upsert).not.toHaveBeenCalled();
expect(machineMock.encodeImage).not.toHaveBeenCalled();
});
it('should save the returned objects', async () => {
smartMock.upsert.mockResolvedValue();
searchMock.upsert.mockResolvedValue();
machineMock.encodeImage.mockResolvedValue([0.01, 0.02, 0.03]);
await sut.handleEncodeClip({ id: asset.id });
@@ -117,7 +117,7 @@ describe(SmartInfoService.name, () => {
{ imagePath: 'path/to/resize.ext' },
{ enabled: true, modelName: 'ViT-B-32__openai' },
);
expect(smartMock.upsert).toHaveBeenCalledWith(
expect(searchMock.upsert).toHaveBeenCalledWith(
{
assetId: 'asset-1',
},