feat(server, web): smart search filtering and pagination (#6525)

* initial pagination impl

* use limit + offset instead of take + skip

* wip web pagination

* working infinite scroll

* update api

* formatting

* fix rebase

* search refactor

* re-add runtime config for vector search

* fix rebase

* fixes

* useless omitBy

* unnecessary handling

* add sql decorator for `searchAssets`

* fixed search builder

* fixed sql

* remove mock method

* linting

* fixed pagination

* fixed unit tests

* formatting

* fix e2e tests

* re-flatten search builder

* refactor endpoints

* clean up dto

* refinements

* don't break everything just yet

* update openapi spec & sql

* update api

* linting

* update sql

* fixes

* optimize web code

* fix typing

* add page limit

* make limit based on asset count

* increase limit

* simpler import
This commit is contained in:
Mert
2024-02-12 20:50:47 -05:00
committed by GitHub
parent f1e4fdf175
commit e334443919
54 changed files with 3993 additions and 790 deletions
+692
View File
@@ -2130,6 +2130,7 @@
},
"/assets": {
"get": {
"deprecated": true,
"operationId": "searchAssets",
"parameters": [
{
@@ -2430,6 +2431,14 @@
"type": "string"
}
},
{
"name": "withArchived",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withDeleted",
"required": false,
@@ -4354,6 +4363,7 @@
},
"/search": {
"get": {
"deprecated": true,
"operationId": "search",
"parameters": [
{
@@ -4374,6 +4384,14 @@
"type": "boolean"
}
},
{
"name": "page",
"required": false,
"in": "query",
"schema": {
"type": "number"
}
},
{
"name": "q",
"required": false,
@@ -4398,6 +4416,14 @@
"type": "boolean"
}
},
{
"name": "size",
"required": false,
"in": "query",
"schema": {
"type": "number"
}
},
{
"name": "smart",
"required": false,
@@ -4492,6 +4518,377 @@
]
}
},
"/search/metadata": {
"get": {
"operationId": "searchMetadata",
"parameters": [
{
"name": "checksum",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "city",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "country",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "createdAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "createdBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "deviceAssetId",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "deviceId",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "encodedVideoPath",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "id",
"required": false,
"in": "query",
"schema": {
"format": "uuid",
"type": "string"
}
},
{
"name": "isArchived",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isEncoded",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isExternal",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isFavorite",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isMotion",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isOffline",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isReadOnly",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isVisible",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "lensModel",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "libraryId",
"required": false,
"in": "query",
"schema": {
"format": "uuid",
"type": "string"
}
},
{
"name": "make",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "model",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "order",
"required": false,
"in": "query",
"schema": {
"$ref": "#/components/schemas/AssetOrder"
}
},
{
"name": "originalFileName",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "originalPath",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "page",
"required": false,
"in": "query",
"schema": {
"type": "number"
}
},
{
"name": "resizePath",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "size",
"required": false,
"in": "query",
"schema": {
"type": "number"
}
},
{
"name": "state",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "takenAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "takenBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "trashedAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "trashedBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "type",
"required": false,
"in": "query",
"schema": {
"$ref": "#/components/schemas/AssetTypeEnum"
}
},
{
"name": "updatedAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "updatedBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "webpPath",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "withArchived",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withDeleted",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withExif",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withPeople",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withStacked",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SearchResponseDto"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Search"
]
}
},
"/search/person": {
"get": {
"operationId": "searchPerson",
@@ -4544,6 +4941,296 @@
]
}
},
"/search/smart": {
"get": {
"operationId": "searchSmart",
"parameters": [
{
"name": "city",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "country",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "createdAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "createdBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "deviceId",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "isArchived",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isEncoded",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isExternal",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isFavorite",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isMotion",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isOffline",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isReadOnly",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isVisible",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "lensModel",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "libraryId",
"required": false,
"in": "query",
"schema": {
"format": "uuid",
"type": "string"
}
},
{
"name": "make",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "model",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "page",
"required": false,
"in": "query",
"schema": {
"type": "number"
}
},
{
"name": "query",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "size",
"required": false,
"in": "query",
"schema": {
"type": "number"
}
},
{
"name": "state",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "takenAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "takenBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "trashedAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "trashedBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "type",
"required": false,
"in": "query",
"schema": {
"$ref": "#/components/schemas/AssetTypeEnum"
}
},
{
"name": "updatedAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "updatedBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "withArchived",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withDeleted",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withExif",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SearchResponseDto"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Search"
]
}
},
"/server-info": {
"get": {
"operationId": "getServerInfo",
@@ -8458,6 +9145,10 @@
},
"type": "array"
},
"nextPage": {
"nullable": true,
"type": "string"
},
"total": {
"type": "integer"
}
@@ -8466,6 +9157,7 @@
"count",
"facets",
"items",
"nextPage",
"total"
],
"type": "object"
File diff suppressed because it is too large Load Diff
+173 -2
View File
@@ -588,6 +588,7 @@ export type SearchAssetResponseDto = {
count: number;
facets: SearchFacetResponseDto[];
items: AssetResponseDto[];
nextPage: string | null;
total: number;
};
export type SearchResponseDto = {
@@ -1461,7 +1462,7 @@ export function updateAsset({ id, updateAssetDto }: {
body: updateAssetDto
})));
}
export function searchAssets({ checksum, city, country, createdAfter, createdBefore, deviceAssetId, deviceId, encodedVideoPath, id, isArchived, isEncoded, isExternal, isFavorite, isMotion, isOffline, isReadOnly, isVisible, lensModel, libraryId, make, model, order, originalFileName, originalPath, page, resizePath, size, state, takenAfter, takenBefore, trashedAfter, trashedBefore, $type, updatedAfter, updatedBefore, webpPath, withDeleted, withExif, withPeople, withStacked }: {
export function searchAssets({ checksum, city, country, createdAfter, createdBefore, deviceAssetId, deviceId, encodedVideoPath, id, isArchived, isEncoded, isExternal, isFavorite, isMotion, isOffline, isReadOnly, isVisible, lensModel, libraryId, make, model, order, originalFileName, originalPath, page, resizePath, size, state, takenAfter, takenBefore, trashedAfter, trashedBefore, $type, updatedAfter, updatedBefore, webpPath, withArchived, withDeleted, withExif, withPeople, withStacked }: {
checksum?: string;
city?: string;
country?: string;
@@ -1498,6 +1499,7 @@ export function searchAssets({ checksum, city, country, createdAfter, createdBef
updatedAfter?: string;
updatedBefore?: string;
webpPath?: string;
withArchived?: boolean;
withDeleted?: boolean;
withExif?: boolean;
withPeople?: boolean;
@@ -1543,6 +1545,7 @@ export function searchAssets({ checksum, city, country, createdAfter, createdBef
updatedAfter,
updatedBefore,
webpPath,
withArchived,
withDeleted,
withExif,
withPeople,
@@ -2047,12 +2050,14 @@ export function getPersonThumbnail({ id }: {
...opts
}));
}
export function search({ clip, motion, q, query, recent, smart, $type, withArchived }: {
export function search({ clip, motion, page, q, query, recent, size, smart, $type, withArchived }: {
clip?: boolean;
motion?: boolean;
page?: number;
q?: string;
query?: string;
recent?: boolean;
size?: number;
smart?: boolean;
$type?: "IMAGE" | "VIDEO" | "AUDIO" | "OTHER";
withArchived?: boolean;
@@ -2063,9 +2068,11 @@ export function search({ clip, motion, q, query, recent, smart, $type, withArchi
}>(`/search${QS.query(QS.explode({
clip,
motion,
page,
q,
query,
recent,
size,
smart,
"type": $type,
withArchived
@@ -2081,6 +2088,98 @@ export function getExploreData(opts?: Oazapfts.RequestOpts) {
...opts
}));
}
export function searchMetadata({ checksum, city, country, createdAfter, createdBefore, deviceAssetId, deviceId, encodedVideoPath, id, isArchived, isEncoded, isExternal, isFavorite, isMotion, isOffline, isReadOnly, isVisible, lensModel, libraryId, make, model, order, originalFileName, originalPath, page, resizePath, size, state, takenAfter, takenBefore, trashedAfter, trashedBefore, $type, updatedAfter, updatedBefore, webpPath, withArchived, withDeleted, withExif, withPeople, withStacked }: {
checksum?: string;
city?: string;
country?: string;
createdAfter?: string;
createdBefore?: string;
deviceAssetId?: string;
deviceId?: string;
encodedVideoPath?: string;
id?: string;
isArchived?: boolean;
isEncoded?: boolean;
isExternal?: boolean;
isFavorite?: boolean;
isMotion?: boolean;
isOffline?: boolean;
isReadOnly?: boolean;
isVisible?: boolean;
lensModel?: string;
libraryId?: string;
make?: string;
model?: string;
order?: AssetOrder;
originalFileName?: string;
originalPath?: string;
page?: number;
resizePath?: string;
size?: number;
state?: string;
takenAfter?: string;
takenBefore?: string;
trashedAfter?: string;
trashedBefore?: string;
$type?: AssetTypeEnum;
updatedAfter?: string;
updatedBefore?: string;
webpPath?: string;
withArchived?: boolean;
withDeleted?: boolean;
withExif?: boolean;
withPeople?: boolean;
withStacked?: boolean;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: SearchResponseDto;
}>(`/search/metadata${QS.query(QS.explode({
checksum,
city,
country,
createdAfter,
createdBefore,
deviceAssetId,
deviceId,
encodedVideoPath,
id,
isArchived,
isEncoded,
isExternal,
isFavorite,
isMotion,
isOffline,
isReadOnly,
isVisible,
lensModel,
libraryId,
make,
model,
order,
originalFileName,
originalPath,
page,
resizePath,
size,
state,
takenAfter,
takenBefore,
trashedAfter,
trashedBefore,
"type": $type,
updatedAfter,
updatedBefore,
webpPath,
withArchived,
withDeleted,
withExif,
withPeople,
withStacked
}))}`, {
...opts
}));
}
export function searchPerson({ name, withHidden }: {
name: string;
withHidden?: boolean;
@@ -2095,6 +2194,78 @@ export function searchPerson({ name, withHidden }: {
...opts
}));
}
export function searchSmart({ city, country, createdAfter, createdBefore, deviceId, isArchived, isEncoded, isExternal, isFavorite, isMotion, isOffline, isReadOnly, isVisible, lensModel, libraryId, make, model, page, query, size, state, takenAfter, takenBefore, trashedAfter, trashedBefore, $type, updatedAfter, updatedBefore, withArchived, withDeleted, withExif }: {
city?: string;
country?: string;
createdAfter?: string;
createdBefore?: string;
deviceId?: string;
isArchived?: boolean;
isEncoded?: boolean;
isExternal?: boolean;
isFavorite?: boolean;
isMotion?: boolean;
isOffline?: boolean;
isReadOnly?: boolean;
isVisible?: boolean;
lensModel?: string;
libraryId?: string;
make?: string;
model?: string;
page?: number;
query: string;
size?: number;
state?: string;
takenAfter?: string;
takenBefore?: string;
trashedAfter?: string;
trashedBefore?: string;
$type?: AssetTypeEnum;
updatedAfter?: string;
updatedBefore?: string;
withArchived?: boolean;
withDeleted?: boolean;
withExif?: boolean;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: SearchResponseDto;
}>(`/search/smart${QS.query(QS.explode({
city,
country,
createdAfter,
createdBefore,
deviceId,
isArchived,
isEncoded,
isExternal,
isFavorite,
isMotion,
isOffline,
isReadOnly,
isVisible,
lensModel,
libraryId,
make,
model,
page,
query,
size,
state,
takenAfter,
takenBefore,
trashedAfter,
trashedBefore,
"type": $type,
updatedAfter,
updatedBefore,
withArchived,
withDeleted,
withExif
}))}`, {
...opts
}));
}
export function getServerInfo(opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;