cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): lighter buckets

Browse Source
This commit is contained in:
Min Idzelis
2025-04-24 00:37:20 +00:00
parent 37f5e6e2cb
commit bfefa36f04
30 changed files with 1735 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
server/src/services/sync.service.ts
View File

@@ -4,7 +4,7 @@ import { DateTime } from 'luxon';
import { Writable } from 'node:stream';
import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
import { SessionSyncCheckpoints } from 'src/db';
import { AssetResponseDto, hexOrBufferToBase64, mapAsset } from 'src/dtos/asset-response.dto';
import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import {
AssetDeltaSyncDto,
@@ -18,6 +18,7 @@ import { DatabaseAction, EntityType, Permission, SyncEntityType, SyncRequestType
import { BaseService } from 'src/services/base.service';
import { SyncAck } from 'src/types';
import { getMyPartnerIds } from 'src/utils/asset.util';
import { hexOrBufferToBase64 } from 'src/utils/bytes';
import { setIsEqual } from 'src/utils/set';
import { fromAck, serialize } from 'src/utils/sync';

139
server/src/services/timeline.service.spec.ts
View File

@@ -1,9 +1,7 @@
import { BadRequestException } from '@nestjs/common';
import { TimeBucketSize } from 'src/repositories/asset.repository';
import { TimelineService } from 'src/services/timeline.service';
import { assetStub } from 'test/fixtures/asset.stub';
import { authStub } from 'test/fixtures/auth.stub';
import { factory } from 'test/small.factory';
import { newTestService, ServiceMocks } from 'test/utils';
describe(TimelineService.name, () => {
@@ -18,13 +16,10 @@ describe(TimelineService.name, () => {
it("should return buckets if userId and albumId aren't set", async () => {
mocks.asset.getTimeBuckets.mockResolvedValue([{ timeBucket: 'bucket', count: 1 }]);
await expect(
sut.getTimeBuckets(authStub.admin, {
size: TimeBucketSize.DAY,
}),
).resolves.toEqual(expect.arrayContaining([{ timeBucket: 'bucket', count: 1 }]));
await expect(sut.getTimeBuckets(authStub.admin, {})).resolves.toEqual(
expect.arrayContaining([{ timeBucket: 'bucket', count: 1 }]),
);
expect(mocks.asset.getTimeBuckets).toHaveBeenCalledWith({
size: TimeBucketSize.DAY,
userIds: [authStub.admin.user.id],
});
});
@@ -35,16 +30,24 @@ describe(TimelineService.name, () => {
mocks.access.album.checkOwnerAccess.mockResolvedValue(new Set(['album-id']));
mocks.asset.getTimeBucket.mockResolvedValue([assetStub.image]);
await expect(
sut.getTimeBucket(authStub.admin, { size: TimeBucketSize.DAY, timeBucket: 'bucket', albumId: 'album-id' }),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })]));
await expect(sut.getTimeBucket(authStub.admin, { timeBucket: 'bucket', albumId: 'album-id' })).resolves.toEqual(
expect.objectContaining({
bucketAssets: expect.objectContaining({ id: expect.arrayContaining(['asset-id']) }),
}),
);
expect(mocks.access.album.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['album-id']));
expect(mocks.asset.getTimeBucket).toHaveBeenCalledWith('bucket', {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
albumId: 'album-id',
});
expect(mocks.asset.getTimeBucket).toHaveBeenCalledWith(
'bucket',
{
timeBucket: 'bucket',
albumId: 'album-id',
},
{
skip: 1,
take: -1,
},
);
});
it('should return the assets for a archive time bucket if user has archive.read', async () => {
@@ -52,20 +55,26 @@ describe(TimelineService.name, () => {
await expect(
sut.getTimeBucket(authStub.admin, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: true,
userId: authStub.admin.user.id,
}),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })]));
).resolves.toEqual(
expect.objectContaining({
bucketAssets: expect.objectContaining({ id: expect.arrayContaining(['asset-id']) }),
}),
);
expect(mocks.asset.getTimeBucket).toHaveBeenCalledWith(
'bucket',
expect.objectContaining({
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: true,
userIds: [authStub.admin.user.id],
}),
{
skip: 1,
take: -1,
},
);
});
@@ -75,20 +84,29 @@ describe(TimelineService.name, () => {
await expect(
sut.getTimeBucket(authStub.admin, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: false,
userId: authStub.admin.user.id,
withPartners: true,
}),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })]));
expect(mocks.asset.getTimeBucket).toHaveBeenCalledWith('bucket', {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: false,
withPartners: true,
userIds: [authStub.admin.user.id],
});
).resolves.toEqual(
expect.objectContaining({
bucketAssets: expect.objectContaining({ id: expect.arrayContaining(['asset-id']) }),
}),
);
expect(mocks.asset.getTimeBucket).toHaveBeenCalledWith(
'bucket',
{
timeBucket: 'bucket',
isArchived: false,
withPartners: true,
userIds: [authStub.admin.user.id],
},
{
skip: 1,
take: -1,
},
);
});
it('should check permissions to read tag', async () => {
@@ -97,41 +115,27 @@ describe(TimelineService.name, () => {
await expect(
sut.getTimeBucket(authStub.admin, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
userId: authStub.admin.user.id,
tagId: 'tag-123',
}),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })]));
expect(mocks.asset.getTimeBucket).toHaveBeenCalledWith('bucket', {
size: TimeBucketSize.DAY,
tagId: 'tag-123',
timeBucket: 'bucket',
userIds: [authStub.admin.user.id],
});
});
it('should strip metadata if showExif is disabled', async () => {
mocks.access.album.checkSharedLinkAccess.mockResolvedValue(new Set(['album-id']));
mocks.asset.getTimeBucket.mockResolvedValue([assetStub.image]);
const auth = factory.auth({ sharedLink: { showExif: false } });
const buckets = await sut.getTimeBucket(auth, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: true,
albumId: 'album-id',
});
expect(buckets).toEqual([expect.objectContaining({ id: 'asset-id' })]);
expect(buckets[0]).not.toHaveProperty('exif');
expect(mocks.asset.getTimeBucket).toHaveBeenCalledWith('bucket', {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: true,
albumId: 'album-id',
});
).resolves.toEqual(
expect.objectContaining({
bucketAssets: expect.objectContaining({ id: expect.arrayContaining(['asset-id']) }),
}),
);
expect(mocks.asset.getTimeBucket).toHaveBeenCalledWith(
'bucket',
{
tagId: 'tag-123',
timeBucket: 'bucket',
userIds: [authStub.admin.user.id],
},
{
skip: 1,
take: -1,
},
);
});
it('should return the assets for a library time bucket if user has library.read', async () => {
@@ -139,25 +143,30 @@ describe(TimelineService.name, () => {
await expect(
sut.getTimeBucket(authStub.admin, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
userId: authStub.admin.user.id,
}),
).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'asset-id' })]));
).resolves.toEqual(
expect.objectContaining({
bucketAssets: expect.objectContaining({ id: expect.arrayContaining(['asset-id']) }),
}),
);
expect(mocks.asset.getTimeBucket).toHaveBeenCalledWith(
'bucket',
expect.objectContaining({
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
userIds: [authStub.admin.user.id],
}),
{
skip: 1,
take: -1,
},
);
});
it('should throw an error if withParners is true and isArchived true or undefined', async () => {
await expect(
sut.getTimeBucket(authStub.admin, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: true,
withPartners: true,
@@ -167,7 +176,6 @@ describe(TimelineService.name, () => {
await expect(
sut.getTimeBucket(authStub.admin, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isArchived: undefined,
withPartners: true,
@@ -179,7 +187,6 @@ describe(TimelineService.name, () => {
it('should throw an error if withParners is true and isFavorite is either true or false', async () => {
await expect(
sut.getTimeBucket(authStub.admin, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isFavorite: true,
withPartners: true,
@@ -189,7 +196,6 @@ describe(TimelineService.name, () => {
await expect(
sut.getTimeBucket(authStub.admin, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isFavorite: false,
withPartners: true,
@@ -201,7 +207,6 @@ describe(TimelineService.name, () => {
it('should throw an error if withParners is true and isTrash is true', async () => {
await expect(
sut.getTimeBucket(authStub.admin, {
size: TimeBucketSize.DAY,
timeBucket: 'bucket',
isTrashed: true,
withPartners: true,

103
server/src/services/timeline.service.ts
View File

@@ -1,30 +1,105 @@
import { BadRequestException, Injectable } from '@nestjs/common';
import { AssetResponseDto, SanitizedAssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
import { round } from 'lodash';
import { Stack } from 'src/database';
import { AuthDto } from 'src/dtos/auth.dto';
import { TimeBucketAssetDto, TimeBucketDto, TimeBucketResponseDto } from 'src/dtos/time-bucket.dto';
import { Permission } from 'src/enum';
import {
TimeBucketAssetDto,
TimeBucketDto,
TimeBucketResponseDto,
TimeBucketsResponseDto,
} from 'src/dtos/time-bucket.dto';
import { AssetType, Permission } from 'src/enum';
import { TimeBucketOptions } from 'src/repositories/asset.repository';
import { BaseService } from 'src/services/base.service';
import { getMyPartnerIds } from 'src/utils/asset.util';
import { TimeBucketAssets } from 'src/services/timeline.service.types';
import { getMyPartnerIds, isFlipped } from 'src/utils/asset.util';
import { hexOrBufferToBase64 } from 'src/utils/bytes';
@Injectable()
export class TimelineService extends BaseService {
async getTimeBuckets(auth: AuthDto, dto: TimeBucketDto): Promise<TimeBucketResponseDto[]> {
async getTimeBuckets(auth: AuthDto, dto: TimeBucketDto): Promise<TimeBucketsResponseDto[]> {
await this.timeBucketChecks(auth, dto);
const timeBucketOptions = await this.buildTimeBucketOptions(auth, dto);
return this.assetRepository.getTimeBuckets(timeBucketOptions);
}
async getTimeBucket(
auth: AuthDto,
dto: TimeBucketAssetDto,
): Promise<AssetResponseDto[] | SanitizedAssetResponseDto[]> {
async getTimeBucket(auth: AuthDto, dto: TimeBucketAssetDto): Promise<TimeBucketResponseDto> {
await this.timeBucketChecks(auth, dto);
const timeBucketOptions = await this.buildTimeBucketOptions(auth, dto);
const assets = await this.assetRepository.getTimeBucket(dto.timeBucket, timeBucketOptions);
return !auth.sharedLink || auth.sharedLink?.showExif
? assets.map((asset) => mapAsset(asset, { withStack: true, auth }))
: assets.map((asset) => mapAsset(asset, { stripMetadata: true, auth }));
const timeBucketOptions = await this.buildTimeBucketOptions(auth, { ...dto });
const page = dto.page || 1;
const size = dto.pageSize || -1;
if (dto.pageSize === 0) {
throw new BadRequestException('pageSize must not be 0');
}
const paginate = page >= 1 && size >= 1;
const items = await this.assetRepository.getTimeBucket(dto.timeBucket, timeBucketOptions, {
skip: page,
take: size,
});
const hasNextPage = paginate && items.length > size;
if (paginate) {
items.splice(size);
}
const bucketAssets: TimeBucketAssets = {
id: [],
ownerId: [],
ratio: [],
isFavorite: [],
isArchived: [],
isTrashed: [],
isVideo: [],
isImage: [],
thumbhash: [],
localDateTime: [],
stack: [],
duration: [],
projectionType: [],
livePhotoVideoId: [],
};
for (const item of items) {
let width = item.width!;
let height = item.height!;
if (isFlipped(item.orientation)) {
const w = item.width!;
const h = item.height!;
height = w;
width = h;
}
bucketAssets.id.push(item.id);
bucketAssets.ownerId.push(item.ownerId);
bucketAssets.ratio.push(round(width / height, 2));
bucketAssets.isArchived.push(item.isArchived ? 1 : 0);
bucketAssets.isFavorite.push(item.isFavorite ? 1 : 0);
bucketAssets.isTrashed.push(item.deletedAt === null ? 0 : 1);
bucketAssets.thumbhash.push(item.thumbhash ? hexOrBufferToBase64(item.thumbhash) : 0);
bucketAssets.localDateTime.push(item.localDateTime);
bucketAssets.stack.push(this.mapStack(item.stack) || 0);
bucketAssets.duration.push(item.duration || 0);
bucketAssets.projectionType.push(item.projectionType || 0);
bucketAssets.livePhotoVideoId.push(item.livePhotoVideoId || 0);
bucketAssets.isImage.push(item.type === AssetType.IMAGE ? 1 : 0);
bucketAssets.isVideo.push(item.type === AssetType.VIDEO ? 1 : 0);
}
return {
bucketAssets,
hasNextPage,
};
}
mapStack(entity?: Stack | null) {
if (!entity) {
return;
}
return {
id: entity.id!,
primaryAssetId: entity.primaryAssetId!,
assetCount: entity.assetCount as number,
};
}
private async buildTimeBucketOptions(auth: AuthDto, dto: TimeBucketDto): Promise<TimeBucketOptions> {

22
server/src/services/timeline.service.types.ts Normal file
View File

@@ -0,0 +1,22 @@
export type TimelineStack = {
id: string;
primaryAssetId: string;
assetCount: number;
};
export type TimeBucketAssets = {
id: string[];
ownerId: string[];
ratio: number[];
isFavorite: number[];
isArchived: number[];
isTrashed: number[];
isVideo: number[];
isImage: number[];
thumbhash: (string | number)[];
localDateTime: Date[];
stack: (TimelineStack | number)[];
duration: (string | number)[];
projectionType: (string | number)[];
livePhotoVideoId: (string | number)[];
};