cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: empty and restore over 1,000 items (#12751)

Browse Source
This commit is contained in:
Jason Rasmussen
2024-09-18 09:57:52 -04:00
committed by GitHub
parent 4f25cec6df
commit 6740c67ed8
39 changed files with 540 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/services/asset-media.service.spec.ts
+14 -5
View File
@@ -4,7 +4,7 @@ import { AssetMediaStatus, AssetRejectReason, AssetUploadAction } from 'src/dtos
import { AssetMediaCreateDto, AssetMediaReplaceDto, UploadFieldName } from 'src/dtos/asset-media.dto';
import { AssetFileEntity } from 'src/entities/asset-files.entity';
import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity';
import { AssetType } from 'src/enum';
import { AssetStatus, AssetType } from 'src/enum';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IEventRepository } from 'src/interfaces/event.interface';
import { IJobRepository, JobName } from 'src/interfaces/job.interface';
@@ -478,7 +478,10 @@ describe(AssetMediaService.name, () => {
}),
);
expect(assetMock.softDeleteAll).toHaveBeenCalledWith([copiedAsset.id]);
expect(assetMock.updateAll).toHaveBeenCalledWith([copiedAsset.id], {
deletedAt: expect.any(Date),
status: AssetStatus.TRASHED,
});
expect(userMock.updateUsage).toHaveBeenCalledWith(authStub.user1.user.id, updatedFile.size);
expect(storageMock.utimes).toHaveBeenCalledWith(
updatedFile.originalPath,
@@ -506,7 +509,10 @@ describe(AssetMediaService.name, () => {
id: 'copied-asset',
});
expect(assetMock.softDeleteAll).toHaveBeenCalledWith(['copied-asset']);
expect(assetMock.updateAll).toHaveBeenCalledWith([copiedAsset.id], {
deletedAt: expect.any(Date),
status: AssetStatus.TRASHED,
});
expect(userMock.updateUsage).toHaveBeenCalledWith(authStub.user1.user.id, updatedFile.size);
expect(storageMock.utimes).toHaveBeenCalledWith(
updatedFile.originalPath,
@@ -532,7 +538,10 @@ describe(AssetMediaService.name, () => {
id: 'copied-asset',
});
expect(assetMock.softDeleteAll).toHaveBeenCalledWith(['copied-asset']);
expect(assetMock.updateAll).toHaveBeenCalledWith([copiedAsset.id], {
deletedAt: expect.any(Date),
status: AssetStatus.TRASHED,
});
expect(userMock.updateUsage).toHaveBeenCalledWith(authStub.user1.user.id, updatedFile.size);
expect(storageMock.utimes).toHaveBeenCalledWith(
updatedFile.originalPath,
@@ -561,7 +570,7 @@ describe(AssetMediaService.name, () => {
});
expect(assetMock.create).not.toHaveBeenCalled();
expect(assetMock.softDeleteAll).not.toHaveBeenCalled();
expect(assetMock.updateAll).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.DELETE_FILES,
data: { files: [updatedFile.originalPath, undefined] },
server/src/services/asset-media.service.ts
+2 -2
View File
@@ -27,7 +27,7 @@ import {
} from 'src/dtos/asset-media.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity';
import { AssetType, Permission } from 'src/enum';
import { AssetStatus, AssetType, Permission } from 'src/enum';
import { IAccessRepository } from 'src/interfaces/access.interface';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IEventRepository } from 'src/interfaces/event.interface';
@@ -193,7 +193,7 @@ export class AssetMediaService {
// but the local variable holds the original file data paths.
const copiedPhoto = await this.createCopy(asset);
// and immediate trash it
await this.assetRepository.softDeleteAll([copiedPhoto.id]);
await this.assetRepository.updateAll([copiedPhoto.id], { deletedAt: new Date(), status: AssetStatus.TRASHED });
await this.eventRepository.emit('asset.trash', { assetId: copiedPhoto.id, userId: auth.user.id });
await this.userRepository.updateUsage(auth.user.id, file.size);
server/src/services/asset.service.spec.ts
+9 -6
View File
@@ -2,7 +2,7 @@ import { BadRequestException } from '@nestjs/common';
import { mapAsset } from 'src/dtos/asset-response.dto';
import { AssetJobName, AssetStatsResponseDto } from 'src/dtos/asset.dto';
import { AssetEntity } from 'src/entities/asset.entity';
import { AssetType } from 'src/enum';
import { AssetStatus, AssetType } from 'src/enum';
import { AssetStats, IAssetRepository } from 'src/interfaces/asset.interface';
import { IEventRepository } from 'src/interfaces/event.interface';
import { IJobRepository, JobName } from 'src/interfaces/job.interface';
@@ -269,10 +269,10 @@ describe(AssetService.name, () => {
await sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'], force: true });
expect(jobMock.queueAll).toHaveBeenCalledWith([
{ name: JobName.ASSET_DELETION, data: { id: 'asset1', deleteOnDisk: true } },
{ name: JobName.ASSET_DELETION, data: { id: 'asset2', deleteOnDisk: true } },
]);
expect(eventMock.emit).toHaveBeenCalledWith('assets.delete', {
assetIds: ['asset1', 'asset2'],
userId: 'user-id',
});
});
it('should soft delete a batch of assets', async () => {
@@ -280,7 +280,10 @@ describe(AssetService.name, () => {
await sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'], force: false });
expect(assetMock.softDeleteAll).toHaveBeenCalledWith(['asset1', 'asset2']);
expect(assetMock.updateAll).toHaveBeenCalledWith(['asset1', 'asset2'], {
deletedAt: expect.any(Date),
status: AssetStatus.TRASHED,
});
expect(jobMock.queue.mock.calls).toEqual([]);
});
});
server/src/services/asset.service.ts
+6 -13
View File
@@ -20,7 +20,7 @@ import {
import { AuthDto } from 'src/dtos/auth.dto';
import { MemoryLaneDto } from 'src/dtos/search.dto';
import { AssetEntity } from 'src/entities/asset.entity';
import { Permission } from 'src/enum';
import { AssetStatus, Permission } from 'src/enum';
import { IAccessRepository } from 'src/interfaces/access.interface';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IEventRepository } from 'src/interfaces/event.interface';
@@ -302,18 +302,11 @@ export class AssetService {
const { ids, force } = dto;
await requireAccess(this.access, { auth, permission: Permission.ASSET_DELETE, ids });
if (force) {
await this.jobRepository.queueAll(
ids.map((id) => ({
name: JobName.ASSET_DELETION,
data: { id, deleteOnDisk: true },
})),
);
} else {
await this.assetRepository.softDeleteAll(ids);
await this.eventRepository.emit('assets.trash', { assetIds: ids, userId: auth.user.id });
}
await this.assetRepository.updateAll(ids, {
deletedAt: new Date(),
status: force ? AssetStatus.DELETED : AssetStatus.TRASHED,
});
await this.eventRepository.emit(force ? 'assets.delete' : 'assets.trash', { assetIds: ids, userId: auth.user.id });
}
async run(auth: AuthDto, dto: AssetJobsDto) {
server/src/services/microservices.service.ts
+3
View File
@@ -16,6 +16,7 @@ import { SmartInfoService } from 'src/services/smart-info.service';
import { StorageTemplateService } from 'src/services/storage-template.service';
import { StorageService } from 'src/services/storage.service';
import { TagService } from 'src/services/tag.service';
import { TrashService } from 'src/services/trash.service';
import { UserService } from 'src/services/user.service';
import { VersionService } from 'src/services/version.service';
import { otelShutdown } from 'src/utils/instrumentation';
@@ -36,6 +37,7 @@ export class MicroservicesService {
private storageTemplateService: StorageTemplateService,
private storageService: StorageService,
private tagService: TagService,
private trashService: TrashService,
private userService: UserService,
private duplicateService: DuplicateService,
private versionService: VersionService,
@@ -97,6 +99,7 @@ export class MicroservicesService {
[JobName.NOTIFY_SIGNUP]: (data) => this.notificationService.handleUserSignup(data),
[JobName.TAG_CLEANUP]: () => this.tagService.handleTagCleanup(),
[JobName.VERSION_CHECK]: () => this.versionService.handleVersionCheck(),
[JobName.QUEUE_TRASH_EMPTY]: () => this.trashService.handleQueueEmptyTrash(),
});
}
server/src/services/trash.service.spec.ts
+53 -24
View File
@@ -1,22 +1,24 @@
import { BadRequestException } from '@nestjs/common';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IEventRepository } from 'src/interfaces/event.interface';
import { IJobRepository, JobName } from 'src/interfaces/job.interface';
import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { ITrashRepository } from 'src/interfaces/trash.interface';
import { TrashService } from 'src/services/trash.service';
import { assetStub } from 'test/fixtures/asset.stub';
import { authStub } from 'test/fixtures/auth.stub';
import { IAccessRepositoryMock, newAccessRepositoryMock } from 'test/repositories/access.repository.mock';
import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
import { newEventRepositoryMock } from 'test/repositories/event.repository.mock';
import { newJobRepositoryMock } from 'test/repositories/job.repository.mock';
import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
import { newTrashRepositoryMock } from 'test/repositories/trash.repository.mock';
import { Mocked } from 'vitest';
describe(TrashService.name, () => {
let sut: TrashService;
let accessMock: IAccessRepositoryMock;
let assetMock: Mocked<IAssetRepository>;
let jobMock: Mocked<IJobRepository>;
let eventMock: Mocked<IEventRepository>;
let jobMock: Mocked<IJobRepository>;
let trashMock: Mocked<ITrashRepository>;
let loggerMock: Mocked<ILoggerRepository>;
it('should work', () => {
expect(sut).toBeDefined();
@@ -24,11 +26,12 @@ describe(TrashService.name, () => {
beforeEach(() => {
accessMock = newAccessRepositoryMock();
assetMock = newAssetRepositoryMock();
eventMock = newEventRepositoryMock();
jobMock = newJobRepositoryMock();
trashMock = newTrashRepositoryMock();
loggerMock = newLoggerRepositoryMock();
sut = new TrashService(accessMock, assetMock, jobMock, eventMock);
sut = new TrashService(accessMock, eventMock, jobMock, trashMock, loggerMock);
});
describe('restoreAssets', () => {
@@ -40,44 +43,70 @@ describe(TrashService.name, () => {
).rejects.toBeInstanceOf(BadRequestException);
});
it('should handle an empty list', async () => {
await expect(sut.restoreAssets(authStub.user1, { ids: [] })).resolves.toEqual({ count: 0 });
expect(accessMock.asset.checkOwnerAccess).not.toHaveBeenCalled();
});
it('should restore a batch of assets', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset1', 'asset2']));
await sut.restoreAssets(authStub.user1, { ids: ['asset1', 'asset2'] });
expect(assetMock.restoreAll).toHaveBeenCalledWith(['asset1', 'asset2']);
expect(trashMock.restoreAll).toHaveBeenCalledWith(['asset1', 'asset2']);
expect(jobMock.queue.mock.calls).toEqual([]);
});
});
describe('restore', () => {
it('should handle an empty trash', async () => {
assetMock.getByUserId.mockResolvedValue({ items: [], hasNextPage: false });
await expect(sut.restore(authStub.user1)).resolves.toBeUndefined();
expect(assetMock.restoreAll).not.toHaveBeenCalled();
expect(eventMock.clientSend).not.toHaveBeenCalled();
trashMock.getDeletedIds.mockResolvedValue({ items: [], hasNextPage: false });
trashMock.restore.mockResolvedValue(0);
await expect(sut.restore(authStub.user1)).resolves.toEqual({ count: 0 });
expect(trashMock.restore).toHaveBeenCalledWith('user-id');
});
it('should restore and notify', async () => {
assetMock.getByUserId.mockResolvedValue({ items: [assetStub.image], hasNextPage: false });
await expect(sut.restore(authStub.user1)).resolves.toBeUndefined();
expect(assetMock.restoreAll).toHaveBeenCalledWith([assetStub.image.id]);
expect(eventMock.emit).toHaveBeenCalledWith('assets.restore', { assetIds: ['asset-id'], userId: 'user-id' });
it('should restore', async () => {
trashMock.getDeletedIds.mockResolvedValue({ items: ['asset-id'], hasNextPage: false });
trashMock.restore.mockResolvedValue(1);
await expect(sut.restore(authStub.user1)).resolves.toEqual({ count: 1 });
expect(trashMock.restore).toHaveBeenCalledWith('user-id');
});
});
describe('empty', () => {
it('should handle an empty trash', async () => {
assetMock.getByUserId.mockResolvedValue({ items: [], hasNextPage: false });
await expect(sut.empty(authStub.user1)).resolves.toBeUndefined();
expect(jobMock.queueAll).toHaveBeenCalledWith([]);
trashMock.getDeletedIds.mockResolvedValue({ items: [], hasNextPage: false });
trashMock.empty.mockResolvedValue(0);
await expect(sut.empty(authStub.user1)).resolves.toEqual({ count: 0 });
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should empty the trash', async () => {
assetMock.getByUserId.mockResolvedValue({ items: [assetStub.image], hasNextPage: false });
await expect(sut.empty(authStub.user1)).resolves.toBeUndefined();
trashMock.getDeletedIds.mockResolvedValue({ items: ['asset-id'], hasNextPage: false });
trashMock.empty.mockResolvedValue(1);
await expect(sut.empty(authStub.user1)).resolves.toEqual({ count: 1 });
expect(trashMock.empty).toHaveBeenCalledWith('user-id');
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_TRASH_EMPTY, data: {} });
});
});
describe('onAssetsDelete', () => {
it('should queue the empty trash job', async () => {
await expect(sut.onAssetsDelete()).resolves.toBeUndefined();
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_TRASH_EMPTY, data: {} });
});
});
describe('handleQueueEmptyTrash', () => {
it('should queue asset delete jobs', async () => {
trashMock.getDeletedIds.mockResolvedValue({ items: ['asset-1'], hasNextPage: false });
await expect(sut.handleQueueEmptyTrash()).resolves.toEqual(JobStatus.SUCCESS);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{ name: JobName.ASSET_DELETION, data: { id: assetStub.image.id, deleteOnDisk: true } },
{
name: JobName.ASSET_DELETION,
data: { id: 'asset-1', deleteOnDisk: true },
},
]);
});
});
server/src/services/trash.service.ts
+53 -36
View File
@@ -1,69 +1,86 @@
import { Inject } from '@nestjs/common';
import { DateTime } from 'luxon';
import { OnEmit } from 'src/decorators';
import { BulkIdsDto } from 'src/dtos/asset-ids.response.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { TrashResponseDto } from 'src/dtos/trash.dto';
import { Permission } from 'src/enum';
import { IAccessRepository } from 'src/interfaces/access.interface';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IEventRepository } from 'src/interfaces/event.interface';
import { IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName } from 'src/interfaces/job.interface';
import { IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName, JobStatus } from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { ITrashRepository } from 'src/interfaces/trash.interface';
import { requireAccess } from 'src/utils/access';
import { usePagination } from 'src/utils/pagination';
export class TrashService {
constructor(
@Inject(IAccessRepository) private access: IAccessRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IEventRepository) private eventRepository: IEventRepository,
) {}
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ITrashRepository) private trashRepository: ITrashRepository,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
) {
this.logger.setContext(TrashService.name);
}
async restoreAssets(auth: AuthDto, dto: BulkIdsDto): Promise<void> {
async restoreAssets(auth: AuthDto, dto: BulkIdsDto): Promise<TrashResponseDto> {
const { ids } = dto;
await requireAccess(this.access, { auth, permission: Permission.ASSET_DELETE, ids });
await this.restoreAndSend(auth, ids);
}
async restore(auth: AuthDto): Promise<void> {
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getByUserId(pagination, auth.user.id, {
trashedBefore: DateTime.now().toJSDate(),
}),
);
for await (const assets of assetPagination) {
const ids = assets.map((a) => a.id);
await this.restoreAndSend(auth, ids);
if (ids.length === 0) {
return { count: 0 };
}
await requireAccess(this.access, { auth, permission: Permission.ASSET_DELETE, ids });
await this.trashRepository.restoreAll(ids);
await this.eventRepository.emit('assets.restore', { assetIds: ids, userId: auth.user.id });
this.logger.log(`Restored ${ids.length} assets from trash`);
return { count: ids.length };
}
async empty(auth: AuthDto): Promise<void> {
async restore(auth: AuthDto): Promise<TrashResponseDto> {
const count = await this.trashRepository.restore(auth.user.id);
if (count > 0) {
this.logger.log(`Restored ${count} assets from trash`);
}
return { count };
}
async empty(auth: AuthDto): Promise<TrashResponseDto> {
const count = await this.trashRepository.empty(auth.user.id);
if (count > 0) {
await this.jobRepository.queue({ name: JobName.QUEUE_TRASH_EMPTY, data: {} });
}
return { count };
}
@OnEmit({ event: 'assets.delete' })
async onAssetsDelete() {
await this.jobRepository.queue({ name: JobName.QUEUE_TRASH_EMPTY, data: {} });
}
async handleQueueEmptyTrash() {
let count = 0;
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getByUserId(pagination, auth.user.id, {
trashedBefore: DateTime.now().toJSDate(),
withArchived: true,
}),
this.trashRepository.getDeletedIds(pagination),
);
for await (const assets of assetPagination) {
for await (const assetIds of assetPagination) {
this.logger.debug(`Queueing ${assetIds.length} assets for deletion from the trash`);
count += assetIds.length;
await this.jobRepository.queueAll(
assets.map((asset) => ({
assetIds.map((assetId) => ({
name: JobName.ASSET_DELETION,
data: {
id: asset.id,
id: assetId,
deleteOnDisk: true,
},
})),
);
}
}
private async restoreAndSend(auth: AuthDto, ids: string[]) {
if (ids.length === 0) {
return;
}
this.logger.log(`Queued ${count} assets for deletion from the trash`);
await this.assetRepository.restoreAll(ids);
await this.eventRepository.emit('assets.restore', { assetIds: ids, userId: auth.user.id });
return JobStatus.SUCCESS;
}
}