feat: storage template file move hardening (#5917)
* fix: pgvecto.rs extension breaks typeorm schema:drop command * fix: parse postgres bigints to javascript number types when selecting data * feat: verify file size is the same as original asset after copying file for storage template job * feat: allow disabling of storage template job, defaults to disabled for new instances * fix: don't allow setting concurrency for storage template migration, can cause race conditions above 1 * feat: add checksum verification when file is copied for storage template job * fix: extract metadata for assets that aren't visible on timeline
This commit is contained in:
Zack Pollard
@@ -1,8 +1,21 @@
|
||||
import { AssetPathType } from '@app/infra/entities';
|
||||
import {
|
||||
IAlbumRepository,
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IMoveRepository,
|
||||
IPersonRepository,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
IUserRepository,
|
||||
StorageTemplateService,
|
||||
defaults,
|
||||
} from '@app/domain';
|
||||
import { AssetPathType, SystemConfigKey } from '@app/infra/entities';
|
||||
import {
|
||||
assetStub,
|
||||
newAlbumRepositoryMock,
|
||||
newAssetRepositoryMock,
|
||||
newCryptoRepositoryMock,
|
||||
newMoveRepositoryMock,
|
||||
newPersonRepositoryMock,
|
||||
newStorageRepositoryMock,
|
||||
@@ -11,17 +24,7 @@ import {
|
||||
userStub,
|
||||
} from '@test';
|
||||
import { when } from 'jest-when';
|
||||
import {
|
||||
IAlbumRepository,
|
||||
IAssetRepository,
|
||||
IMoveRepository,
|
||||
IPersonRepository,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
IUserRepository,
|
||||
} from '../repositories';
|
||||
import { defaults } from '../system-config/system-config.core';
|
||||
import { StorageTemplateService } from './storage-template.service';
|
||||
import { Stats } from 'node:fs';
|
||||
|
||||
describe(StorageTemplateService.name, () => {
|
||||
let sut: StorageTemplateService;
|
||||
@@ -32,19 +35,21 @@ describe(StorageTemplateService.name, () => {
|
||||
let personMock: jest.Mocked<IPersonRepository>;
|
||||
let storageMock: jest.Mocked<IStorageRepository>;
|
||||
let userMock: jest.Mocked<IUserRepository>;
|
||||
let cryptoMock: jest.Mocked<ICryptoRepository>;
|
||||
|
||||
it('should work', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
configMock = newSystemConfigRepositoryMock();
|
||||
assetMock = newAssetRepositoryMock();
|
||||
albumMock = newAlbumRepositoryMock();
|
||||
configMock = newSystemConfigRepositoryMock();
|
||||
moveMock = newMoveRepositoryMock();
|
||||
personMock = newPersonRepositoryMock();
|
||||
storageMock = newStorageRepositoryMock();
|
||||
userMock = newUserRepositoryMock();
|
||||
cryptoMock = newCryptoRepositoryMock();
|
||||
|
||||
sut = new StorageTemplateService(
|
||||
albumMock,
|
||||
@@ -55,27 +60,37 @@ describe(StorageTemplateService.name, () => {
|
||||
personMock,
|
||||
storageMock,
|
||||
userMock,
|
||||
cryptoMock,
|
||||
);
|
||||
|
||||
configMock.load.mockResolvedValue([{ key: SystemConfigKey.STORAGE_TEMPLATE_ENABLED, value: true }]);
|
||||
});
|
||||
|
||||
describe('handleMigrationSingle', () => {
|
||||
it('should skip when storage template is disabled', async () => {
|
||||
configMock.load.mockResolvedValue([{ key: SystemConfigKey.STORAGE_TEMPLATE_ENABLED, value: false }]);
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
expect(assetMock.getByIds).not.toHaveBeenCalled();
|
||||
expect(storageMock.checkFileExists).not.toHaveBeenCalled();
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
expect(moveMock.create).not.toHaveBeenCalled();
|
||||
expect(moveMock.update).not.toHaveBeenCalled();
|
||||
expect(storageMock.stat).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should migrate single moving picture', async () => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const path = (id: string) => `upload/library/${userStub.user1.id}/2023/2023-02-23/${id}.jpg`;
|
||||
const newPath = (id: string) => `upload/library/${userStub.user1.id}/2023/2023-02-23/${id}+1.jpg`;
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(path(assetStub.livePhotoStillAsset.id)).mockResolvedValue(true);
|
||||
when(storageMock.checkFileExists).calledWith(newPath(assetStub.livePhotoStillAsset.id)).mockResolvedValue(false);
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(path(assetStub.livePhotoMotionAsset.id)).mockResolvedValue(true);
|
||||
when(storageMock.checkFileExists).calledWith(newPath(assetStub.livePhotoMotionAsset.id)).mockResolvedValue(false);
|
||||
const newMotionPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.mp4`;
|
||||
const newStillPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.jpeg`;
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.livePhotoStillAsset.id, originalPath: newPath(assetStub.livePhotoStillAsset.id) })
|
||||
.calledWith({ id: assetStub.livePhotoStillAsset.id, originalPath: newStillPicturePath })
|
||||
.mockResolvedValue(assetStub.livePhotoStillAsset);
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.livePhotoMotionAsset.id, originalPath: newPath(assetStub.livePhotoMotionAsset.id) })
|
||||
.calledWith({ id: assetStub.livePhotoMotionAsset.id, originalPath: newMotionPicturePath })
|
||||
.mockResolvedValue(assetStub.livePhotoMotionAsset);
|
||||
|
||||
when(assetMock.getByIds)
|
||||
@@ -86,11 +101,265 @@ describe(StorageTemplateService.name, () => {
|
||||
.calledWith([assetStub.livePhotoMotionAsset.id])
|
||||
.mockResolvedValue([assetStub.livePhotoMotionAsset]);
|
||||
|
||||
when(moveMock.create)
|
||||
.calledWith({
|
||||
entityId: assetStub.livePhotoStillAsset.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.livePhotoStillAsset.originalPath,
|
||||
newPath: newStillPicturePath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.livePhotoStillAsset.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.livePhotoStillAsset.originalPath,
|
||||
newPath: newStillPicturePath,
|
||||
});
|
||||
|
||||
when(moveMock.create)
|
||||
.calledWith({
|
||||
entityId: assetStub.livePhotoMotionAsset.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.livePhotoMotionAsset.originalPath,
|
||||
newPath: newMotionPicturePath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '124',
|
||||
entityId: assetStub.livePhotoMotionAsset.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.livePhotoMotionAsset.originalPath,
|
||||
newPath: newMotionPicturePath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id]);
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2);
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.livePhotoStillAsset.id,
|
||||
originalPath: newStillPicturePath,
|
||||
});
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.livePhotoMotionAsset.id,
|
||||
originalPath: newMotionPicturePath,
|
||||
});
|
||||
});
|
||||
it('should migrate previously failed move from original path when it still exists', async () => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
|
||||
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(true);
|
||||
when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(false);
|
||||
|
||||
when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: previousFailedNewPath,
|
||||
});
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.image.id, originalPath: newPath })
|
||||
.mockResolvedValue(assetStub.image);
|
||||
|
||||
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
|
||||
|
||||
when(moveMock.update)
|
||||
.calledWith({
|
||||
id: '123',
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
|
||||
expect(moveMock.update).toHaveBeenCalledWith({
|
||||
id: '123',
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
});
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.image.id,
|
||||
originalPath: newPath,
|
||||
});
|
||||
});
|
||||
it('should migrate previously failed move from previous new path when old path no longer exists, should validate file size still matches before moving', async () => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
|
||||
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(false);
|
||||
when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(true);
|
||||
when(storageMock.stat)
|
||||
.calledWith(previousFailedNewPath)
|
||||
.mockResolvedValue({ size: 5000 } as Stats);
|
||||
when(cryptoMock.hashFile).calledWith(previousFailedNewPath).mockResolvedValue(assetStub.image.checksum);
|
||||
|
||||
when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: previousFailedNewPath,
|
||||
});
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.image.id, originalPath: newPath })
|
||||
.mockResolvedValue(assetStub.image);
|
||||
|
||||
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
|
||||
|
||||
when(moveMock.update)
|
||||
.calledWith({
|
||||
id: '123',
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(moveMock.update).toHaveBeenCalledWith({
|
||||
id: '123',
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
});
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.image.id,
|
||||
originalPath: newPath,
|
||||
});
|
||||
});
|
||||
|
||||
it('should fail move if copying and hash of asset and the new file do not match', async () => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
|
||||
|
||||
when(storageMock.rename).calledWith(assetStub.image.originalPath, newPath).mockRejectedValue({ code: 'EXDEV' });
|
||||
when(storageMock.stat)
|
||||
.calledWith(newPath)
|
||||
.mockResolvedValue({ size: 5000 } as Stats);
|
||||
when(cryptoMock.hashFile).calledWith(newPath).mockResolvedValue(Buffer.from('different-hash', 'utf-8'));
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.image.id, originalPath: newPath })
|
||||
.mockResolvedValue(assetStub.image);
|
||||
|
||||
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
|
||||
|
||||
when(moveMock.create)
|
||||
.calledWith({
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: newPath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(1);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith(newPath);
|
||||
expect(moveMock.create).toHaveBeenCalledWith({
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: newPath,
|
||||
});
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
|
||||
expect(storageMock.copyFile).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
|
||||
expect(storageMock.unlink).toHaveBeenCalledWith(newPath);
|
||||
expect(storageMock.unlink).toHaveBeenCalledTimes(1);
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each`
|
||||
failedPathChecksum | failedPathSize | reason
|
||||
${assetStub.image.checksum} | ${500} | ${'file size'}
|
||||
${Buffer.from('bad checksum', 'utf-8')} | ${assetStub.image.exifInfo?.fileSizeInByte} | ${'checksum'}
|
||||
`(
|
||||
'should fail to migrate previously failed move from previous new path when old path no longer exists if $reason validation fails',
|
||||
async ({ failedPathChecksum, failedPathSize }) => {
|
||||
userMock.get.mockResolvedValue(userStub.user1);
|
||||
const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
|
||||
const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
|
||||
|
||||
when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(false);
|
||||
when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(true);
|
||||
when(storageMock.stat)
|
||||
.calledWith(previousFailedNewPath)
|
||||
.mockResolvedValue({ size: failedPathSize } as Stats);
|
||||
when(cryptoMock.hashFile).calledWith(previousFailedNewPath).mockResolvedValue(failedPathChecksum);
|
||||
|
||||
when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: previousFailedNewPath,
|
||||
});
|
||||
|
||||
when(assetMock.save)
|
||||
.calledWith({ id: assetStub.image.id, originalPath: newPath })
|
||||
.mockResolvedValue(assetStub.image);
|
||||
|
||||
when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
|
||||
|
||||
when(moveMock.update)
|
||||
.calledWith({
|
||||
id: '123',
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
})
|
||||
.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: previousFailedNewPath,
|
||||
newPath,
|
||||
});
|
||||
|
||||
await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
|
||||
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
|
||||
expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(moveMock.update).not.toHaveBeenCalled();
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('handle template migration', () => {
|
||||
@@ -155,7 +424,8 @@ describe(StorageTemplateService.name, () => {
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -175,7 +445,8 @@ describe(StorageTemplateService.name, () => {
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -198,7 +469,7 @@ describe(StorageTemplateService.name, () => {
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).toHaveBeenCalledWith(
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
@@ -226,7 +497,7 @@ describe(StorageTemplateService.name, () => {
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).toHaveBeenCalledWith(
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/label-1/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
@@ -236,12 +507,84 @@ describe(StorageTemplateService.name, () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('should copy the file if rename fails due to EXDEV (rename across filesystems)', async () => {
|
||||
const newPath = 'upload/library/user-id/2023/2023-02-23/asset-id.jpg';
|
||||
assetMock.getAll.mockResolvedValue({
|
||||
items: [assetStub.image],
|
||||
hasNextPage: false,
|
||||
});
|
||||
storageMock.rename.mockRejectedValue({ code: 'EXDEV' });
|
||||
userMock.getList.mockResolvedValue([userStub.user1]);
|
||||
moveMock.create.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath,
|
||||
});
|
||||
when(storageMock.stat)
|
||||
.calledWith(newPath)
|
||||
.mockResolvedValue({
|
||||
size: 5000,
|
||||
} as Stats);
|
||||
when(cryptoMock.hashFile).calledWith(newPath).mockResolvedValue(assetStub.image.checksum);
|
||||
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.rename).toHaveBeenCalledWith('/original/path.jpg', newPath);
|
||||
expect(storageMock.copyFile).toHaveBeenCalledWith('/original/path.jpg', newPath);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith(newPath);
|
||||
expect(storageMock.unlink).toHaveBeenCalledWith(assetStub.image.originalPath);
|
||||
expect(storageMock.unlink).toHaveBeenCalledTimes(1);
|
||||
expect(assetMock.save).toHaveBeenCalledWith({
|
||||
id: assetStub.image.id,
|
||||
originalPath: newPath,
|
||||
});
|
||||
});
|
||||
|
||||
it('should not update the database if the move fails due to incorrect newPath filesize', async () => {
|
||||
assetMock.getAll.mockResolvedValue({
|
||||
items: [assetStub.image],
|
||||
hasNextPage: false,
|
||||
});
|
||||
storageMock.rename.mockRejectedValue({ code: 'EXDEV' });
|
||||
userMock.getList.mockResolvedValue([userStub.user1]);
|
||||
moveMock.create.mockResolvedValue({
|
||||
id: '123',
|
||||
entityId: assetStub.image.id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath: assetStub.image.originalPath,
|
||||
newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
});
|
||||
when(storageMock.stat)
|
||||
.calledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg')
|
||||
.mockResolvedValue({
|
||||
size: 100,
|
||||
} as Stats);
|
||||
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
expect(storageMock.copyFile).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
expect(storageMock.stat).toHaveBeenCalledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg');
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not update the database if the move fails', async () => {
|
||||
assetMock.getAll.mockResolvedValue({
|
||||
items: [assetStub.image],
|
||||
hasNextPage: false,
|
||||
});
|
||||
storageMock.moveFile.mockRejectedValue(new Error('Read only system'));
|
||||
storageMock.rename.mockRejectedValue(new Error('Read only system'));
|
||||
storageMock.copyFile.mockRejectedValue(new Error('Read only system'));
|
||||
moveMock.create.mockResolvedValue({
|
||||
id: 'move-123',
|
||||
entityId: '123',
|
||||
@@ -254,7 +597,7 @@ describe(StorageTemplateService.name, () => {
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).toHaveBeenCalledWith(
|
||||
expect(storageMock.rename).toHaveBeenCalledWith(
|
||||
'/original/path.jpg',
|
||||
'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
|
||||
);
|
||||
@@ -278,7 +621,8 @@ describe(StorageTemplateService.name, () => {
|
||||
await sut.handleMigration();
|
||||
|
||||
expect(assetMock.getAll).toHaveBeenCalled();
|
||||
expect(storageMock.moveFile).not.toHaveBeenCalled();
|
||||
expect(storageMock.rename).not.toHaveBeenCalled();
|
||||
expect(storageMock.copyFile).not.toHaveBeenCalled();
|
||||
expect(assetMock.save).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,6 +10,7 @@ import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE } from '../job';
|
||||
import {
|
||||
IAlbumRepository,
|
||||
IAssetRepository,
|
||||
ICryptoRepository,
|
||||
IMoveRepository,
|
||||
IPersonRepository,
|
||||
IStorageRepository,
|
||||
@@ -61,6 +62,7 @@ export class StorageTemplateService {
|
||||
@Inject(IPersonRepository) personRepository: IPersonRepository,
|
||||
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
||||
@Inject(IUserRepository) private userRepository: IUserRepository,
|
||||
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
|
||||
) {
|
||||
this.template = this.compile(config.storageTemplate.template);
|
||||
this.configCore = SystemConfigCore.create(configRepository);
|
||||
@@ -70,10 +72,22 @@ export class StorageTemplateService {
|
||||
this.logger.debug(`Received config, compiling storage template: ${template}`);
|
||||
this.template = this.compile(template);
|
||||
});
|
||||
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
|
||||
this.storageCore = StorageCore.create(
|
||||
assetRepository,
|
||||
moveRepository,
|
||||
personRepository,
|
||||
cryptoRepository,
|
||||
configRepository,
|
||||
storageRepository,
|
||||
);
|
||||
}
|
||||
|
||||
async handleMigrationSingle({ id }: IEntityJob) {
|
||||
const storageTemplateEnabled = (await this.configCore.getConfig()).storageTemplate.enabled;
|
||||
if (!storageTemplateEnabled) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const [asset] = await this.assetRepository.getByIds([id]);
|
||||
|
||||
const user = await this.userRepository.get(asset.ownerId, {});
|
||||
@@ -93,6 +107,11 @@ export class StorageTemplateService {
|
||||
|
||||
async handleMigration() {
|
||||
this.logger.log('Starting storage template migration');
|
||||
const storageTemplateEnabled = (await this.configCore.getConfig()).storageTemplate.enabled;
|
||||
if (!storageTemplateEnabled) {
|
||||
this.logger.log('Storage template migration disabled, skipping');
|
||||
return true;
|
||||
}
|
||||
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
|
||||
this.assetRepository.getAll(pagination),
|
||||
);
|
||||
@@ -123,12 +142,23 @@ export class StorageTemplateService {
|
||||
return;
|
||||
}
|
||||
|
||||
const { id, sidecarPath, originalPath } = asset;
|
||||
const { id, sidecarPath, originalPath, exifInfo } = asset;
|
||||
const oldPath = originalPath;
|
||||
const newPath = await this.getTemplatePath(asset, metadata);
|
||||
|
||||
if (!exifInfo || !exifInfo.fileSizeInByte) {
|
||||
this.logger.error(`Asset ${id} missing exif info, skipping storage template migration`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.storageCore.moveFile({ entityId: id, pathType: AssetPathType.ORIGINAL, oldPath, newPath });
|
||||
await this.storageCore.moveFile({
|
||||
entityId: id,
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
oldPath,
|
||||
newPath,
|
||||
assetInfo: { sizeInBytes: exifInfo.fileSizeInByte, checksum: asset.checksum },
|
||||
});
|
||||
if (sidecarPath) {
|
||||
await this.storageCore.moveFile({
|
||||
entityId: id,
|
||||
|
||||
Reference in New Issue
Block a user