chore: remove audit file report (#17994)
This commit is contained in:
Jason Rasmussen
@@ -1,6 +1,4 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { FileReportItemDto } from 'src/dtos/audit.dto';
|
||||
import { AssetFileType, AssetPathType, JobStatus, PersonPathType, UserPathType } from 'src/enum';
|
||||
import { JobStatus } from 'src/enum';
|
||||
import { AuditService } from 'src/services/audit.service';
|
||||
import { newTestService, ServiceMocks } from 'test/utils';
|
||||
|
||||
@@ -25,148 +23,4 @@ describe(AuditService.name, () => {
|
||||
expect(mocks.audit.removeBefore).toHaveBeenCalledWith(expect.any(Date));
|
||||
});
|
||||
});
|
||||
|
||||
describe('getChecksums', () => {
|
||||
it('should fail if the file is not in the immich path', async () => {
|
||||
await expect(sut.getChecksums({ filenames: ['foo/bar'] })).rejects.toBeInstanceOf(BadRequestException);
|
||||
|
||||
expect(mocks.crypto.hashFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should get checksum for valid file', async () => {
|
||||
await expect(sut.getChecksums({ filenames: ['./upload/my-file.jpg'] })).resolves.toEqual([
|
||||
{ filename: './upload/my-file.jpg', checksum: expect.any(String) },
|
||||
]);
|
||||
|
||||
expect(mocks.crypto.hashFile).toHaveBeenCalledWith('./upload/my-file.jpg');
|
||||
});
|
||||
});
|
||||
|
||||
describe('fixItems', () => {
|
||||
it('should fail if the file is not in the immich path', async () => {
|
||||
await expect(
|
||||
sut.fixItems([
|
||||
{ entityId: 'my-id', pathType: AssetPathType.ORIGINAL, pathValue: 'foo/bar' } as FileReportItemDto,
|
||||
]),
|
||||
).rejects.toBeInstanceOf(BadRequestException);
|
||||
|
||||
expect(mocks.asset.update).not.toHaveBeenCalled();
|
||||
expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
|
||||
expect(mocks.person.update).not.toHaveBeenCalled();
|
||||
expect(mocks.user.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should update encoded video path', async () => {
|
||||
await sut.fixItems([
|
||||
{
|
||||
entityId: 'my-id',
|
||||
pathType: AssetPathType.ENCODED_VIDEO,
|
||||
pathValue: './upload/my-video.mp4',
|
||||
} as FileReportItemDto,
|
||||
]);
|
||||
|
||||
expect(mocks.asset.update).toHaveBeenCalledWith({ id: 'my-id', encodedVideoPath: './upload/my-video.mp4' });
|
||||
expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
|
||||
expect(mocks.person.update).not.toHaveBeenCalled();
|
||||
expect(mocks.user.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should update preview path', async () => {
|
||||
await sut.fixItems([
|
||||
{
|
||||
entityId: 'my-id',
|
||||
pathType: AssetPathType.PREVIEW,
|
||||
pathValue: './upload/my-preview.png',
|
||||
} as FileReportItemDto,
|
||||
]);
|
||||
|
||||
expect(mocks.asset.upsertFile).toHaveBeenCalledWith({
|
||||
assetId: 'my-id',
|
||||
type: AssetFileType.PREVIEW,
|
||||
path: './upload/my-preview.png',
|
||||
});
|
||||
expect(mocks.asset.update).not.toHaveBeenCalled();
|
||||
expect(mocks.person.update).not.toHaveBeenCalled();
|
||||
expect(mocks.user.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should update thumbnail path', async () => {
|
||||
await sut.fixItems([
|
||||
{
|
||||
entityId: 'my-id',
|
||||
pathType: AssetPathType.THUMBNAIL,
|
||||
pathValue: './upload/my-thumbnail.webp',
|
||||
} as FileReportItemDto,
|
||||
]);
|
||||
|
||||
expect(mocks.asset.upsertFile).toHaveBeenCalledWith({
|
||||
assetId: 'my-id',
|
||||
type: AssetFileType.THUMBNAIL,
|
||||
path: './upload/my-thumbnail.webp',
|
||||
});
|
||||
expect(mocks.asset.update).not.toHaveBeenCalled();
|
||||
expect(mocks.person.update).not.toHaveBeenCalled();
|
||||
expect(mocks.user.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should update original path', async () => {
|
||||
await sut.fixItems([
|
||||
{
|
||||
entityId: 'my-id',
|
||||
pathType: AssetPathType.ORIGINAL,
|
||||
pathValue: './upload/my-original.png',
|
||||
} as FileReportItemDto,
|
||||
]);
|
||||
|
||||
expect(mocks.asset.update).toHaveBeenCalledWith({ id: 'my-id', originalPath: './upload/my-original.png' });
|
||||
expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
|
||||
expect(mocks.person.update).not.toHaveBeenCalled();
|
||||
expect(mocks.user.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should update sidecar path', async () => {
|
||||
await sut.fixItems([
|
||||
{
|
||||
entityId: 'my-id',
|
||||
pathType: AssetPathType.SIDECAR,
|
||||
pathValue: './upload/my-sidecar.xmp',
|
||||
} as FileReportItemDto,
|
||||
]);
|
||||
|
||||
expect(mocks.asset.update).toHaveBeenCalledWith({ id: 'my-id', sidecarPath: './upload/my-sidecar.xmp' });
|
||||
expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
|
||||
expect(mocks.person.update).not.toHaveBeenCalled();
|
||||
expect(mocks.user.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should update face path', async () => {
|
||||
await sut.fixItems([
|
||||
{
|
||||
entityId: 'my-id',
|
||||
pathType: PersonPathType.FACE,
|
||||
pathValue: './upload/my-face.jpg',
|
||||
} as FileReportItemDto,
|
||||
]);
|
||||
|
||||
expect(mocks.person.update).toHaveBeenCalledWith({ id: 'my-id', thumbnailPath: './upload/my-face.jpg' });
|
||||
expect(mocks.asset.update).not.toHaveBeenCalled();
|
||||
expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
|
||||
expect(mocks.user.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should update profile path', async () => {
|
||||
await sut.fixItems([
|
||||
{
|
||||
entityId: 'my-id',
|
||||
pathType: UserPathType.PROFILE,
|
||||
pathValue: './upload/my-profile-pic.jpg',
|
||||
} as FileReportItemDto,
|
||||
]);
|
||||
|
||||
expect(mocks.user.update).toHaveBeenCalledWith('my-id', { profileImagePath: './upload/my-profile-pic.jpg' });
|
||||
expect(mocks.asset.update).not.toHaveBeenCalled();
|
||||
expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
|
||||
expect(mocks.person.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,23 +1,9 @@
|
||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { DateTime } from 'luxon';
|
||||
import { resolve } from 'node:path';
|
||||
import { AUDIT_LOG_MAX_DURATION, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
|
||||
import { StorageCore } from 'src/cores/storage.core';
|
||||
import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
|
||||
import { OnJob } from 'src/decorators';
|
||||
import { FileChecksumDto, FileChecksumResponseDto, FileReportItemDto, PathEntityType } from 'src/dtos/audit.dto';
|
||||
import {
|
||||
AssetFileType,
|
||||
AssetPathType,
|
||||
JobName,
|
||||
JobStatus,
|
||||
PersonPathType,
|
||||
QueueName,
|
||||
StorageFolder,
|
||||
UserPathType,
|
||||
} from 'src/enum';
|
||||
import { JobName, JobStatus, QueueName } from 'src/enum';
|
||||
import { BaseService } from 'src/services/base.service';
|
||||
import { getAssetFiles } from 'src/utils/asset.util';
|
||||
import { usePagination } from 'src/utils/pagination';
|
||||
|
||||
@Injectable()
|
||||
export class AuditService extends BaseService {
|
||||
@@ -26,187 +12,4 @@ export class AuditService extends BaseService {
|
||||
await this.auditRepository.removeBefore(DateTime.now().minus(AUDIT_LOG_MAX_DURATION).toJSDate());
|
||||
return JobStatus.SUCCESS;
|
||||
}
|
||||
|
||||
async getChecksums(dto: FileChecksumDto) {
|
||||
const results: FileChecksumResponseDto[] = [];
|
||||
for (const filename of dto.filenames) {
|
||||
if (!StorageCore.isImmichPath(filename)) {
|
||||
throw new BadRequestException(
|
||||
`Could not get the checksum of ${filename} because the file isn't accessible by Immich`,
|
||||
);
|
||||
}
|
||||
|
||||
const checksum = await this.cryptoRepository.hashFile(filename);
|
||||
results.push({ filename, checksum: checksum.toString('base64') });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async fixItems(items: FileReportItemDto[]) {
|
||||
for (const { entityId: id, pathType, pathValue } of items) {
|
||||
if (!StorageCore.isImmichPath(pathValue)) {
|
||||
throw new BadRequestException(
|
||||
`Could not fix item ${id} with path ${pathValue} because the file isn't accessible by Immich`,
|
||||
);
|
||||
}
|
||||
|
||||
switch (pathType) {
|
||||
case AssetPathType.ENCODED_VIDEO: {
|
||||
await this.assetRepository.update({ id, encodedVideoPath: pathValue });
|
||||
break;
|
||||
}
|
||||
|
||||
case AssetPathType.PREVIEW: {
|
||||
await this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.PREVIEW, path: pathValue });
|
||||
break;
|
||||
}
|
||||
|
||||
case AssetPathType.THUMBNAIL: {
|
||||
await this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.THUMBNAIL, path: pathValue });
|
||||
break;
|
||||
}
|
||||
|
||||
case AssetPathType.ORIGINAL: {
|
||||
await this.assetRepository.update({ id, originalPath: pathValue });
|
||||
break;
|
||||
}
|
||||
|
||||
case AssetPathType.SIDECAR: {
|
||||
await this.assetRepository.update({ id, sidecarPath: pathValue });
|
||||
break;
|
||||
}
|
||||
|
||||
case PersonPathType.FACE: {
|
||||
await this.personRepository.update({ id, thumbnailPath: pathValue });
|
||||
break;
|
||||
}
|
||||
|
||||
case UserPathType.PROFILE: {
|
||||
await this.userRepository.update(id, { profileImagePath: pathValue });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fullPath(filename: string) {
|
||||
return resolve(filename);
|
||||
}
|
||||
|
||||
async getFileReport() {
|
||||
const hasFile = (items: Set<string>, filename: string) => items.has(filename) || items.has(this.fullPath(filename));
|
||||
const crawl = async (folder: StorageFolder) =>
|
||||
new Set(
|
||||
await this.storageRepository.crawl({
|
||||
includeHidden: true,
|
||||
pathsToCrawl: [StorageCore.getBaseFolder(folder)],
|
||||
}),
|
||||
);
|
||||
|
||||
const uploadFiles = await crawl(StorageFolder.UPLOAD);
|
||||
const libraryFiles = await crawl(StorageFolder.LIBRARY);
|
||||
const thumbFiles = await crawl(StorageFolder.THUMBNAILS);
|
||||
const videoFiles = await crawl(StorageFolder.ENCODED_VIDEO);
|
||||
const profileFiles = await crawl(StorageFolder.PROFILE);
|
||||
const allFiles = new Set<string>();
|
||||
for (const list of [libraryFiles, thumbFiles, videoFiles, profileFiles, uploadFiles]) {
|
||||
for (const item of list) {
|
||||
allFiles.add(item);
|
||||
}
|
||||
}
|
||||
|
||||
const track = (filename: string | null | undefined) => {
|
||||
if (!filename) {
|
||||
return;
|
||||
}
|
||||
allFiles.delete(filename);
|
||||
allFiles.delete(this.fullPath(filename));
|
||||
};
|
||||
|
||||
this.logger.log(
|
||||
`Found ${libraryFiles.size} original files, ${thumbFiles.size} thumbnails, ${videoFiles.size} encoded videos, ${profileFiles.size} profile files`,
|
||||
);
|
||||
const pagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (options) =>
|
||||
this.assetRepository.getAll(options, { withDeleted: true, withArchived: true }),
|
||||
);
|
||||
|
||||
let assetCount = 0;
|
||||
|
||||
const orphans: FileReportItemDto[] = [];
|
||||
for await (const assets of pagination) {
|
||||
assetCount += assets.length;
|
||||
for (const { id, files, originalPath, encodedVideoPath, isExternal, checksum } of assets) {
|
||||
const { fullsizeFile, previewFile, thumbnailFile } = getAssetFiles(files);
|
||||
for (const file of [
|
||||
originalPath,
|
||||
fullsizeFile?.path,
|
||||
previewFile?.path,
|
||||
encodedVideoPath,
|
||||
thumbnailFile?.path,
|
||||
]) {
|
||||
track(file);
|
||||
}
|
||||
|
||||
const entity = { entityId: id, entityType: PathEntityType.ASSET, checksum: checksum.toString('base64') };
|
||||
if (
|
||||
originalPath &&
|
||||
!hasFile(libraryFiles, originalPath) &&
|
||||
!hasFile(uploadFiles, originalPath) &&
|
||||
// Android motion assets
|
||||
!hasFile(videoFiles, originalPath) &&
|
||||
// ignore external library assets
|
||||
!isExternal
|
||||
) {
|
||||
orphans.push({ ...entity, pathType: AssetPathType.ORIGINAL, pathValue: originalPath });
|
||||
}
|
||||
if (previewFile && !hasFile(thumbFiles, previewFile.path)) {
|
||||
orphans.push({ ...entity, pathType: AssetPathType.PREVIEW, pathValue: previewFile.path });
|
||||
}
|
||||
if (thumbnailFile && !hasFile(thumbFiles, thumbnailFile.path)) {
|
||||
orphans.push({ ...entity, pathType: AssetPathType.THUMBNAIL, pathValue: thumbnailFile.path });
|
||||
}
|
||||
if (encodedVideoPath && !hasFile(videoFiles, encodedVideoPath)) {
|
||||
orphans.push({ ...entity, pathType: AssetPathType.THUMBNAIL, pathValue: encodedVideoPath });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const users = await this.userRepository.getList();
|
||||
for (const { id, profileImagePath } of users) {
|
||||
track(profileImagePath);
|
||||
|
||||
const entity = { entityId: id, entityType: PathEntityType.USER };
|
||||
if (profileImagePath && !hasFile(profileFiles, profileImagePath)) {
|
||||
orphans.push({ ...entity, pathType: UserPathType.PROFILE, pathValue: profileImagePath });
|
||||
}
|
||||
}
|
||||
|
||||
let peopleCount = 0;
|
||||
for await (const { id, thumbnailPath } of this.personRepository.getAll()) {
|
||||
track(thumbnailPath);
|
||||
const entity = { entityId: id, entityType: PathEntityType.PERSON };
|
||||
if (thumbnailPath && !hasFile(thumbFiles, thumbnailPath)) {
|
||||
orphans.push({ ...entity, pathType: PersonPathType.FACE, pathValue: thumbnailPath });
|
||||
}
|
||||
|
||||
if (peopleCount === JOBS_ASSET_PAGINATION_SIZE) {
|
||||
this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${peopleCount} people`);
|
||||
peopleCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${peopleCount} people`);
|
||||
|
||||
const extras: string[] = [];
|
||||
for (const file of allFiles) {
|
||||
extras.push(file);
|
||||
}
|
||||
|
||||
// send as absolute paths
|
||||
for (const orphan of orphans) {
|
||||
orphan.pathValue = this.fullPath(orphan.pathValue);
|
||||
}
|
||||
|
||||
return { orphans, extras };
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user