cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: original-sized previews for non-web-friendly images (#14446)

Browse Source 
* feat(server): extract full-size previews from RAW images

* feat(web): load fullsize preview for RAW images when zoomed in

* refactor: tweaks for code review

* refactor: rename "converted" preview/assets to "fullsize"

* feat(web/server): fullsize preview for non-web-friendly images

* feat: tweaks for code review

* feat(server): require ASSET_DOWNLOAD premission for fullsize previews

* test: fix types and interfaces

* chore: gen open-api

* feat(server): keep only essential exif in fullsize preview

* chore: regen openapi

* test: revert unnecessary timeout

* feat: move full-size preview config to standalone entry

* feat(i18n): update en texts

* fix: don't return fullsizePath when disabled

* test: full-size previews

* test(web): full-size previews

* chore: make open-api

* feat(server): redirect to preview/original URL when fullsize thumbnail not available

* fix(server): delete fullsize preview image on thumbnail regen after fullsize preview turned off

* refactor(server): AssetRepository.deleteFiles with Kysely

* fix(server): type of MediaRepository.writeExif

* minor simplification

* minor styling changes and condensed wording

* simplify

* chore: reuild open-api

* test(server): fix media.service tests

* test(web): fix photo-viewer test

* fix(server):  use fullsize image when requested

* fix file path extension

* formatting

* use fullsize when zooming back out or when "display original photos" is enabled

* simplify condition

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
This commit is contained in:
Eli Gao
2025-04-01 01:24:28 +08:00
committed by GitHub
parent a5093a9434
commit 5c80e8734b
33 changed files with 778 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
server/src/services/asset-media.service.ts
View File

@@ -37,6 +37,10 @@ interface UploadRequest {
file: UploadFile;
}
export interface AssetMediaRedirectResponse {
targetSize: AssetMediaSize | 'original';
}
@Injectable()
export class AssetMediaService extends BaseService {
async getUploadAssetIdByChecksum(auth: AuthDto, checksum?: string): Promise<AssetMediaResponseDto | undefined> {
@@ -194,16 +198,31 @@ export class AssetMediaService extends BaseService {
});
}
async viewThumbnail(auth: AuthDto, id: string, dto: AssetMediaOptionsDto): Promise<ImmichFileResponse> {
async viewThumbnail(
auth: AuthDto,
id: string,
dto: AssetMediaOptionsDto,
): Promise<ImmichFileResponse | AssetMediaRedirectResponse> {
await this.requireAccess({ auth, permission: Permission.ASSET_VIEW, ids: [id] });
const asset = await this.findOrFail(id);
const size = dto.size ?? AssetMediaSize.THUMBNAIL;
const { thumbnailFile, previewFile } = getAssetFiles(asset.files);
const { thumbnailFile, previewFile, fullsizeFile } = getAssetFiles(asset.files);
let filepath = previewFile?.path;
if (size === AssetMediaSize.THUMBNAIL && thumbnailFile) {
filepath = thumbnailFile.path;
} else if (size === AssetMediaSize.FULLSIZE) {
if (mimeTypes.isWebSupportedImage(asset.originalPath)) {
// use original file for web supported images
return { targetSize: 'original' };
}
if (!fullsizeFile) {
// downgrade to preview if fullsize is not available.
// e.g. disabled or not yet (re)generated
return { targetSize: AssetMediaSize.PREVIEW };
}
filepath = fullsizeFile.path;
}
if (!filepath) {

179
server/src/services/media.service.spec.ts
View File

@@ -1,5 +1,6 @@
import { OutputInfo } from 'sharp';
import { SystemConfig } from 'src/config';
import { AssetMediaSize } from 'src/dtos/asset-media.dto';
import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import {
@@ -233,18 +234,26 @@ describe(MediaService.name, () => {
});
await expect(sut.handleAssetMigration({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);
expect(mocks.move.create).toHaveBeenCalledTimes(2);
expect(mocks.move.create).toHaveBeenCalledTimes(3);
});
});
describe('handleGenerateThumbnails', () => {
let rawBuffer: Buffer;
let fullsizeBuffer: Buffer;
let rawInfo: RawImageInfo;
beforeEach(() => {
fullsizeBuffer = Buffer.from('embedded image data');
rawBuffer = Buffer.from('image data');
rawInfo = { width: 100, height: 100, channels: 3 };
mocks.media.decodeImage.mockResolvedValue({ data: rawBuffer, info: rawInfo as OutputInfo });
mocks.media.decodeImage.mockImplementation((path) =>
Promise.resolve(
path.includes(AssetMediaSize.FULLSIZE)
? { data: fullsizeBuffer, info: rawInfo as OutputInfo }
: { data: rawBuffer, info: rawInfo as OutputInfo },
),
);
});
it('should skip thumbnail generation if asset not found', async () => {
@@ -591,15 +600,13 @@ describe(MediaService.name, () => {
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
const extractedPath = mocks.media.extract.mock.calls.at(-1)?.[1].toString();
const convertedPath = mocks.media.extract.mock.lastCall?.[1].toString();
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
expect(mocks.media.decodeImage).toHaveBeenCalledWith(extractedPath, {
expect(mocks.media.decodeImage).toHaveBeenCalledWith(convertedPath, {
colorspace: Colorspace.P3,
processInvalidImages: false,
size: 1440,
});
expect(extractedPath?.endsWith('.tmp')).toBe(true);
expect(mocks.storage.unlink).toHaveBeenCalledWith(extractedPath);
});
it('should resize original image if embedded image is too small', async () => {
@@ -610,15 +617,14 @@ describe(MediaService.name, () => {
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
const extractedPath = mocks.media.extract.mock.lastCall?.[1].toString();
expect(extractedPath).toMatch(/-fullsize\.jpeg$/);
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
colorspace: Colorspace.P3,
processInvalidImages: false,
size: 1440,
});
const extractedPath = mocks.media.extract.mock.calls.at(-1)?.[1].toString();
expect(extractedPath?.endsWith('.tmp')).toBe(true);
expect(mocks.storage.unlink).toHaveBeenCalledWith(extractedPath);
});
it('should resize original image if embedded image not found', async () => {
@@ -686,6 +692,159 @@ describe(MediaService.name, () => {
expect(mocks.media.getImageDimensions).not.toHaveBeenCalled();
vi.unstubAllEnvs();
});
it('should generate full-size preview using embedded JPEG from RAW images when extractEmbedded is true', async () => {
mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true }, extractEmbedded: true } });
mocks.media.extract.mockResolvedValue(true);
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
mocks.asset.getById.mockResolvedValue(assetStub.imageDng);
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
const extractedPath = mocks.media.extract.mock.lastCall?.[1].toString();
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
expect(mocks.media.decodeImage).toHaveBeenCalledWith(extractedPath, {
colorspace: Colorspace.P3,
processInvalidImages: false,
});
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(2);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
fullsizeBuffer,
{
colorspace: Colorspace.P3,
format: ImageFormat.JPEG,
size: 1440,
quality: 80,
processInvalidImages: false,
raw: rawInfo,
},
'upload/thumbs/user-id/as/se/asset-id-preview.jpeg',
);
});
it('should generate full-size preview directly from RAW images when extractEmbedded is false', async () => {
mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true }, extractEmbedded: false } });
mocks.media.extract.mockResolvedValue(true);
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
mocks.asset.getById.mockResolvedValue(assetStub.imageDng);
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
colorspace: Colorspace.P3,
processInvalidImages: false,
});
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
{
colorspace: Colorspace.P3,
format: ImageFormat.JPEG,
quality: 80,
processInvalidImages: false,
raw: rawInfo,
},
'upload/thumbs/user-id/as/se/asset-id-fullsize.jpeg',
);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
{
colorspace: Colorspace.P3,
format: ImageFormat.JPEG,
quality: 80,
size: 1440,
processInvalidImages: false,
raw: rawInfo,
},
'upload/thumbs/user-id/as/se/asset-id-preview.jpeg',
);
});
it('should generate full-size preview from non-web-friendly images', async () => {
mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });
mocks.media.extract.mockResolvedValue(true);
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
// HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari.
mocks.asset.getById.mockResolvedValue(assetStub.imageHif);
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.imageHif.originalPath, {
colorspace: Colorspace.P3,
processInvalidImages: false,
});
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
{
colorspace: Colorspace.P3,
format: ImageFormat.JPEG,
quality: 80,
processInvalidImages: false,
raw: rawInfo,
},
'upload/thumbs/user-id/as/se/asset-id-fullsize.jpeg',
);
});
it('should skip generating full-size preview for web-friendly images', async () => {
mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });
mocks.media.extract.mockResolvedValue(true);
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
mocks.asset.getById.mockResolvedValue(assetStub.image);
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.image.originalPath, {
colorspace: Colorspace.SRGB,
processInvalidImages: false,
size: 1440,
});
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(2);
expect(mocks.media.generateThumbnail).not.toHaveBeenCalledWith(
expect.anything(),
expect.anything(),
'upload/thumbs/user-id/as/se/asset-id-fullsize.jpeg',
);
});
it('should respect encoding options when generating full-size preview', async () => {
mocks.systemMetadata.get.mockResolvedValue({
image: { fullsize: { enabled: true, format: ImageFormat.WEBP, quality: 90 } },
});
mocks.media.extract.mockResolvedValue(true);
mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
// HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari.
mocks.asset.getById.mockResolvedValue(assetStub.imageHif);
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
expect(mocks.media.decodeImage).toHaveBeenCalledOnce();
expect(mocks.media.decodeImage).toHaveBeenCalledWith(assetStub.imageHif.originalPath, {
colorspace: Colorspace.P3,
processInvalidImages: false,
});
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
{
colorspace: Colorspace.P3,
format: ImageFormat.WEBP,
quality: 90,
processInvalidImages: false,
raw: rawInfo,
},
'upload/thumbs/user-id/as/se/asset-id-fullsize.webp',
);
});
});
describe('handleQueueVideoConversion', () => {

118
server/src/services/media.service.ts
View File

@@ -1,5 +1,4 @@
import { Injectable } from '@nestjs/common';
import { dirname } from 'node:path';
import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core';
import { OnEvent, OnJob } from 'src/decorators';
@@ -11,6 +10,7 @@ import {
AssetType,
AudioCodec,
Colorspace,
ImageFormat,
JobName,
JobStatus,
LogLevel,
@@ -24,7 +24,16 @@ import {
} from 'src/enum';
import { UpsertFileOptions, WithoutProperty } from 'src/repositories/asset.repository';
import { BaseService } from 'src/services/base.service';
import { AudioStreamInfo, JobItem, JobOf, VideoFormat, VideoInterfaces, VideoStreamInfo } from 'src/types';
import {
AudioStreamInfo,
DecodeToBufferOptions,
GenerateThumbnailOptions,
JobItem,
JobOf,
VideoFormat,
VideoInterfaces,
VideoStreamInfo,
} from 'src/types';
import { getAssetFiles } from 'src/utils/asset.util';
import { BaseConfig, ThumbnailConfig } from 'src/utils/media';
import { mimeTypes } from 'src/utils/mime-types';
@@ -131,6 +140,7 @@ export class MediaService extends BaseService {
return JobStatus.FAILED;
}
await this.storageCore.moveAssetImage(asset, AssetPathType.FULLSIZE, ImageFormat.JPEG);
await this.storageCore.moveAssetImage(asset, AssetPathType.PREVIEW, image.preview.format);
await this.storageCore.moveAssetImage(asset, AssetPathType.THUMBNAIL, image.thumbnail.format);
await this.storageCore.moveAssetVideo(asset);
@@ -151,7 +161,12 @@ export class MediaService extends BaseService {
return JobStatus.SKIPPED;
}
let generated: { previewPath: string; thumbnailPath: string; thumbhash: Buffer };
let generated: {
previewPath: string;
thumbnailPath: string;
fullsizePath?: string;
thumbhash: Buffer;
};
if (asset.type === AssetType.VIDEO || asset.originalFileName.toLowerCase().endsWith('.gif')) {
generated = await this.generateVideoThumbnails(asset);
} else if (asset.type === AssetType.IMAGE) {
@@ -161,7 +176,7 @@ export class MediaService extends BaseService {
return JobStatus.SKIPPED;
}
const { previewFile, thumbnailFile } = getAssetFiles(asset.files);
const { previewFile, thumbnailFile, fullsizeFile } = getAssetFiles(asset.files);
const toUpsert: UpsertFileOptions[] = [];
if (previewFile?.path !== generated.previewPath) {
toUpsert.push({ assetId: asset.id, path: generated.previewPath, type: AssetFileType.PREVIEW });
@@ -171,11 +186,15 @@ export class MediaService extends BaseService {
toUpsert.push({ assetId: asset.id, path: generated.thumbnailPath, type: AssetFileType.THUMBNAIL });
}
if (generated.fullsizePath && fullsizeFile?.path !== generated.fullsizePath) {
toUpsert.push({ assetId: asset.id, path: generated.fullsizePath, type: AssetFileType.FULLSIZE });
}
if (toUpsert.length > 0) {
await this.assetRepository.upsertFiles(toUpsert);
}
const pathsToDelete = [];
const pathsToDelete: string[] = [];
if (previewFile && previewFile.path !== generated.previewPath) {
this.logger.debug(`Deleting old preview for asset ${asset.id}`);
pathsToDelete.push(previewFile.path);
@@ -186,6 +205,15 @@ export class MediaService extends BaseService {
pathsToDelete.push(thumbnailFile.path);
}
if (fullsizeFile && fullsizeFile.path !== generated.fullsizePath) {
this.logger.debug(`Deleting old fullsize preview image for asset ${asset.id}`);
pathsToDelete.push(fullsizeFile.path);
if (!generated.fullsizePath) {
// did not generate a new fullsize image, delete the existing record
await this.assetRepository.deleteFiles([fullsizeFile]);
}
}
if (pathsToDelete.length > 0) {
await Promise.all(pathsToDelete.map((path) => this.storageRepository.unlink(path)));
}
@@ -205,33 +233,71 @@ export class MediaService extends BaseService {
const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format);
this.storageCore.ensureFolders(previewPath);
const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath);
const extractedPath = StorageCore.getTempPathInDir(dirname(previewPath));
const didExtract = shouldExtract && (await this.mediaRepository.extract(asset.originalPath, extractedPath));
const processInvalidImages = process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true';
const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
try {
const useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.preview.size));
const inputPath = useExtracted ? extractedPath : asset.originalPath;
const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
const processInvalidImages = process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true';
// prevents this extra "enabled" from leaking into fullsizeOptions later
const { enabled: imageFullsizeEnabled, ...imageFullsizeConfig } = image.fullsize;
const orientation = useExtracted && asset.exifInfo?.orientation ? Number(asset.exifInfo.orientation) : undefined;
const decodeOptions = { colorspace, processInvalidImages, size: image.preview.size, orientation };
const { data, info } = await this.mediaRepository.decodeImage(inputPath, decodeOptions);
const shouldConvertFullsize = imageFullsizeEnabled && !mimeTypes.isWebSupportedImage(asset.originalFileName);
const shouldExtractEmbedded = image.extractEmbedded && mimeTypes.isRaw(asset.originalFileName);
const decodeOptions: DecodeToBufferOptions = { colorspace, processInvalidImages, size: image.preview.size };
const options = { colorspace, processInvalidImages, raw: info };
const outputs = await Promise.all([
this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...options }, thumbnailPath),
this.mediaRepository.generateThumbnail(data, { ...image.preview, ...options }, previewPath),
this.mediaRepository.generateThumbhash(data, options),
]);
let useExtracted = false;
let decodeInputPath: string = asset.originalPath;
// Converted or extracted image from non-web-supported formats (e.g. RAW)
let fullsizePath: string | undefined;
return { previewPath, thumbnailPath, thumbhash: outputs[2] };
} finally {
if (didExtract) {
await this.storageRepository.unlink(extractedPath);
if (shouldConvertFullsize) {
// unset size to decode fullsize image
decodeOptions.size = undefined;
fullsizePath = StorageCore.getImagePath(asset, AssetPathType.FULLSIZE, image.fullsize.format);
}
if (shouldExtractEmbedded) {
// For RAW files, try extracting embedded preview first
// Assume extracted image from RAW always in JPEG format, as implied from the `jpgFromRaw` tag name
const extractedPath = StorageCore.getImagePath(asset, AssetPathType.FULLSIZE, ImageFormat.JPEG);
const didExtract = await this.mediaRepository.extract(asset.originalPath, extractedPath);
useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.preview.size));
if (useExtracted) {
if (shouldConvertFullsize) {
// skip re-encoding and directly use extracted as fullsize preview
// as usually the extracted image is already heavily compressed, no point doing lossy conversion again
fullsizePath = extractedPath;
}
// use this as origin of preview and thumbnail
decodeInputPath = extractedPath;
if (asset.exifInfo) {
// write essential orientation and colorspace EXIF for correct fullsize preview and subsequent processing
const exif = { orientation: asset.exifInfo.orientation, colorspace: asset.exifInfo.colorspace };
await this.mediaRepository.writeExif(exif, extractedPath);
}
}
}
const { info, data } = await this.mediaRepository.decodeImage(decodeInputPath, decodeOptions);
const thumbnailOptions = { colorspace, processInvalidImages, raw: info };
const promises = [
this.mediaRepository.generateThumbhash(data, thumbnailOptions),
this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...thumbnailOptions }, thumbnailPath),
this.mediaRepository.generateThumbnail(data, { ...image.preview, ...thumbnailOptions }, previewPath),
];
// did not extract a usable image from RAW
if (fullsizePath && !useExtracted) {
const fullsizeOptions: GenerateThumbnailOptions = {
...imageFullsizeConfig,
...thumbnailOptions,
size: undefined,
};
promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizePath));
}
const outputs = await Promise.all(promises);
return { previewPath, thumbnailPath, fullsizePath, thumbhash: outputs[0] as Buffer };
}
private async generateVideoThumbnails(asset: AssetEntity) {

1
server/src/services/system-config.service.spec.ts
View File

@@ -146,6 +146,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
format: ImageFormat.JPEG,
quality: 80,
},
fullsize: { enabled: false, format: ImageFormat.JPEG, quality: 80 },
colorspace: Colorspace.P3,
extractEmbedded: false,
},