cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): use tonemapx for software tone-mapping (#13785)

Browse Source
This commit is contained in:
Mert
2024-10-31 20:48:23 -04:00
committed by GitHub
parent 5ac236d6fd
commit b9096f3e99
16 changed files with 182 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/services/media.service.spec.ts
+85 -25
View File
@@ -9,7 +9,6 @@ import {
AudioCodec,
Colorspace,
ImageFormat,
ToneMapping,
TranscodeHWAccel,
TranscodePolicy,
VideoCodec,
@@ -410,7 +409,7 @@ describe(MediaService.name, () => {
'-frames:v 1',
'-update 1',
'-v verbose',
String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_color_matrix=bt601:out_range=pc,format=yuv420p`,
String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc`,
],
twoPass: false,
}),
@@ -445,7 +444,7 @@ describe(MediaService.name, () => {
'-frames:v 1',
'-update 1',
'-v verbose',
String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=601:m=470bg:range=pc,format=yuv420p`,
String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
],
twoPass: false,
}),
@@ -482,7 +481,7 @@ describe(MediaService.name, () => {
'-frames:v 1',
'-update 1',
'-v verbose',
String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=601:m=470bg:range=pc,format=yuv420p`,
String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
],
twoPass: false,
}),
@@ -1328,7 +1327,7 @@ describe(MediaService.name, () => {
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-vf scale=-2:720',
'-preset 12',
'-crf 23',
]),
@@ -1454,7 +1453,7 @@ describe(MediaService.name, () => {
'-map 0:1',
'-g 256',
'-v verbose',
'-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
'-vf hwupload_cuda,scale_cuda=-2:720:format=nv12',
'-preset p1',
'-cq:v 23',
]),
@@ -1586,7 +1585,7 @@ describe(MediaService.name, () => {
inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']),
outputOptions: expect.arrayContaining([
expect.stringContaining(
'tonemap_cuda=desat=0:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709:format=nv12',
'tonemap_cuda=desat=0:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:tonemap_mode=lum:transfer=bt709:peak=100:format=nv12',
),
]),
twoPass: false,
@@ -1594,6 +1593,24 @@ describe(MediaService.name, () => {
);
});
it('should set format to nv12 for nvenc if input is not yuv420p', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit);
systemMock.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHWAccel.NVENC, accelDecode: true },
});
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
expect.objectContaining({
inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']),
outputOptions: expect.arrayContaining([expect.stringContaining('format=nv12')]),
twoPass: false,
}),
);
});
it('should set options for qsv', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
@@ -1616,7 +1633,7 @@ describe(MediaService.name, () => {
'-refs 5',
'-g 256',
'-v verbose',
'-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq',
'-vf hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq:format=nv12',
'-preset 7',
'-global_quality:v 23',
'-maxrate 10000k',
@@ -1748,7 +1765,7 @@ describe(MediaService.name, () => {
]),
outputOptions: expect.arrayContaining([
expect.stringContaining(
'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=qsv:reverse=1,format=qsv',
'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:transfer=bt709:range=pc:tonemap=hable:tonemap_mode=lum:peak=100,hwmap=derive_device=qsv:reverse=1,format=qsv',
),
]),
twoPass: false,
@@ -1776,6 +1793,32 @@ describe(MediaService.name, () => {
);
});
it('should set format to nv12 for qsv if input is not yuv420p', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit);
systemMock.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHWAccel.QSV, accelDecode: true },
});
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
expect.objectContaining({
inputOptions: expect.arrayContaining([
'-hwaccel qsv',
'-hwaccel_output_format qsv',
'-async_depth 4',
'-threads 1',
]),
outputOptions: expect.arrayContaining([expect.stringContaining('format=nv12')]),
twoPass: false,
}),
);
});
it('should set options for vaapi', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
@@ -1799,7 +1842,7 @@ describe(MediaService.name, () => {
'-map 0:1',
'-g 256',
'-v verbose',
'-vf format=nv12,hwupload,scale_vaapi=-2:720:mode=hq:out_range=pc',
'-vf hwupload=extra_hw_frames=64,scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12',
'-compression_level 7',
'-rc_mode 1',
]),
@@ -1970,7 +2013,7 @@ describe(MediaService.name, () => {
);
});
it('should use hardware tone-mapping for qsv if hardware decoding is enabled and should tone map', async () => {
it('should use hardware tone-mapping for vaapi if hardware decoding is enabled and should tone map', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
systemMock.get.mockResolvedValue({
@@ -1987,7 +2030,7 @@ describe(MediaService.name, () => {
inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']),
outputOptions: expect.arrayContaining([
expect.stringContaining(
'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=vaapi:reverse=1,format=vaapi',
'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:transfer=bt709:range=pc:tonemap=hable:tonemap_mode=lum:peak=100,hwmap=derive_device=vaapi:reverse=1,format=vaapi',
),
]),
twoPass: false,
@@ -1995,6 +2038,27 @@ describe(MediaService.name, () => {
);
});
it('should set format to nv12 for vaapi if input is not yuv420p', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit);
systemMock.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true },
});
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
expect.objectContaining({
inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']),
outputOptions: expect.arrayContaining([expect.stringContaining('format=nv12')]),
twoPass: false,
}),
);
});
it('should use preferred device for vaapi when hardware decoding', async () => {
storageMock.readdir.mockResolvedValue(['renderD128', 'renderD129', 'renderD130']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
@@ -2140,7 +2204,7 @@ describe(MediaService.name, () => {
inputOptions: expect.arrayContaining(['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']),
outputOptions: expect.arrayContaining([
expect.stringContaining(
'scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime',
'scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0:tonemap_mode=lum:peak=100,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime',
),
]),
twoPass: false,
@@ -2164,7 +2228,7 @@ describe(MediaService.name, () => {
inputOptions: [],
outputOptions: expect.arrayContaining([
expect.stringContaining(
'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',
),
]),
twoPass: false,
@@ -2188,7 +2252,7 @@ describe(MediaService.name, () => {
inputOptions: [],
outputOptions: expect.arrayContaining([
expect.stringContaining(
'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',
),
]),
twoPass: false,
@@ -2209,7 +2273,7 @@ describe(MediaService.name, () => {
outputOptions: expect.arrayContaining([
'-c:v h264',
'-c:a copy',
'-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
'-vf tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',
]),
twoPass: false,
}),
@@ -2229,16 +2293,16 @@ describe(MediaService.name, () => {
outputOptions: expect.arrayContaining([
'-c:v h264',
'-c:a copy',
'-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
'-vf tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',
]),
twoPass: false,
}),
);
});
it('should set npl to 250 for reinhard and mobius tone-mapping algorithms', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
systemMock.get.mockResolvedValue({ ffmpeg: { tonemap: ToneMapping.MOBIUS } });
it('should transcode when policy is required and video is not yuv420p', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit);
systemMock.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.REQUIRED } });
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
@@ -2246,11 +2310,7 @@ describe(MediaService.name, () => {
'upload/encoded-video/user-id/as/se/asset-id.mp4',
expect.objectContaining({
inputOptions: expect.any(Array),
outputOptions: expect.arrayContaining([
'-c:v h264',
'-c:a copy',
'-vf zscale=t=linear:npl=250,tonemap=mobius:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
]),
outputOptions: expect.arrayContaining(['-c:v h264', '-c:a copy', '-vf format=yuv420p']),
twoPass: false,
}),
);
server/src/services/media.service.ts
+1 -1
View File
@@ -413,7 +413,7 @@ export class MediaService extends BaseService {
const isLargerThanTargetBitrate = stream.bitrate > this.parseBitrateToBps(ffmpegConfig.maxBitrate);
const isTargetVideoCodec = ffmpegConfig.acceptedVideoCodecs.includes(stream.codecName as VideoCodec);
const isRequired = !isTargetVideoCodec || stream.isHDR;
const isRequired = !isTargetVideoCodec || !stream.pixelFormat.endsWith('420p');
switch (ffmpegConfig.transcode) {
case TranscodePolicy.DISABLED: {
server/src/services/system-config.service.spec.ts
-1
View File
@@ -65,7 +65,6 @@ const updatedConfig = Object.freeze<SystemConfig>({
bframes: -1,
refs: 0,
gopSize: 0,
npl: 0,
temporalAQ: false,
cqMode: CQMode.AUTO,
twoPass: false,