toggle for hardware decoding, software / hardware decoding for nvenc and rkmpp

server/src/config.ts

@@ -97,6 +97,7 @@ export interface SystemConfig {
     preferredHwDevice: string;
     transcode: TranscodePolicy;
     accel: TranscodeHWAccel;
+    accelDecode: false;
     tonemap: ToneMapping;
   };
   job: Record<ConcurrentQueueName, { concurrency: number }>;
@@ -224,6 +225,7 @@ export const defaults = Object.freeze<SystemConfig>({
     transcode: TranscodePolicy.REQUIRED,
     tonemap: ToneMapping.HABLE,
     accel: TranscodeHWAccel.DISABLED,
+    accelDecode: false,
   },
   job: {
     [QueueName.BACKGROUND_TASK]: { concurrency: 5 },

server/src/dtos/system-config.dto.ts

@@ -132,6 +132,9 @@ export class SystemConfigFFmpegDto {
   @ApiProperty({ enumName: 'TranscodeHWAccel', enum: TranscodeHWAccel })
   accel!: TranscodeHWAccel;
 
+  @ValidateBoolean()
+  accelDecode!: boolean;
+
   @IsEnum(ToneMapping)
   @ApiProperty({ enumName: 'ToneMapping', enum: ToneMapping })
   tonemap!: ToneMapping;

server/src/entities/system-config.entity.ts

@@ -43,6 +43,7 @@ export const SystemConfigKey = {
   FFMPEG_PREFERRED_HW_DEVICE: 'ffmpeg.preferredHwDevice',
   FFMPEG_TRANSCODE: 'ffmpeg.transcode',
   FFMPEG_ACCEL: 'ffmpeg.accel',
+  FFMPEG_ACCEL_DECODE: 'ffmpeg.accelDecode',
   FFMPEG_TONEMAP: 'ffmpeg.tonemap',
 
   JOB_THUMBNAIL_GENERATION_CONCURRENCY: 'job.thumbnailGeneration.concurrency',

server/src/services/media.service.spec.ts

@@ -1381,6 +1381,52 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should use hardware decoding for nvenc if enabled', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
+        { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']),
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining(
+              'hwupload=derive_device=vulkan,scale_vulkan=w=1280:h=720,libplacebo=color_primaries=bt709:color_trc=bt709:colorspace=bt709:deband=true:deband_iterations=3:deband_radius=8:deband_threshold=6:downscaler=none:format=yuv420p:tonemapping=clip:upscaler=none,hwupload=derive_device=cuda',
+            ),
+          ]),
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should use hardware tone-mapping for nvenc if hardware decoding is enabled and should tone map', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC },
+        { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']),
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining(
+              'hwupload=derive_device=vulkan,libplacebo=color_primaries=bt709:color_trc=bt709:colorspace=bt709:deband=true:deband_iterations=3:deband_radius=8:deband_threshold=6:downscaler=none:format=yuv420p:tonemapping=hable:upscaler=none,hwupload=derive_device=cuda',
+            ),
+          ]),
+          twoPass: false,
+        },
+      );
+    });
+
     it('should set options for qsv', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128']);
       mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
@@ -1694,7 +1740,10 @@ describe(MediaService.name, () => {
     it('should set options for rkmpp', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128']);
       mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
-      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }]);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true },
+      ]);
       assetMock.getByIds.mockResolvedValue([assetStub.video]);
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
@@ -1726,6 +1775,7 @@ describe(MediaService.name, () => {
       mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
       configMock.load.mockResolvedValue([
         { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true },
         { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
         { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
       ]);
@@ -1747,6 +1797,7 @@ describe(MediaService.name, () => {
       mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
       configMock.load.mockResolvedValue([
         { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true },
         { key: SystemConfigKey.FFMPEG_CRF, value: 30 },
         { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' },
       ]);
@@ -1769,6 +1820,7 @@ describe(MediaService.name, () => {
       mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
       configMock.load.mockResolvedValue([
         { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: true },
         { key: SystemConfigKey.FFMPEG_CRF, value: 30 },
         { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' },
       ]);
@@ -1788,6 +1840,33 @@ describe(MediaService.name, () => {
         },
       );
     });
+
+    it('should use software decoding and tone-mapping if hardware decoding is disabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => true, isCharacterDevice: () => true });
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_ACCEL_DECODE, value: false },
+        { key: SystemConfigKey.FFMPEG_CRF, value: 30 },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining(
+              'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p',
+            ),
+          ]),
+          twoPass: false,
+        },
+      );
+    });
   });
 
   it('should tonemap when policy is required and video is hdr', async () => {

server/src/services/system-config.service.spec.ts

@@ -65,6 +65,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     preferredHwDevice: 'auto',
     transcode: TranscodePolicy.REQUIRED,
     accel: TranscodeHWAccel.DISABLED,
+    accelDecode: false,
     tonemap: ToneMapping.HABLE,
   },
   logging: {

server/src/utils/media.ts

@@ -26,14 +26,18 @@ class BaseConfig implements VideoCodecSWConfig {
       }
     }
 
-    options.outputOptions.push(...this.getPresetOptions(), ...this.getThreadOptions(), ...this.getBitrateOptions());
+    options.outputOptions.push(
+      ...this.getPresetOptions(),
+      ...this.getOutputThreadOptions(),
+      ...this.getBitrateOptions(),
+    );
 
     return options;
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   getBaseInputOptions(videoStream: VideoStreamInfo): string[] {
-    return [];
+    return this.getInputThreadOptions();
   }
 
   getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
@@ -80,11 +84,7 @@ class BaseConfig implements VideoCodecSWConfig {
       options.push(`scale=${this.getScaling(videoStream)}`);
     }
 
-    if (this.shouldToneMap(videoStream)) {
+    options.push(...this.getToneMapping(videoStream), 'format=yuv420p');
-      options.push(...this.getToneMapping());
-    }
-    options.push('format=yuv420p');
-
     return options;
   }
 
@@ -112,7 +112,11 @@ class BaseConfig implements VideoCodecSWConfig {
     }
   }
 
-  getThreadOptions(): Array<string> {
+  getInputThreadOptions(): Array<string> {
+    return [];
+  }
+
+  getOutputThreadOptions(): Array<string> {
     if (this.config.threads <= 0) {
       return [];
     }
@@ -218,7 +222,11 @@ class BaseConfig implements VideoCodecSWConfig {
     }
   }
 
-  getToneMapping() {
+  getToneMapping(videoStream: VideoStreamInfo) {
+    if (!this.shouldToneMap(videoStream)) {
+      return [];
+    }
+
     const colors = this.getColors();
 
     return [
@@ -348,8 +356,8 @@ export class ThumbnailConfig extends BaseConfig {
 }
 
 export class H264Config extends BaseConfig {
-  getThreadOptions() {
+  getOutputThreadOptions() {
-    const options = super.getThreadOptions();
+    const options = super.getOutputThreadOptions();
     if (this.config.threads === 1) {
       options.push('-x264-params frame-threads=1:pools=none');
     }
@@ -359,8 +367,8 @@ export class H264Config extends BaseConfig {
 }
 
 export class HEVCConfig extends BaseConfig {
-  getThreadOptions() {
+  getOutputThreadOptions() {
-    const options = super.getThreadOptions();
+    const options = super.getOutputThreadOptions();
     if (this.config.threads === 1) {
       options.push('-x265-params frame-threads=1:pools=none');
     }
@@ -391,8 +399,8 @@ export class VP9Config extends BaseConfig {
     return [`-${this.useCQP() ? 'q:v' : 'crf'} ${this.config.crf}`, `-b:v ${bitrates.max}${bitrates.unit}`];
   }
 
-  getThreadOptions() {
+  getOutputThreadOptions() {
-    return ['-row-mt 1', ...super.getThreadOptions()];
+    return ['-row-mt 1', ...super.getOutputThreadOptions()];
   }
 
   eligibleForTwoPass() {
@@ -425,7 +433,7 @@ export class AV1Config extends BaseConfig {
     return options;
   }
 
-  getThreadOptions() {
+  getOutputThreadOptions() {
     return []; // Already set above with svtav1-params
   }
 
@@ -440,7 +448,11 @@ export class NVENCConfig extends BaseHWConfig {
   }
 
   getBaseInputOptions() {
-    return ['-hwaccel cuda', '-hwaccel_output_format cuda', ...this.getThreadOptions()];
+    if (!this.config.accelDecode) {
+      return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda', ...this.getInputThreadOptions()];
+    }
+
+    return ['-hwaccel cuda', '-hwaccel_output_format cuda', ...this.getInputThreadOptions()];
   }
 
   getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
@@ -462,12 +474,30 @@ export class NVENCConfig extends BaseHWConfig {
   }
 
   getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = ['hwupload=derive_device=vulkan'];
+    const options = [];
+    if (!this.config.accelDecode) {
+      options.push('format=nv12', 'hwupload_cuda');
+      if (this.shouldScale(videoStream)) {
+        options.push(`scale_cuda=${this.getScaling(videoStream)}`);
+      }
+      return options;
+    }
+
+    options.push('hwupload=derive_device=vulkan');
     if (this.shouldScale(videoStream)) {
       const { width, height } = this.getSize(videoStream);
       options.push(`scale_vulkan=w=${width}:h=${height}`);
     }
 
+    options.push(...this.getToneMapping(videoStream), 'hwupload=derive_device=cuda');
+    return options;
+  }
+
+  getToneMapping(videoStream: VideoStreamInfo) {
+    if (!this.config.accelDecode) {
+      return super.getToneMapping(videoStream);
+    }
+
     const colors = this.getColors();
     const libplaceboOptions = [
       `color_primaries=${colors.primaries}`,
@@ -483,9 +513,7 @@ export class NVENCConfig extends BaseHWConfig {
       'upscaler=none',
     ];
 
-    const libplacebo = `libplacebo=${libplaceboOptions.join(':')}`;
+    return [`libplacebo=${libplaceboOptions.join(':')}`];
-    options.push(libplacebo, 'hwupload=derive_device=cuda');
-    return options;
   }
 
   getPresetOptions() {
@@ -517,10 +545,14 @@ export class NVENCConfig extends BaseHWConfig {
     }
   }
 
-  getThreadOptions() {
+  getInputThreadOptions() {
     return [`-threads ${this.config.threads <= 0 ? 1 : this.config.threads}`];
   }
 
+  getOutputThreadOptions() {
+    return [];
+  }
+
   getRefs() {
     const bframes = this.getBFrames();
     if (bframes > 0 && bframes < 3 && this.config.refs < 3) {
@@ -555,7 +587,7 @@ export class QSVConfig extends BaseHWConfig {
   }
 
   getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+    const options = this.getToneMapping(videoStream);
     options.push('format=nv12', 'hwupload=extra_hw_frames=64');
     if (this.shouldScale(videoStream)) {
       options.push(`scale_qsv=${this.getScaling(videoStream)}`);
@@ -621,7 +653,7 @@ export class VAAPIConfig extends BaseHWConfig {
   }
 
   getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+    const options = this.getToneMapping(videoStream);
     options.push('format=nv12', 'hwupload');
     if (this.shouldScale(videoStream)) {
       options.push(`scale_vaapi=${this.getScaling(videoStream)}`);
@@ -693,12 +725,16 @@ export class RKMPPConfig extends BaseHWConfig {
     if (this.devices.length === 0) {
       throw new Error('No RKMPP device found');
     }
-    return this.shouldToneMap(videoStream) && !this.hasOpenCL
+    return !this.config.accelDecode || (this.shouldToneMap(videoStream) && !this.hasOpenCL)
       ? [] // disable hardware decoding & filters
       : ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga'];
   }
 
   getFilterOptions(videoStream: VideoStreamInfo) {
+    if (!this.config.accelDecode) {
+      return super.getFilterOptions(videoStream);
+    }
+
     if (this.shouldToneMap(videoStream)) {
       if (!this.hasOpenCL) {
         return super.getFilterOptions(videoStream);