fix(streaming): convert to proper pixel format before cuda upload + scale

server/src/ffmpeg/FfmpegStreamFactory.ts

@@ -579,24 +579,25 @@ export class FfmpegStreamFactory extends IFFMPEG {
 
     const frameSize = FrameSize.fromResolution(this.transcodeConfig.resolution);
 
+    let scaledSize: Maybe<FrameSize>;
     let errorInput: VideoInputSource;
     switch (this.transcodeConfig.errorScreen) {
-      case 'pic':
+      case 'pic': {
+        const stream = StillImageStream.create({
+          index: 0,
+          frameSize: FrameSize.create({ width: 1920, height: 1080 }),
+        });
+        scaledSize = stream.squarePixelFrameSize(
+          FrameSize.fromResolution(this.transcodeConfig.resolution),
+        );
         errorInput = VideoInputSource.withStream(
           new HttpStreamSource(
             makeLocalUrl('/images/generic-error-screen.png'),
           ),
-          VideoStream.create({
-            inputKind: 'stillimage',
-            codec: 'unknown',
-            frameSize: FrameSize.create({ width: 1920, height: 1080 }),
-            index: 0,
-            sampleAspectRatio: '1:1',
-            displayAspectRatio: '1:1',
-            pixelFormat: PixelFormatUnknown(),
-          }),
+          stream,
         );
         break;
+      }
       case 'blank':
         throw new Error('');
       case 'testsrc':
@@ -664,7 +665,9 @@ export class FfmpegStreamFactory extends IFFMPEG {
       }),
       new FrameState({
         isAnamorphic: false,
-        scaledSize: FrameSize.fromResolution(this.transcodeConfig.resolution),
+        scaledSize:
+          scaledSize ??
+          FrameSize.fromResolution(this.transcodeConfig.resolution),
         paddedSize: FrameSize.fromResolution(this.transcodeConfig.resolution),
         videoBitrate: playbackParams.videoBitrate,
         videoBufferSize: playbackParams.videoBufferSize,

server/src/ffmpeg/builder/MediaStream.ts

@@ -1,7 +1,8 @@
 import type { ExcludeByValueType, Nullable } from '@/types/util.js';
 import { isEmpty, isNull, merge } from 'lodash-es';
 import type { AnyFunction, MarkOptional, StrictOmit } from 'ts-essentials';
-import type { PixelFormat } from './format/PixelFormat.ts';
+import { VideoFormats } from './constants.ts';
+import { PixelFormatUnknown, type PixelFormat } from './format/PixelFormat.ts';
 import type { DataProps, StreamKind } from './types.ts';
 import { FrameSize } from './types.ts';
 
@@ -128,15 +129,17 @@ export class VideoStream implements MediaStream {
   }
 }
 
-type StillImageStreamFields = StrictOmit<
-  DataProps<StillImageStream>,
-  | 'codec'
-  | 'kind'
-  | 'pixelFormat'
-  | 'isAnamorphic'
-  | 'sampleAspectRatio'
-  | 'displayAspectRatio'
-  | 'inputKind'
+type StillImageStreamFields = MarkOptional<
+  StrictOmit<
+    DataProps<StillImageStream>,
+    | 'codec'
+    | 'kind'
+    | 'isAnamorphic'
+    | 'sampleAspectRatio'
+    | 'displayAspectRatio'
+    | 'inputKind'
+  >,
+  'pixelFormat'
 >;
 
 export class StillImageStream extends VideoStream {
@@ -144,11 +147,11 @@ export class StillImageStream extends VideoStream {
 
   private constructor(fields: StillImageStreamFields) {
     super({
-      ...fields,
-      codec: '',
+      codec: VideoFormats.Undetermined,
       sampleAspectRatio: '1:1',
       displayAspectRatio: '1:1',
-      pixelFormat: null,
+      ...fields,
+      pixelFormat: fields.pixelFormat ?? PixelFormatUnknown(),
     });
   }
 

server/src/ffmpeg/builder/filter/nvidia/HardwareUploadCudaFilter.ts

@@ -23,7 +23,11 @@ export class HardwareUploadCudaFilter extends FilterOption {
       return '';
     } else {
       let fmtPart = '';
-      if (this.currentState.pixelFormat?.name === PixelFormats.Unknown) {
+      console.log(this.currentState);
+      if (
+        !this.currentState.pixelFormat ||
+        this.currentState.pixelFormat.name === PixelFormats.Unknown
+      ) {
         const bitDepth = this.currentState.bitDepth;
         this.changedPixelFormat =
           bitDepth === 10

server/src/ffmpeg/builder/filter/nvidia/ScaleCudaFilter.ts

@@ -2,17 +2,23 @@ import { FilterOption } from '@/ffmpeg/builder/filter/FilterOption.js';
 import type { FrameState } from '@/ffmpeg/builder/state/FrameState.js';
 import type { FrameSize } from '@/ffmpeg/builder/types.js';
 import { FrameDataLocation } from '@/ffmpeg/builder/types.js';
+import { isNonEmptyString } from '@tunarr/shared/util';
 import { isEmpty } from 'lodash-es';
+import type { Maybe } from '../../../../types/util.ts';
 import type {
   PixelFormat,
   ValidPixelFormatName,
 } from '../../format/PixelFormat.ts';
 import { PixelFormats } from '../../format/PixelFormat.ts';
+import { HardwareUploadCudaFilter } from './HardwareUploadCudaFilter.ts';
 
 export class ScaleCudaFilter extends FilterOption {
   public filter: string;
   readonly affectsFrameState: boolean = true;
 
+  private uploadFilter: Maybe<HardwareUploadCudaFilter>;
+  private changedPixelFormat = false;
+
   static supportedPixelFormats: ValidPixelFormatName[] = [
     PixelFormats.YUV420P,
     PixelFormats.NV12,
@@ -45,7 +51,8 @@ export class ScaleCudaFilter extends FilterOption {
   }
 
   nextState(currentState: FrameState): FrameState {
-    let nextState = currentState.update({
+    let nextState = this.uploadFilter?.nextState(currentState) ?? currentState;
+    nextState = currentState.update({
       scaledSize: this.scaledSize,
       paddedSize: this.scaledSize,
       frameDataLocation: FrameDataLocation.Hardware,
@@ -54,7 +61,7 @@ export class ScaleCudaFilter extends FilterOption {
     });
 
     const targetPixelFormat = this.supportedPixelFormat;
-    if (targetPixelFormat) {
+    if (targetPixelFormat && this.changedPixelFormat) {
       nextState = nextState.update({
         pixelFormat: targetPixelFormat,
       });
@@ -69,6 +76,7 @@ export class ScaleCudaFilter extends FilterOption {
     if (this.currentState.scaledSize.equals(this.scaledSize)) {
       const targetPixelFormat = this.supportedPixelFormat;
       if (targetPixelFormat) {
+        this.changedPixelFormat = true;
         const passthrough = this.passthrough ? ':passthrough=1' : '';
         scale = `${this.filterName}=format=${targetPixelFormat.name}${passthrough}`;
       }
@@ -83,6 +91,7 @@ export class ScaleCudaFilter extends FilterOption {
       let format = '';
       const targetPixelFormat = this.supportedPixelFormat;
       if (targetPixelFormat) {
+        this.changedPixelFormat = true;
         format = `:format=${targetPixelFormat.name}`;
       }
 
@@ -99,9 +108,14 @@ export class ScaleCudaFilter extends FilterOption {
       return scale;
     }
 
-    return this.currentState.frameDataLocation === FrameDataLocation.Hardware
-      ? scale
-      : `hwupload_cuda,${scale}`;
+    const filters = [scale];
+    if (this.currentState.frameDataLocation === FrameDataLocation.Software) {
+      this.uploadFilter = new HardwareUploadCudaFilter(this.currentState);
+      console.log('apply upload filter');
+      filters.unshift(this.uploadFilter.filter);
+    }
+
+    return filters.filter((f) => isNonEmptyString(f)).join(',');
   }
 
   private get supportedPixelFormat() {

server/src/ffmpeg/builder/format/PixelFormat.ts

@@ -6,6 +6,7 @@ export interface Equatable<T> {
 
 export const PixelFormats = {
   ARGB: 'argb',
+  RGBA: 'rgba',
   YUV420P: 'yuv420p',
   YUVA420P: 'yuva420p',
   YUV420P10LE: 'yuv420p10le',
@@ -110,6 +111,11 @@ export function PixelFormatUnknown(bitDepth: number = 8): BasePixelFormat {
   })();
 }
 
+export class PixelFormatRgba extends SoftwarePixelFormat {
+  readonly name = PixelFormats.RGBA;
+  readonly bitDepth: number = 8; // Shrug
+}
+
 export class PixelFormatYuv420P extends SoftwarePixelFormat {
   readonly name = PixelFormats.YUV420P;
   readonly bitDepth: number = 8;

server/src/ffmpeg/builder/pipeline/nvidia/NvidiaScaler.test.ts

@@ -2,12 +2,16 @@ import { first } from 'lodash-es';
 import { HardwareAccelerationMode } from '../../../../db/schema/TranscodeConfig.ts';
 import { FileStreamSource } from '../../../../stream/types.ts';
 import { FilterChain } from '../../filter/FilterChain.ts';
+import { HardwareUploadCudaFilter } from '../../filter/nvidia/HardwareUploadCudaFilter.ts';
 import { ScaleCudaFilter } from '../../filter/nvidia/ScaleCudaFilter.ts';
 import { ScaleFilter } from '../../filter/ScaleFilter.ts';
 import { PixelFormatYuv420P } from '../../format/PixelFormat.ts';
 import { VideoInputSource } from '../../input/VideoInputSource.ts';
-import { VideoStream } from '../../MediaStream.ts';
-import { FfmpegState } from '../../state/FfmpegState.ts';
+import { StillImageStream, VideoStream } from '../../MediaStream.ts';
+import {
+  DefaultPipelineOptions,
+  FfmpegState,
+} from '../../state/FfmpegState.ts';
 import { FrameState } from '../../state/FrameState.ts';
 import { FrameDataLocation, FrameSize } from '../../types.ts';
 import { PipelineBuilderContext } from '../BasePipelineBuilder.ts';
@@ -44,6 +48,7 @@ describe('NvidiaScaler', () => {
       }),
       shouldDeinterlace: false,
       pipelineSteps: [],
+      pipelineOptions: DefaultPipelineOptions,
     });
 
     const currentState = new FrameState({
@@ -91,6 +96,7 @@ describe('NvidiaScaler', () => {
       }),
       shouldDeinterlace: false,
       pipelineSteps: [],
+      pipelineOptions: DefaultPipelineOptions,
     });
 
     const currentState = new FrameState({
@@ -150,6 +156,7 @@ describe('NvidiaScaler', () => {
       }),
       shouldDeinterlace: false,
       pipelineSteps: [],
+      pipelineOptions: DefaultPipelineOptions,
     });
 
     const currentState = new FrameState({
@@ -175,4 +182,66 @@ describe('NvidiaScaler', () => {
       }),
     );
   });
+
+  test('adds hwupload if necessary before hardware scale', () => {
+    const stream = new FileStreamSource('/path/to/video.mkv');
+    const videoInputSource = VideoInputSource.withStream(
+      stream,
+      StillImageStream.create({
+        index: 0,
+        frameSize: FrameSize.FHD,
+      }),
+    );
+
+    const context = new PipelineBuilderContext({
+      desiredState: new FrameState({
+        isAnamorphic: false,
+        scaledSize: videoInputSource.streams[0].squarePixelFrameSize(
+          FrameSize.SVGA43,
+        ),
+        paddedSize: FrameSize.SVGA43,
+      }),
+      filterChain: new FilterChain(),
+      is10BitOutput: false,
+      isIntelVaapiOrQsv: false,
+      hasWatermark: false,
+      videoStream: videoInputSource.streams[0],
+      ffmpegState: FfmpegState.create({
+        version: { versionString: '7.0.1', isUnknown: false },
+        decoderHwAccelMode: HardwareAccelerationMode.Cuda,
+        encoderHwAccelMode: HardwareAccelerationMode.Cuda,
+      }),
+      shouldDeinterlace: false,
+      pipelineSteps: [],
+      pipelineOptions: DefaultPipelineOptions,
+    });
+
+    const currentState = new FrameState({
+      isAnamorphic: false,
+      scaledSize: FrameSize.SevenTwenty,
+      paddedSize: FrameSize.FHD,
+      frameDataLocation: FrameDataLocation.Software,
+    });
+
+    const nextState = NvidiaScaler.setScale(
+      context,
+      videoInputSource,
+      currentState,
+    );
+
+    const [hwuploadStep, scaleStep] = videoInputSource.filterSteps;
+    expect(hwuploadStep).toBeInstanceOf(HardwareUploadCudaFilter);
+    expect(hwuploadStep.filter).toEqual('format=yuv420p,hwupload_cuda');
+    expect(scaleStep).toBeInstanceOf(ScaleCudaFilter);
+    expect(scaleStep?.filter).toEqual(
+      'scale_cuda=800:600:force_original_aspect_ratio=decrease,setsar=1',
+    );
+    expect(nextState).toStrictEqual(
+      currentState.update({
+        frameDataLocation: FrameDataLocation.Hardware,
+        scaledSize: FrameSize.withDimensions(800, 450),
+        paddedSize: FrameSize.withDimensions(800, 450), // Pad not applied yet
+      }),
+    );
+  });
 });

server/src/ffmpeg/builder/pipeline/nvidia/NvidiaScaler.ts

@@ -1,11 +1,13 @@
 import { HardwareAccelerationMode } from '../../../../db/schema/TranscodeConfig.ts';
 import { isNonEmptyString } from '../../../../util/index.ts';
 import type { FilterOption } from '../../filter/FilterOption.ts';
+import { HardwareUploadCudaFilter } from '../../filter/nvidia/HardwareUploadCudaFilter.ts';
 import { ScaleCudaFilter } from '../../filter/nvidia/ScaleCudaFilter.ts';
 import { ScaleFilter } from '../../filter/ScaleFilter.ts';
 import { PixelFormatNv12 } from '../../format/PixelFormat.ts';
 import type { VideoInputSource } from '../../input/VideoInputSource.ts';
 import type { FrameState } from '../../state/FrameState.ts';
+import { FrameDataLocation } from '../../types.ts';
 import type { PipelineBuilderContext } from '../BasePipelineBuilder.ts';
 
 export class NvidiaScaler {
@@ -59,6 +61,13 @@ export class NvidiaScaler {
             ? new PixelFormatNv12(desiredState.pixelFormat)
             : null
           : null;
+      if (currentState.frameDataLocation === FrameDataLocation.Software) {
+        const hwupload = new HardwareUploadCudaFilter(currentState);
+        if (isNonEmptyString(hwupload.filter)) {
+          videoInputSource.filterSteps.push(hwupload);
+          currentState = hwupload.nextState(currentState);
+        }
+      }
       scaleStep = new ScaleCudaFilter(
         currentState.update({
           pixelFormat: outPixelFormat,

server/src/ffmpeg/builder/types.ts

@@ -67,6 +67,7 @@ export class FrameSize {
   public static SevenTwenty = FrameSize.withDimensions(1280, 720);
   public static FHD = FrameSize.withDimensions(1920, 1080);
   public static FourK = FrameSize.withDimensions(3840, 2160);
+  public static SVGA43 = FrameSize.withDimensions(800, 600);
 }
 
 export enum RateControlMode {