fix: fixes for CUDA pipeline state management (#998)

Also enables "watermark duration" for the OverlayWatermarkFilter

.gitignore

@@ -17,3 +17,4 @@ log.txt
 .turbo/
 *.env*
 *.tar*
+coverage/

server/package.json

@@ -89,6 +89,7 @@
     "@types/unzip-stream": "^0.3.4",
     "@types/uuid": "^9.0.6",
     "@types/yargs": "^17.0.29",
+    "@vitest/coverage-v8": "^2.1.8",
     "copyfiles": "^2.2.0",
     "del-cli": "^3.0.0",
     "dotenv": "^16.4.5",
@@ -119,7 +120,7 @@
     "typescript": "5.4.3",
     "typescript-eslint": "^7.5.0",
     "unzip-stream": "^0.3.4",
-    "vitest": "^2.0.5"
+    "vitest": "^2.1.8"
   },
   "mikro-orm": {
     "configPaths": [

server/src/ffmpeg/builder/FfmpegCommandGenerator.test.ts

@@ -1,4 +1,4 @@
-import { bootstrapTunarr } from '@/ffmpeg/builder/bootstrap.ts';
+import { bootstrapTunarr } from '@/bootstrap.ts';
 import { setGlobalOptions } from '@/globals.ts';
 import tmp from 'tmp';
 import { FfmpegCommandGenerator } from './FfmpegCommandGenerator.ts';

server/src/ffmpeg/builder/FfmpegCommandGenerator.ts

@@ -36,16 +36,6 @@ export class FfmpegCommandGenerator {
     concatInputSource: Nullable<ConcatInputSource>,
     steps: PipelineStep[],
   ): string[] {
-    // const stepsByType = reduce(
-    //   steps,
-    //   (acc, curr) => {
-    //     acc[curr.type].push(curr);
-    //     return acc;
-    //   },
-    //   emptyStepMap(),
-    // );
-
-    // const args: string[] = [...flatMap(steps, (step) => step.globalOptions())];
     const args = [
       ...flatMap(filter(steps, isGlobalOption), (step) => step.options()),
     ];

server/src/ffmpeg/builder/filter/PixelFormatFilter.ts

@@ -11,7 +11,6 @@ export class PixelFormatFilter extends FilterOption {
     this.filter = `format=${pixelFormat.ffmpegName}`;
   }
 
-  // TOOD: update pixel format in state
   nextState(currentState: FrameState): FrameState {
     return currentState.update({
       pixelFormat: this.pixelFormat,

server/src/ffmpeg/builder/filter/nvidia/HardwareDownloadCudaFilter.test.ts

@@ -0,0 +1,85 @@
+import {
+  PixelFormatNv12,
+  PixelFormatYuv420P,
+  PixelFormatYuv444P,
+} from '../../format/PixelFormat.ts';
+import { FrameState } from '../../state/FrameState.ts';
+import { FrameDataLocation, FrameSize } from '../../types.ts';
+import { HardwareDownloadCudaFilter } from './HardwareDownloadCudaFilter.ts';
+
+describe('HardwareDownloadCudaFilter', () => {
+  test('currentFormat=null', () => {
+    const filter = new HardwareDownloadCudaFilter(null, null);
+    const currentState = new FrameState({
+      isAnamorphic: false,
+      paddedSize: FrameSize.withDimensions(1920, 1080),
+      scaledSize: FrameSize.withDimensions(1920, 1080),
+      frameDataLocation: FrameDataLocation.Hardware,
+    });
+
+    expect(filter.filter).to.eq('hwdownload');
+
+    const nextState = filter.nextState(currentState);
+    expect(nextState).toMatchObject({
+      frameDataLocation: FrameDataLocation.Software,
+    });
+
+    // Does not mutate
+    expect(nextState).not.toBe(currentState);
+  });
+
+  test('currentFormat=nv12', () => {
+    const underlyingPixelFormat = new PixelFormatYuv420P();
+    const currentState = new FrameState({
+      isAnamorphic: false,
+      paddedSize: FrameSize.withDimensions(1920, 1080),
+      scaledSize: FrameSize.withDimensions(1920, 1080),
+      frameDataLocation: FrameDataLocation.Hardware,
+      pixelFormat: new PixelFormatNv12(underlyingPixelFormat.name),
+    });
+
+    const filter = new HardwareDownloadCudaFilter(
+      currentState.pixelFormat,
+      null,
+    );
+
+    expect(filter.filter).to.eq('hwdownload,format=nv12,format=yuv420p');
+
+    const nextState = filter.nextState(currentState);
+    expect(nextState).toMatchObject({
+      frameDataLocation: FrameDataLocation.Software,
+      pixelFormat: underlyingPixelFormat,
+    });
+
+    // Does not mutate
+    expect(nextState).not.toBe(currentState);
+  });
+
+  test('currentFormat=nv12 targetFormat=yuv444p', () => {
+    const underlyingPixelFormat = new PixelFormatYuv420P();
+    const targetFormat = new PixelFormatYuv444P();
+    const currentState = new FrameState({
+      isAnamorphic: false,
+      paddedSize: FrameSize.withDimensions(1920, 1080),
+      scaledSize: FrameSize.withDimensions(1920, 1080),
+      frameDataLocation: FrameDataLocation.Hardware,
+      pixelFormat: new PixelFormatNv12(underlyingPixelFormat.name),
+    });
+
+    const filter = new HardwareDownloadCudaFilter(
+      currentState.pixelFormat,
+      targetFormat,
+    );
+
+    expect(filter.filter).to.eq('hwdownload,format=nv12,format=yuv444p');
+
+    const nextState = filter.nextState(currentState);
+    expect(nextState).toMatchObject({
+      frameDataLocation: FrameDataLocation.Software,
+      pixelFormat: targetFormat,
+    });
+
+    // Does not mutate
+    expect(nextState).not.toBe(currentState);
+  });
+});

server/src/ffmpeg/builder/filter/nvidia/HardwareDownloadCudaFilter.ts

@@ -7,6 +7,7 @@ import { FrameState } from '@/ffmpeg/builder/state/FrameState.ts';
 import { Nullable } from '@/types/util.ts';
 import { isNonEmptyString } from '@/util/index.ts';
 import { isNull } from 'lodash-es';
+import { FrameDataLocation } from '../../types.ts';
 
 export class HardwareDownloadCudaFilter extends FilterOption {
   public affectsFrameState: boolean = true;
@@ -41,14 +42,19 @@ export class HardwareDownloadCudaFilter extends FilterOption {
   }
 
   nextState(currentState: FrameState): FrameState {
-    let nextState = currentState.updateFrameLocation('software');
+    let nextState = currentState.updateFrameLocation(
+      FrameDataLocation.Software,
+    );
+
     if (!isNull(this.targetPixelFormat)) {
       return nextState.update({ pixelFormat: this.targetPixelFormat });
     }
 
     if (!isNull(this.currentPixelFormat)) {
       if (this.currentPixelFormat.ffmpegName === FfmpegPixelFormats.NV12) {
-        nextState = nextState.update({ pixelFormat: null });
+        nextState = nextState.update({
+          pixelFormat: this.currentPixelFormat.unwrap(),
+        });
       } else {
         nextState = nextState.update({ pixelFormat: this.currentPixelFormat });
       }

server/src/ffmpeg/builder/filter/nvidia/HardwareUploadCudaFilter.ts

@@ -1,5 +1,6 @@
 import { FilterOption } from '@/ffmpeg/builder/filter/FilterOption.ts';
 import { FrameState } from '@/ffmpeg/builder/state/FrameState.ts';
+import { FrameDataLocation } from '../../types.ts';
 
 export class HardwareUploadCudaFilter extends FilterOption {
   public affectsFrameState: boolean = true;
@@ -9,7 +10,7 @@ export class HardwareUploadCudaFilter extends FilterOption {
   }
 
   get filter() {
-    if (this.currentState.frameDataLocation === 'hardware') {
+    if (this.currentState.frameDataLocation === FrameDataLocation.Hardware) {
       return '';
     } else {
       return 'hwupload_cuda';
@@ -17,8 +18,6 @@ export class HardwareUploadCudaFilter extends FilterOption {
   }
 
   nextState(currentState: FrameState): FrameState {
-    return currentState.update({
-      frameDataLocation: 'hardware',
-    });
+    return currentState.updateFrameLocation(FrameDataLocation.Hardware);
   }
 }

server/src/ffmpeg/builder/filter/nvidia/OverlayWatermarkCudaFilter.ts

@@ -1,7 +1,7 @@
 import { OverlayWatermarkFilter } from '@/ffmpeg/builder/filter/watermark/OverlayWatermarkFilter.ts';
 import { PixelFormatUnknown } from '@/ffmpeg/builder/format/PixelFormat.ts';
 import { FrameState } from '@/ffmpeg/builder/state/FrameState.ts';
-import { FrameSize } from '@/ffmpeg/builder/types.ts';
+import { FrameDataLocation, FrameSize } from '@/ffmpeg/builder/types.ts';
 import { Watermark } from '@tunarr/types';
 
 export class OverlayWatermarkCudaFilter extends OverlayWatermarkFilter {
@@ -13,6 +13,6 @@ export class OverlayWatermarkCudaFilter extends OverlayWatermarkFilter {
   }
 
   nextState(currentState: FrameState): FrameState {
-    return currentState.updateFrameLocation('hardware');
+    return currentState.updateFrameLocation(FrameDataLocation.Hardware);
   }
 }

server/src/ffmpeg/builder/filter/nvidia/ScaleCudaFilter.ts

@@ -17,13 +17,21 @@ export class ScaleCudaFilter extends FilterOption {
   }
 
   nextState(currentState: FrameState): FrameState {
-    return currentState.update({
+    let nextState = currentState.update({
       scaledSize: this.scaledSize,
       paddedSize: this.scaledSize,
       frameDataLocation: FrameDataLocation.Hardware,
       // this filter always outputs square pixels
       isAnamorphic: false,
     });
+
+    if (this.currentState.pixelFormat) {
+      nextState = nextState.update({
+        pixelFormat: this.currentState.pixelFormat,
+      });
+    }
+
+    return nextState;
   }
 
   private generateFilter(): string {

server/src/ffmpeg/builder/filter/watermark/OverlayWatermarkFilter.ts

@@ -9,7 +9,7 @@ export class OverlayWatermarkFilter extends FilterOption {
   public filter: string;
 
   constructor(
-    private watermark: Watermark,
+    protected watermark: Watermark,
     private resolution: FrameSize,
     // @ts-expect-error - We're going to use this soon
     private squarePixelResolution: FrameSize,
@@ -52,8 +52,11 @@ export class OverlayWatermarkFilter extends FilterOption {
   }
 
   private generateFilter() {
-    return `overlay=${this.getPosition()}:format=${
-      this.outputPixelFormat.bitDepth === 10 ? 1 : 0
-    }`;
+    const enablePart =
+      this.watermark.duration > 0
+        ? `:enable='between(t,0,${this.watermark.duration})'`
+        : '';
+    const format = this.outputPixelFormat.bitDepth === 10 ? 1 : 0;
+    return `overlay=${this.getPosition()}:format=${format}${enablePart}`;
   }
 }

server/src/ffmpeg/builder/pipeline/hardware/NvidiaPipelineBuilder.ts

@@ -33,8 +33,8 @@ import {
   HardwareAccelerationMode,
 } from '@/ffmpeg/builder/types.ts';
 import { Nullable } from '@/types/util.ts';
-import { isNonEmptyString } from '@/util/index.ts';
-import { isNil, isNull, reject, some } from 'lodash-es';
+import { isDefined, isNonEmptyString } from '@/util/index.ts';
+import { isEmpty, isNil, isNull, reject, some } from 'lodash-es';
 import {
   NvidiaH264Encoder,
   NvidiaHevcEncoder,
@@ -179,6 +179,12 @@ export class NvidiaPipelineBuilder extends SoftwarePipelineBuilder {
       }
     }
 
+    const needsSoftwareWatermarkOverlay =
+      (this.context.hasWatermark &&
+        !isEmpty(this.watermarkInputSource?.watermark.fadeConfig)) ||
+      (isDefined(this.watermarkInputSource?.watermark.duration) &&
+        this.watermarkInputSource.watermark.duration > 0);
+
     if (
       currentState.frameDataLocation === FrameDataLocation.Software &&
       currentState.bitDepth === 8 &&
@@ -186,13 +192,27 @@ export class NvidiaPipelineBuilder extends SoftwarePipelineBuilder {
       // filter unless we're on >=5.0.0 because overlay_cuda does
       // not support the W/w/H/h params on earlier versions
       this.ffmpegState.isAtLeastVersion({ major: 5 }) &&
-      this.watermarkInputSource
+      !needsSoftwareWatermarkOverlay
     ) {
       const filter = new HardwareUploadCudaFilter(currentState);
       currentState = filter.nextState(currentState);
       this.videoInputSource.filterSteps.push(filter);
     }
 
+    // Overlay watermark in software if we have any timeline-enabled features
+    // (e.g. intermittent watermarks or absolute duration)
+    if (
+      currentState.frameDataLocation === FrameDataLocation.Hardware &&
+      needsSoftwareWatermarkOverlay
+    ) {
+      const hwDownloadFilter = new HardwareDownloadCudaFilter(
+        currentState.pixelFormat,
+        null,
+      );
+      currentState = hwDownloadFilter.nextState(currentState);
+      this.videoInputSource.filterSteps.push(hwDownloadFilter);
+    }
+
     currentState = this.setWatermark(currentState);
 
     let encoder: Nullable<VideoEncoder> = null;
@@ -222,12 +242,6 @@ export class NvidiaPipelineBuilder extends SoftwarePipelineBuilder {
 
     currentState = this.setPixelFormat(currentState);
 
-    if (currentState.frameDataLocation === FrameDataLocation.Software) {
-      this.videoInputSource.filterSteps.push(
-        new HardwareUploadCudaFilter(currentState),
-      );
-    }
-
     this.context.filterChain.videoFilterSteps =
       this.videoInputSource.filterSteps;
   }
@@ -420,9 +434,9 @@ export class NvidiaPipelineBuilder extends SoftwarePipelineBuilder {
         ) {
           // We are in software with possibly hardware formatted pixels... if the underlying
           // pixel format type is the same as our desired type, we shouldn't need to do anything!
-          this.pipelineSteps.push(new PixelFormatFilter(desiredFormat));
+          // this.pipelineSteps.push(new PixelFormatFilter(desiredFormat));
           // Using the output option seems to break with NVENC...
-          // this.pipelineSteps.push(new PixelFormatOutputOption(desiredFormat));
+          this.pipelineSteps.push(new PixelFormatOutputOption(desiredFormat));
         }
       }
 
@@ -485,6 +499,11 @@ export class NvidiaPipelineBuilder extends SoftwarePipelineBuilder {
           overlayFilter,
         );
         currentState = overlayFilter.nextState(currentState);
+      } else {
+        this.logger.warn(
+          'Cannot overlay watermark without a known pixel format target! Desired state was: %s',
+          this.desiredState.pixelFormat?.name,
+        );
       }
     } else {
       this.watermarkInputSource.filterSteps.push(

server/src/stream/hls/HlsSession.ts

@@ -145,7 +145,7 @@ export class HlsSession extends BaseHlsSession<HlsSessionOptions> {
       async (result) => {
         this.logger.debug(
           'About to play item: %s',
-          JSON.stringify(lineupItemResult, undefined, 4),
+          JSON.stringify(result, undefined, 4),
         );
         const context = new PlayerContext(
           result.lineupItem,

server/vitest.config.ts

@@ -1,9 +1,18 @@
+import path from 'node:path';
 import { defineConfig } from 'vitest/config';
 
 export default defineConfig({
+  resolve: {
+    alias: {
+      '@': path.resolve(__dirname, 'src'),
+    },
+  },
   test: {
     globals: true,
     includeSource: ['src/**/*.test.ts'],
+    coverage: {
+      provider: 'v8',
+    },
   },
   define: {
     'import.meta.vitest': false,