bybrooklyn/tunarr
1
0
Fork 0
mirror of https://github.com/chrisbenincasa/tunarr.git synced 2026-04-18 09:03:35 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix: more QSV pixel format fixes (#1723)

Browse Source
This commit is contained in:
Christian Benincasa
2026-03-30 15:28:55 -04:00
committed by GitHub
parent 2d1f0cbc87
commit 73954b2a26
5 changed files with 1069 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/ffmpeg/builder/MediaStream.ts
View File

@@ -26,7 +26,7 @@ type MediaStreamFields<T extends MediaStream> = Omit<
// semantics with class construction, but still enabling us
// to have hierarchies, methods, etc.
type AudioStreamFields = MediaStreamFields<AudioStream>;
type VideoStreamFields = StrictOmit<
export type VideoStreamFields = StrictOmit<
MediaStreamFields<VideoStream>,
'isAnamorphic' | 'sampleAspectRatio'
>;

699
server/src/ffmpeg/builder/pipeline/hardware/QsvPipeline.local.test.ts
View File

@@ -1,3 +1,5 @@
import dayjs from 'dayjs';
import duration from 'dayjs/plugin/duration.js';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { FileStreamSource } from '../../../../stream/types.ts';
@@ -11,11 +13,24 @@ import {
qsvInfo,
qsvTest,
} from '../../../../testing/ffmpeg/FfmpegTestFixtures.ts';
import { AudioFormats, FileOutputLocation } from '../../constants.ts';
import {
AudioFormats,
FileOutputLocation,
VideoFormats,
} from '../../constants.ts';
import { PixelFormatYuv420P } from '../../format/PixelFormat.ts';
import { AudioInputSource } from '../../input/AudioInputSource.ts';
import {
AudioInputFilterSource,
AudioInputSource,
} from '../../input/AudioInputSource.ts';
import { LavfiVideoInputSource } from '../../input/LavfiVideoInputSource.ts';
import { VideoInputSource } from '../../input/VideoInputSource.ts';
import { AudioStream, VideoStream } from '../../MediaStream.ts';
import { WatermarkInputSource } from '../../input/WatermarkInputSource.ts';
import {
AudioStream,
StillImageStream,
VideoStream,
} from '../../MediaStream.ts';
import { AudioState } from '../../state/AudioState.ts';
import {
DefaultPipelineOptions,
@@ -25,6 +40,8 @@ import { FrameState } from '../../state/FrameState.ts';
import { FrameSize } from '../../types.ts';
import { QsvPipelineBuilder } from './QsvPipelineBuilder.ts';
dayjs.extend(duration);
const fixturesDir = path.join(
path.dirname(fileURLToPath(import.meta.url)),
'../../../../testing/ffmpeg/fixtures',
@@ -33,8 +50,83 @@ const fixturesDir = path.join(
const Fixtures = {
video720p: path.join(fixturesDir, '720p_h264.ts'),
video1080p: path.join(fixturesDir, '1080p_h264.ts'),
video480p43: path.join(fixturesDir, '480p_h264.ts'),
watermark: path.join(fixturesDir, 'watermark.png'),
blackWatermark: path.join(fixturesDir, 'black_watermark.png'),
} as const;
// Limit output to 1 second in all integration tests to keep runs fast
const testDuration = dayjs.duration(1, 'second');
// ─── Shared helpers ───────────────────────────────────────────────────────────
function makeH264VideoInput(inputPath: string, frameSize: FrameSize) {
return VideoInputSource.withStream(
new FileStreamSource(inputPath),
VideoStream.create({
codec: 'h264',
displayAspectRatio: '16:9',
frameSize,
index: 0,
pixelFormat: new PixelFormatYuv420P(),
providedSampleAspectRatio: null,
colorFormat: null,
}),
);
}
function make43VideoInput(inputPath: string) {
return VideoInputSource.withStream(
new FileStreamSource(inputPath),
VideoStream.create({
codec: 'h264',
profile: 'main',
displayAspectRatio: '4:3',
frameSize: FrameSize.withDimensions(640, 480),
index: 0,
pixelFormat: new PixelFormatYuv420P(),
providedSampleAspectRatio: null,
colorFormat: null,
}),
);
}
function makeAudioInput(inputPath: string) {
return AudioInputSource.withStream(
new FileStreamSource(inputPath),
AudioStream.create({ channels: 2, codec: 'aac', index: 1 }),
AudioState.create({
audioEncoder: AudioFormats.Aac,
audioChannels: 2,
audioBitrate: 192,
audioBufferSize: 384,
}),
);
}
function makeWatermark(color: 'white' | 'black' = 'white') {
return new WatermarkInputSource(
new FileStreamSource(
color === 'white' ? Fixtures.watermark : Fixtures.blackWatermark,
),
StillImageStream.create({
frameSize: FrameSize.withDimensions(100, 100),
index: 0,
}),
{
enabled: true,
position: 'bottom-right',
width: 10,
verticalMargin: 5,
horizontalMargin: 5,
duration: 0,
opacity: 100,
},
);
}
// ─────────────────────────────────────────────────────────────────────────────
describe.skipIf(!binaries || !qsvInfo)('QsvPipelineBuilder integration', () => {
let workdir: string;
let cleanup: () => Promise<void>;
@@ -45,193 +137,458 @@ describe.skipIf(!binaries || !qsvInfo)('QsvPipelineBuilder integration', () => {
afterAll(() => cleanup());
function makeVideoInput(inputPath: string, frameSize: FrameSize) {
return VideoInputSource.withStream(
new FileStreamSource(inputPath),
VideoStream.create({
codec: 'h264',
displayAspectRatio: '16:9',
frameSize,
index: 0,
pixelFormat: new PixelFormatYuv420P(),
providedSampleAspectRatio: null,
colorFormat: null,
}),
);
}
function makeAudioInput(inputPath: string) {
return AudioInputSource.withStream(
new FileStreamSource(inputPath),
AudioStream.create({
channels: 2,
codec: 'aac',
index: 1,
}),
AudioState.create({
audioEncoder: AudioFormats.Aac,
audioChannels: 2,
audioBitrate: 192,
audioBufferSize: 384,
}),
);
}
// QsvPipelineBuilder arg order: hardwareCaps, binaryCaps, video, audio, concat, watermark, subtitle
qsvTest('basic h264 qsv transcode', async ({
binaryCapabilities,
ffmpegVersion,
resolvedQsv,
}) => {
const video = makeVideoInput(
Fixtures.video720p,
FrameSize.withDimensions(1280, 720),
);
const audio = makeAudioInput(Fixtures.video720p);
qsvTest(
'basic h264 qsv transcode',
async ({ binaryCapabilities, ffmpegVersion, resolvedQsv }) => {
const video = makeH264VideoInput(
Fixtures.video720p,
FrameSize.withDimensions(1280, 720),
);
const audio = makeAudioInput(Fixtures.video720p);
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
null,
null,
);
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
null,
null,
);
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.withDimensions(1280, 720),
paddedSize: FrameSize.withDimensions(1280, 720),
});
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.withDimensions(1280, 720),
paddedSize: FrameSize.withDimensions(1280, 720),
});
const outputPath = path.join(workdir, 'qsv_transcode.ts');
const pipeline = builder.build(
FfmpegState.create({
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const outputPath = path.join(workdir, 'qsv_transcode.ts');
const pipeline = builder.build(
FfmpegState.create({
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
pipeline.getCommandArgs(),
);
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
pipeline.getCommandArgs(),
);
expect(
exitCode,
`Pipeline command failed: ${pipeline.getCommandArgs().join(' ')}\n${stderr}`,
).toBe(0);
expect(
exitCode,
`Pipeline command failed: ${pipeline.getCommandArgs().join(' ')}\n${stderr}`,
).toBe(0);
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
});
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
},
);
qsvTest('scale from 1080p to 720p via qsv', async ({
binaryCapabilities,
ffmpegVersion,
resolvedQsv,
}) => {
const video = makeVideoInput(
Fixtures.video1080p,
FrameSize.withDimensions(1920, 1080),
);
const audio = makeAudioInput(Fixtures.video1080p);
qsvTest(
'scale from 1080p to 720p via qsv',
async ({ binaryCapabilities, ffmpegVersion, resolvedQsv }) => {
const video = makeH264VideoInput(
Fixtures.video1080p,
FrameSize.withDimensions(1920, 1080),
);
const audio = makeAudioInput(Fixtures.video1080p);
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
null,
null,
);
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
null,
null,
);
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.withDimensions(1280, 720),
paddedSize: FrameSize.withDimensions(1280, 720),
});
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.withDimensions(1280, 720),
paddedSize: FrameSize.withDimensions(1280, 720),
});
const outputPath = path.join(workdir, 'qsv_scale.ts');
const pipeline = builder.build(
FfmpegState.create({
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const outputPath = path.join(workdir, 'qsv_scale.ts');
const pipeline = builder.build(
FfmpegState.create({
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
pipeline.getCommandArgs(),
);
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
pipeline.getCommandArgs(),
);
expect(
exitCode,
`Pipeline command failed: ${pipeline.getCommandArgs().join(' ')}\n${stderr}`,
).toBe(0);
expect(
exitCode,
`Pipeline command failed: ${pipeline.getCommandArgs().join(' ')}\n${stderr}`,
).toBe(0);
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
});
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
},
);
qsvTest('copy mode (qsv pipeline, no hw transcode needed)', async ({
binaryCapabilities,
ffmpegVersion,
resolvedQsv,
}) => {
const video = makeVideoInput(
Fixtures.video720p,
FrameSize.withDimensions(1280, 720),
);
const audio = makeAudioInput(Fixtures.video720p);
qsvTest(
'copy mode (qsv pipeline, no hw transcode needed)',
async ({ binaryCapabilities, ffmpegVersion, resolvedQsv }) => {
const video = makeH264VideoInput(
Fixtures.video720p,
FrameSize.withDimensions(1280, 720),
);
const audio = makeAudioInput(Fixtures.video720p);
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
null,
null,
);
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
null,
null,
);
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.withDimensions(1280, 720),
paddedSize: FrameSize.withDimensions(1280, 720),
videoFormat: 'copy',
});
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.withDimensions(1280, 720),
paddedSize: FrameSize.withDimensions(1280, 720),
videoFormat: 'copy',
});
const outputPath = path.join(workdir, 'qsv_copy.ts');
const pipeline = builder.build(
FfmpegState.create({
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const outputPath = path.join(workdir, 'qsv_copy.ts');
const pipeline = builder.build(
FfmpegState.create({
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
pipeline.getCommandArgs(),
);
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
pipeline.getCommandArgs(),
);
expect(
exitCode,
`Pipeline command failed: ${pipeline.getCommandArgs().join(' ')}\n${stderr}`,
).toBe(0);
expect(
exitCode,
`Pipeline command failed: ${pipeline.getCommandArgs().join(' ')}\n${stderr}`,
).toBe(0);
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
});
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
},
);
});
// ─── Pixel format fix integration tests ──────────────────────────────────────
describe.skipIf(!binaries || !qsvInfo)(
'QsvPipelineBuilder pixel format fixes',
() => {
let workdir: string;
let cleanup: () => Promise<void>;
beforeAll(async () => {
({ dir: workdir, cleanup } = await createTempWorkdir());
});
afterAll(() => cleanup());
// Bug 3: after watermark overlay frames are in software (yuv420p); without
// the fix, bare hwupload would fail format negotiation.
qsvTest(
'QSV transcode with watermark (Bug 3: format=nv12 before hwupload)',
async ({ binaryCapabilities, ffmpegVersion, resolvedQsv }) => {
const video = makeH264VideoInput(
Fixtures.video720p,
FrameSize.withDimensions(1280, 720),
);
const audio = makeAudioInput(Fixtures.video720p);
const watermark = makeWatermark('black');
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
watermark,
null,
);
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.withDimensions(1280, 720),
paddedSize: FrameSize.withDimensions(1280, 720),
});
const outputPath = path.join(workdir, 'qsv_watermark.ts');
const pipeline = builder.build(
FfmpegState.create({
duration: testDuration,
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
pipeline.getCommandArgs(),
);
expect(
exitCode,
`Pipeline command failed: ${pipeline.getCommandArgs().join(' ')}\n${stderr}`,
).toBe(0);
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
},
);
// Bug 3: scaling + padding + watermark path — ensures the format=nv12,hwupload
// sequence is correct even when scale_qsv and pad filters also run.
qsvTest(
'QSV transcode with scaling + padding + watermark (Bug 3)',
async ({ binaryCapabilities, ffmpegVersion, resolvedQsv }) => {
const video = makeH264VideoInput(
Fixtures.video1080p,
FrameSize.withDimensions(1920, 1080),
);
const audio = makeAudioInput(Fixtures.video1080p);
const watermark = makeWatermark();
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
watermark,
null,
);
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.withDimensions(1280, 720),
paddedSize: FrameSize.withDimensions(1280, 720),
});
const outputPath = path.join(workdir, 'qsv_scale_watermark.ts');
const pipeline = builder.build(
FfmpegState.create({
duration: testDuration,
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
pipeline.getCommandArgs(),
);
expect(
exitCode,
`Pipeline command failed: ${pipeline.getCommandArgs().join(' ')}\n${stderr}`,
).toBe(0);
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
},
);
// Bug 3: anamorphic (4:3) source forces scale + pillarbox pad + overlay +
// hwupload chain — the deepest exercise of the fix.
qsvTest(
'QSV transcode of anamorphic content with watermark (Bug 3)',
async ({ binaryCapabilities, ffmpegVersion, resolvedQsv }) => {
const video = make43VideoInput(Fixtures.video480p43);
const audio = makeAudioInput(Fixtures.video480p43);
const watermark = makeWatermark();
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
watermark,
null,
);
// 4:3 → squarePixelFrameSize gives 1440x1080, padded to 1920x1080
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: video.streams[0]!.squarePixelFrameSize(FrameSize.FHD),
paddedSize: FrameSize.FHD,
});
const outputPath = path.join(workdir, 'qsv_anamorphic_watermark.ts');
const pipeline = builder.build(
FfmpegState.create({
duration: testDuration,
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
pipeline.getCommandArgs(),
);
expect(
exitCode,
`Pipeline command failed: ${pipeline.getCommandArgs().join(' ')}\n${stderr}`,
).toBe(0);
const probe = probeFile(binaries!.ffprobe, outputPath);
const videoOut = probe.streams.find((s) => s.codec_type === 'video');
expect(videoOut).toBeDefined();
expect(videoOut!.width).toBe(1920);
expect(videoOut!.height).toBe(1080);
},
);
// Bug 1: LavfiVideoInputSource sets PixelFormatUnknown(); without the fix
// this produces vpp_qsv=format=unknown and ffmpeg fails.
qsvTest(
'error screen (LavfiVideoInputSource) does not produce format=unknown (Bug 1)',
async ({ binaryCapabilities, ffmpegVersion, resolvedQsv }) => {
const audioState = AudioState.create({
audioEncoder: AudioFormats.Aac,
audioChannels: 2,
audioBitrate: 192,
audioBufferSize: 384,
});
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
LavfiVideoInputSource.errorText(FrameSize.FHD, 'Error', 'Test'),
AudioInputFilterSource.noise(audioState),
null,
null,
null,
);
const outputPath = path.join(workdir, 'qsv_error_screen.ts');
const pipeline = builder.build(
FfmpegState.create({
duration: testDuration,
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.FHD,
paddedSize: FrameSize.FHD,
videoFormat: VideoFormats.H264,
pixelFormat: new PixelFormatYuv420P(),
}),
DefaultPipelineOptions,
);
const args = pipeline.getCommandArgs();
expect(args.join(' ')).not.toContain('format=unknown');
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
args,
);
expect(
exitCode,
`Pipeline command failed: ${args.join(' ')}\n${stderr}`,
).toBe(0);
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
},
);
// Bug 2: -pix_fmt yuv420p is incompatible with h264_qsv operating on hardware
// frames; without the fix ffmpeg crashes with a swscaler error.
qsvTest(
'no scaling path does not emit -pix_fmt yuv420p for QSV encode (Bug 2)',
async ({ binaryCapabilities, ffmpegVersion, resolvedQsv }) => {
const video = makeH264VideoInput(
Fixtures.video720p,
FrameSize.withDimensions(1280, 720),
);
const audio = makeAudioInput(Fixtures.video720p);
const builder = new QsvPipelineBuilder(
resolvedQsv.capabilities,
binaryCapabilities,
video,
audio,
null,
null,
null,
);
const frameState = new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.withDimensions(1280, 720),
paddedSize: FrameSize.withDimensions(1280, 720),
videoFormat: VideoFormats.H264,
pixelFormat: new PixelFormatYuv420P(),
});
const outputPath = path.join(workdir, 'qsv_no_scale_pix_fmt.ts');
const pipeline = builder.build(
FfmpegState.create({
duration: testDuration,
version: ffmpegVersion,
outputLocation: FileOutputLocation(outputPath, true),
vaapiDevice: resolvedQsv.device,
}),
frameState,
DefaultPipelineOptions,
);
const args = pipeline.getCommandArgs();
expect(args).not.toContain('-pix_fmt');
const { exitCode, stderr } = runFfmpegWithPipeline(
binaries!.ffmpeg,
args,
);
expect(
exitCode,
`Pipeline command failed: ${args.join(' ')}\n${stderr}`,
).toBe(0);
expect(stderr).not.toContain('swscaler');
const probe = probeFile(binaries!.ffprobe, outputPath);
expect(probe.streams.some((s) => s.codec_type === 'video')).toBe(true);
},
);
},
);

596
server/src/ffmpeg/builder/pipeline/hardware/QsvPipelineBuilder.test.ts
View File

@@ -1,5 +1,6 @@
import { Watermark } from '@tunarr/types';
import dayjs from 'dayjs';
import { StrictOmit } from 'ts-essentials';
import { FileStreamSource } from '../../../../stream/types.ts';
import { TUNARR_ENV_VARS } from '../../../../util/env.ts';
import { EmptyFfmpegCapabilities } from '../../capabilities/FfmpegCapabilities.ts';
@@ -14,20 +15,22 @@ import {
ColorRanges,
ColorSpaces,
ColorTransferFormats,
VideoFormats,
} from '../../constants.ts';
import { HardwareDownloadFilter } from '../../filter/HardwareDownloadFilter.ts';
import { PixelFormatFilter } from '../../filter/PixelFormatFilter.ts';
import { HardwareUploadQsvFilter } from '../../filter/qsv/HardwareUploadQsvFilter.ts';
import { QsvFormatFilter } from '../../filter/qsv/QsvFormatFilter.ts';
import { TonemapQsvFilter } from '../../filter/qsv/TonemapQsvFilter.ts';
import { TonemapFilter } from '../../filter/TonemapFilter.ts';
import { OverlayWatermarkFilter } from '../../filter/watermark/OverlayWatermarkFilter.ts';
import { WatermarkOpacityFilter } from '../../filter/watermark/WatermarkOpacityFilter.ts';
import { WatermarkScaleFilter } from '../../filter/watermark/WatermarkScaleFilter.ts';
import { ColorFormat } from '../../format/ColorFormat.ts';
import {
PixelFormats,
PixelFormatYuv420P,
PixelFormatYuv420P10Le,
} from '../../format/PixelFormat.ts';
import { LavfiVideoInputSource } from '../../input/LavfiVideoInputSource.ts';
import { SubtitlesInputSource } from '../../input/SubtitlesInputSource.ts';
import { VideoInputSource } from '../../input/VideoInputSource.ts';
import { WatermarkInputSource } from '../../input/WatermarkInputSource.ts';
@@ -36,6 +39,7 @@ import {
StillImageStream,
SubtitleMethods,
VideoStream,
VideoStreamFields,
} from '../../MediaStream.ts';
import {
DefaultPipelineOptions,
@@ -46,6 +50,151 @@ import { FrameState } from '../../state/FrameState.ts';
import { FrameSize } from '../../types.ts';
import { QsvPipelineBuilder } from './QsvPipelineBuilder.ts';
// ─── Module-level constants ───────────────────────────────────────────────────
const ffmpegVersion = {
versionString: 'n7.0.2-15-g0458a86656-20240904',
majorVersion: 7,
minorVersion: 0,
patchVersion: 2,
isUnknown: false,
} as const;
const hdrColorFormat = new ColorFormat({
colorRange: ColorRanges.Tv,
colorSpace: ColorSpaces.Bt2020nc,
colorTransfer: ColorTransferFormats.Smpte2084,
colorPrimaries: ColorPrimaries.Bt2020,
});
// ─── Shared input factories ───────────────────────────────────────────────────
/**
* H264 1080p video input. Pass `sar: '1:1'` to force square pixels (no
* scaling), or leave it null to let the pipeline decide.
*/
function makeH264VideoInput(sar: string | null = null) {
return VideoInputSource.withStream(
new FileStreamSource('/path/to/video.mkv'),
VideoStream.create({
codec: 'h264',
profile: 'main',
displayAspectRatio: '16:9',
frameSize: FrameSize.FHD,
index: 0,
pixelFormat: new PixelFormatYuv420P(),
providedSampleAspectRatio: sar,
colorFormat: ColorFormat.unknown,
}),
);
}
/** HEVC 10-bit 1080p video input with HDR color metadata. */
function makeHevc10BitVideoInput() {
return VideoInputSource.withStream(
new FileStreamSource('/path/to/hdr-video.mkv'),
VideoStream.create({
codec: 'hevc',
displayAspectRatio: '16:9',
frameSize: FrameSize.FHD,
index: 0,
pixelFormat: new PixelFormatYuv420P10Le(),
providedSampleAspectRatio: null,
colorFormat: hdrColorFormat,
profile: 'main 10',
}),
);
}
function makeWatermarkSource(overrides: Partial<Watermark> = {}) {
return new WatermarkInputSource(
new FileStreamSource('/path/to/watermark.png'),
StillImageStream.create({
frameSize: FrameSize.withDimensions(100, 100),
index: 1,
}),
{
duration: 0,
enabled: true,
horizontalMargin: 5,
opacity: 100,
position: 'bottom-right',
verticalMargin: 5,
width: 10,
...overrides,
} satisfies Watermark,
);
}
/** FrameState targeting the input video at FHD with yuv420p output. */
function makeDesiredFrameState(video: VideoInputSource) {
return new FrameState({
isAnamorphic: false,
scaledSize: video.streams[0]!.squarePixelFrameSize(FrameSize.FHD),
paddedSize: FrameSize.FHD,
pixelFormat: new PixelFormatYuv420P(),
});
}
function makeHevcVideoInput(
fields?: Partial<StrictOmit<VideoStreamFields, 'codec'>>,
) {
return VideoInputSource.withStream(
new FileStreamSource('/path/to/video.mkv'),
VideoStream.create({
codec: VideoFormats.Hevc,
profile: 'main',
displayAspectRatio: '16:9',
frameSize: FrameSize.FHD,
index: 0,
pixelFormat: new PixelFormatYuv420P(),
// SAR 1:1 means non-anamorphic: squarePixelFrameSize(FHD) == FHD,
// so no scaling or padding is needed. The frame stays on hardware
// from the QSV decoder until the watermark path.
providedSampleAspectRatio: '1:1',
colorFormat: ColorFormat.unknown,
...fields,
}),
);
}
// H264 with both decode and encode capabilities — frame goes to hardware
const fullCapabilities = new VaapiHardwareCapabilities([
new VaapiProfileEntrypoint(VaapiProfiles.H264Main, VaapiEntrypoint.Decode),
new VaapiProfileEntrypoint(VaapiProfiles.H264Main, VaapiEntrypoint.Encode),
new VaapiProfileEntrypoint(VaapiProfiles.HevcMain, VaapiEntrypoint.Decode),
]);
function buildPipeline(opts: {
videoInput?: VideoInputSource;
watermark?: WatermarkInputSource | null;
capabilities?: VaapiHardwareCapabilities;
pipelineOptions?: Partial<PipelineOptions>;
}) {
const video = opts.videoInput ?? makeH264VideoInput();
const builder = new QsvPipelineBuilder(
opts.capabilities ?? fullCapabilities,
EmptyFfmpegCapabilities,
video,
null,
null,
opts.watermark ?? null,
null,
);
return builder.build(
FfmpegState.create({
version: { versionString: '7.1.1', isUnknown: false },
}),
new FrameState({
isAnamorphic: false,
scaledSize: video.streams[0]!.squarePixelFrameSize(FrameSize.FHD),
paddedSize: FrameSize.FHD,
pixelFormat: new PixelFormatYuv420P(),
}),
{ ...DefaultPipelineOptions, ...(opts.pipelineOptions ?? {}) },
);
}
describe('QsvPipelineBuilder', () => {
test('should work', () => {
const capabilities = new VaapiHardwareCapabilities([]);
@@ -361,21 +510,8 @@ describe('QsvPipelineBuilder', () => {
});
describe('tonemapping', () => {
const ffmpegVersion = {
versionString: 'n7.0.2-15-g0458a86656-20240904',
majorVersion: 7,
minorVersion: 0,
patchVersion: 2,
isUnknown: false,
} as const;
const hdrColorFormat = new ColorFormat({
colorRange: ColorRanges.Tv,
colorSpace: ColorSpaces.Bt2020nc,
colorTransfer: ColorTransferFormats.Smpte2084,
colorPrimaries: ColorPrimaries.Bt2020,
});
// Capabilities covering both H264 and HEVC 10-bit decode+encode —
// needed to test hardware tonemap paths.
const fullCapabilities = new VaapiHardwareCapabilities([
new VaapiProfileEntrypoint(
VaapiProfiles.H264Main,
@@ -399,47 +535,6 @@ describe('QsvPipelineBuilder', () => {
vi.unstubAllEnvs();
});
function makeH264VideoInput() {
return VideoInputSource.withStream(
new FileStreamSource('/path/to/video.mkv'),
VideoStream.create({
codec: 'h264',
profile: 'main',
displayAspectRatio: '16:9',
frameSize: FrameSize.FHD,
index: 0,
pixelFormat: new PixelFormatYuv420P(),
providedSampleAspectRatio: null,
colorFormat: ColorFormat.unknown,
}),
);
}
function makeHevc10BitVideoInput() {
return VideoInputSource.withStream(
new FileStreamSource('/path/to/hdr-video.mkv'),
VideoStream.create({
codec: 'hevc',
displayAspectRatio: '16:9',
frameSize: FrameSize.FHD,
index: 0,
pixelFormat: new PixelFormatYuv420P10Le(),
providedSampleAspectRatio: null,
colorFormat: hdrColorFormat,
profile: 'main 10',
}),
);
}
function makeDesiredFrameState(video: VideoInputSource) {
return new FrameState({
isAnamorphic: false,
scaledSize: video.streams[0]!.squarePixelFrameSize(FrameSize.FHD),
paddedSize: FrameSize.FHD,
pixelFormat: new PixelFormatYuv420P(),
});
}
test('does not apply tonemap when TUNARR_TONEMAP_ENABLED is not set', () => {
const video = makeH264VideoInput();
@@ -588,21 +683,6 @@ describe('QsvPipelineBuilder', () => {
});
describe('initial current state', () => {
const ffmpegVersion = {
versionString: 'n7.0.2-15-g0458a86656-20240904',
majorVersion: 7,
minorVersion: 0,
patchVersion: 2,
isUnknown: false,
} as const;
const hdrColorFormat = new ColorFormat({
colorRange: ColorRanges.Tv,
colorSpace: ColorSpaces.Bt2020nc,
colorTransfer: ColorTransferFormats.Smpte2084,
colorPrimaries: ColorPrimaries.Bt2020,
});
const emptyCapabilities = new VaapiHardwareCapabilities([]);
afterEach(() => {
@@ -610,18 +690,7 @@ describe('QsvPipelineBuilder', () => {
});
test('initializes with the input pixel format when it matches the desired format', () => {
const video = VideoInputSource.withStream(
new FileStreamSource('/path/to/video.mkv'),
VideoStream.create({
codec: 'h264',
displayAspectRatio: '16:9',
frameSize: FrameSize.FHD,
index: 0,
pixelFormat: new PixelFormatYuv420P(),
providedSampleAspectRatio: null,
colorFormat: ColorFormat.unknown,
}),
);
const video = makeH264VideoInput();
const builder = new QsvPipelineBuilder(
emptyCapabilities,
@@ -635,12 +704,7 @@ describe('QsvPipelineBuilder', () => {
const out = builder.build(
FfmpegState.create({ version: ffmpegVersion }),
new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.FHD,
paddedSize: FrameSize.FHD,
pixelFormat: new PixelFormatYuv420P(),
}),
makeDesiredFrameState(video),
DefaultPipelineOptions,
);
@@ -693,32 +757,22 @@ describe('QsvPipelineBuilder', () => {
},
);
// A QsvFormatFilter should be present because the initial currentState
// A PixelFormatFilter should be present because the initial currentState
// correctly reflects the 10-bit input pixel format (yuv420p10le), which
// differs from the desired 8-bit output (yuv420p).
// differs from the desired 8-bit output (yuv420p). Both HW decode and
// encode are disabled, so the frame stays on software and PixelFormatFilter
// (not QsvFormatFilter) is used for the conversion.
const pixelFormatFilterSteps =
out.getComplexFilter()?.filterChain.pixelFormatFilterSteps ?? [];
expect(
pixelFormatFilterSteps.some((s) => s instanceof QsvFormatFilter),
pixelFormatFilterSteps.some((s) => s instanceof PixelFormatFilter),
).toBe(true);
});
test('initializes with the input color format, used by software tonemap', () => {
vi.stubEnv(TUNARR_ENV_VARS.TONEMAP_ENABLED, 'true');
const video = VideoInputSource.withStream(
new FileStreamSource('/path/to/video.mkv'),
VideoStream.create({
codec: 'hevc',
profile: 'main 10',
displayAspectRatio: '16:9',
frameSize: FrameSize.FHD,
index: 0,
pixelFormat: new PixelFormatYuv420P10Le(),
providedSampleAspectRatio: null,
colorFormat: hdrColorFormat,
}),
);
const video = makeHevc10BitVideoInput();
const builder = new QsvPipelineBuilder(
emptyCapabilities,
@@ -732,12 +786,7 @@ describe('QsvPipelineBuilder', () => {
const out = builder.build(
FfmpegState.create({ version: ffmpegVersion }),
new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.FHD,
paddedSize: FrameSize.FHD,
pixelFormat: new PixelFormatYuv420P(),
}),
makeDesiredFrameState(video),
{
...DefaultPipelineOptions,
disableHardwareDecoding: true,
@@ -762,15 +811,8 @@ describe('QsvPipelineBuilder', () => {
});
describe('watermark', () => {
const ffmpegVersion = {
versionString: 'n7.0.2-15-g0458a86656-20240904',
majorVersion: 7,
minorVersion: 0,
patchVersion: 2,
isUnknown: false,
} as const;
// H264 with both decode and encode capabilities — frame goes to hardware
// H264-only capabilities: frame goes to hardware for decode and encode,
// which exercises the hwdownload→overlay→hwupload watermark path.
const fullCapabilities = new VaapiHardwareCapabilities([
new VaapiProfileEntrypoint(
VaapiProfiles.H264Main,
@@ -782,52 +824,15 @@ describe('QsvPipelineBuilder', () => {
),
]);
function makeH264VideoInput() {
return VideoInputSource.withStream(
new FileStreamSource('/path/to/video.mkv'),
VideoStream.create({
codec: 'h264',
profile: 'main',
displayAspectRatio: '16:9',
frameSize: FrameSize.FHD,
index: 0,
pixelFormat: new PixelFormatYuv420P(),
// SAR 1:1 means non-anamorphic: squarePixelFrameSize(FHD) == FHD,
// so no scaling or padding is needed. The frame stays on hardware
// from the QSV decoder until the watermark path.
providedSampleAspectRatio: '1:1',
colorFormat: ColorFormat.unknown,
}),
);
}
function makeWatermarkSource(overrides: Partial<Watermark> = {}) {
return new WatermarkInputSource(
new FileStreamSource('/path/to/watermark.png'),
StillImageStream.create({
frameSize: FrameSize.withDimensions(100, 100),
index: 1,
}),
{
duration: 0,
enabled: true,
horizontalMargin: 5,
opacity: 100,
position: 'bottom-right',
verticalMargin: 5,
width: 10,
...overrides,
} satisfies Watermark,
);
}
function buildPipeline(opts: {
videoInput?: VideoInputSource;
watermark?: WatermarkInputSource | null;
capabilities?: VaapiHardwareCapabilities;
pipelineOptions?: Partial<PipelineOptions>;
}) {
const video = opts.videoInput ?? makeH264VideoInput();
// SAR 1:1 → squarePixelFrameSize(FHD) == FHD, so no scaling/padding is
// needed and the frame stays on hardware from QSV decode until watermark.
const video = opts.videoInput ?? makeH264VideoInput('1:1');
const builder = new QsvPipelineBuilder(
opts.capabilities ?? fullCapabilities,
EmptyFfmpegCapabilities,
@@ -839,12 +844,7 @@ describe('QsvPipelineBuilder', () => {
);
return builder.build(
FfmpegState.create({ version: ffmpegVersion }),
new FrameState({
isAnamorphic: false,
scaledSize: video.streams[0]!.squarePixelFrameSize(FrameSize.FHD),
paddedSize: FrameSize.FHD,
pixelFormat: new PixelFormatYuv420P(),
}),
makeDesiredFrameState(video),
{ ...DefaultPipelineOptions, ...(opts.pipelineOptions ?? {}) },
);
}
@@ -1023,6 +1023,177 @@ describe('QsvPipelineBuilder', () => {
});
});
describe('pixel format fixes', () => {
test('no format=unknown for error screens (Fix 1)', () => {
const errorScreen = LavfiVideoInputSource.errorText(
FrameSize.FHD,
'Error',
'Subtitle',
);
const builder = new QsvPipelineBuilder(
fullCapabilities,
EmptyFfmpegCapabilities,
errorScreen,
null,
null,
null,
null,
);
const pipeline = builder.build(
FfmpegState.create({
version: { versionString: '7.1.1', isUnknown: false },
}),
new FrameState({
isAnamorphic: false,
scaledSize: FrameSize.FHD,
paddedSize: FrameSize.FHD,
pixelFormat: new PixelFormatYuv420P(),
}),
DefaultPipelineOptions,
);
const args = pipeline.getCommandArgs().join(' ');
expect(args, args).not.toContain('format=unknown');
});
test('no -pix_fmt with QSV encode (Fix 2)', () => {
const pipeline = buildPipeline({});
const args = pipeline.getCommandArgs().join(' ');
expect(args, args).not.toContain('-pix_fmt');
});
test('no -pix_fmt for QSV encode without scaling (Fix 2)', () => {
// H264 FHD input → FHD output: no scaling or padding, so no QSV scale filter.
// Still must not emit -pix_fmt because the encoder is QSV.
const pipeline = buildPipeline({
pipelineOptions: { disableHardwareDecoding: true },
});
const args = pipeline.getCommandArgs().join(' ');
expect(args, args).not.toContain('-pix_fmt');
});
test('format=nv12 inserted before hwupload after watermark overlay (Fix 3)', () => {
// Software decode leaves frame in yuv420p. After the watermark overlay,
// hwupload needs the frame in nv12 format; without Fix 3 the conversion
// filter was absent, causing format negotiation failures.
const pipeline = buildPipeline({
watermark: makeWatermarkSource(),
pipelineOptions: { disableHardwareDecoding: true },
});
const pixelFormatFilterSteps =
pipeline.getComplexFilter()!.filterChain.pixelFormatFilterSteps;
const hwUploadIdx = pixelFormatFilterSteps.findIndex(
(s) => s instanceof HardwareUploadQsvFilter,
);
expect(
hwUploadIdx,
'HardwareUploadQsvFilter should be present',
).toBeGreaterThan(-1);
const fmtFilterIdx = pixelFormatFilterSteps.findIndex(
(s) => s instanceof PixelFormatFilter,
);
expect(
fmtFilterIdx,
'PixelFormatFilter should be present before hwupload',
).toBeGreaterThan(-1);
expect(fmtFilterIdx).toBeLessThan(hwUploadIdx);
const fmtFilter = pixelFormatFilterSteps[
fmtFilterIdx
] as PixelFormatFilter;
expect(fmtFilter.filter).toBe(`format=${PixelFormats.NV12}`);
const args = pipeline.getCommandArgs().join(' ');
const nv12Idx = args.indexOf(`format=${PixelFormats.NV12}`);
const hwuploadIdx = args.indexOf('hwupload');
expect(nv12Idx).toBeGreaterThan(-1);
expect(nv12Idx).toBeLessThan(hwuploadIdx);
});
test('format=p010le inserted before hwupload for 10-bit input + watermark (Fix 3)', () => {
// 10-bit HEVC: after watermark overlay the frame is in yuv420p10le on software.
// hwupload for QSV requires p010le; without Fix 3 the format conversion was missing.
const pipeline = buildPipeline({
videoInput: makeHevcVideoInput({
pixelFormat: new PixelFormatYuv420P10Le(),
}),
watermark: makeWatermarkSource(),
pipelineOptions: { disableHardwareDecoding: true },
});
const pixelFormatFilterSteps =
pipeline.getComplexFilter()!.filterChain.pixelFormatFilterSteps;
const hwUploadIdx = pixelFormatFilterSteps.findIndex(
(s) => s instanceof HardwareUploadQsvFilter,
);
expect(
hwUploadIdx,
'HardwareUploadQsvFilter should be present',
).toBeGreaterThan(-1);
// The last PixelFormatFilter before hwupload should be format=p010le
const fmtFiltersBeforeUpload = pixelFormatFilterSteps
.slice(0, hwUploadIdx)
.filter((s) => s instanceof PixelFormatFilter);
const lastFmtFilter = fmtFiltersBeforeUpload.at(-1) as
| PixelFormatFilter
| undefined;
expect(lastFmtFilter).toBeDefined();
expect(lastFmtFilter!.filter).toBe(`format=${PixelFormats.P010}`);
const args = pipeline.getCommandArgs().join(' ');
const p010Idx = args.indexOf(`format=${PixelFormats.P010}`);
const hwuploadIdx = args.indexOf('hwupload');
expect(p010Idx).toBeGreaterThan(-1);
expect(p010Idx).toBeLessThan(hwuploadIdx);
});
test('software-only pipeline still emits -pix_fmt when formats differ (regression guard for Fix 2)', () => {
// With no hardware capabilities (software decode + encode) and a 10-bit HEVC
// input transcoded to 8-bit yuv420p, the code must still emit -pix_fmt yuv420p
// via the unconditional path (encoder is not QSV). Fix 2 only suppresses
// -pix_fmt for QSV encoders.
const noCapabilities = new VaapiHardwareCapabilities([]);
const hevc10bitInput = makeHevcVideoInput({
pixelFormat: new PixelFormatYuv420P10Le(),
});
const builder = new QsvPipelineBuilder(
noCapabilities,
EmptyFfmpegCapabilities,
hevc10bitInput,
null,
null,
null,
null,
);
const pipeline = builder.build(
FfmpegState.create({
version: { versionString: '7.1.1', isUnknown: false },
}),
new FrameState({
isAnamorphic: false,
scaledSize: hevc10bitInput.streams[0]!.squarePixelFrameSize(
FrameSize.FHD,
),
paddedSize: FrameSize.FHD,
pixelFormat: new PixelFormatYuv420P(),
}),
DefaultPipelineOptions,
);
const args = pipeline.getCommandArgs().join(' ');
expect(args, args).toContain('-pix_fmt yuv420p');
});
});
test('hwdownload bug', async () => {
const wm = new WatermarkInputSource(
new FileStreamSource('/path/to/img'),
@@ -1150,6 +1321,95 @@ describe('QsvPipelineBuilder', () => {
disableHardwareFilters: false,
vaapiDevice: null,
vaapiDriver: null,
vaapiPipelineOptions: null,
},
);
console.log(x.getCommandArgs().join(' '));
});
test('10-bit input, 8-bit output', async () => {
const builder = new QsvPipelineBuilder(
new VaapiHardwareCapabilities([
new VaapiProfileEntrypoint(
VaapiProfiles.H264Main,
VaapiEntrypoint.Decode,
),
new VaapiProfileEntrypoint(
VaapiProfiles.H264Main,
VaapiEntrypoint.Encode,
),
new VaapiProfileEntrypoint(
VaapiProfiles.HevcMain,
VaapiEntrypoint.Decode,
),
]),
EmptyFfmpegCapabilities,
makeHevcVideoInput({
frameSize: FrameSize.FHD,
pixelFormat: new PixelFormatYuv420P10Le(),
}),
null,
null,
null,
null,
);
const x = builder.build(
FfmpegState.create({
version: {
versionString: 'n7.1.1-56-gc2184b65d2-20250716',
majorVersion: 7,
minorVersion: 1,
patchVersion: 1,
versionDetails: '56-gc2184b65d2-20250716',
isUnknown: false,
},
threadCount: 0,
start: dayjs.duration({ minutes: 5, seconds: 19.253 }),
duration: dayjs.duration({ minutes: 18, seconds: 2.348 }),
logLevel: 'debug',
mapMetadata: false,
metadataServiceName: null,
metadataServiceProvider: null,
decoderHwAccelMode: 'none',
encoderHwAccelMode: 'none',
softwareScalingAlgorithm: 'bicubic',
softwareDeinterlaceFilter: 'none',
vaapiDevice: null,
vaapiDriver: null,
outputLocation: 'stdout',
ptsOffset: 0,
tonemapHdr: false,
}),
new FrameState({
scaledSize: FrameSize.FHD,
paddedSize: FrameSize.FHD,
isAnamorphic: false,
realtime: false,
videoFormat: 'h264',
videoPreset: null,
videoProfile: null,
frameRate: null,
videoTrackTimescale: 90000,
videoBitrate: 10000,
videoBufferSize: 20000,
frameDataLocation: 'unknown',
deinterlace: false,
pixelFormat: new PixelFormatYuv420P(),
colorFormat: ColorFormat.bt709,
infiniteLoop: false,
forceSoftwareOverlay: false,
}),
{
decoderThreadCount: 0,
encoderThreadCount: 0,
filterThreadCount: null,
disableHardwareDecoding: false,
disableHardwareEncoding: false,
disableHardwareFilters: false,
vaapiDevice: null,
vaapiDriver: null,
vaapiPipelineOptions: null,
},
);
console.log(x.getCommandArgs().join(' '));

181
server/src/ffmpeg/builder/pipeline/hardware/QsvPipelineBuilder.ts
View File

@@ -4,7 +4,7 @@ import type { FfmpegCapabilities } from '@/ffmpeg/builder/capabilities/FfmpegCap
import { OutputFormatTypes, VideoFormats } from '@/ffmpeg/builder/constants.js';
import type { Decoder } from '@/ffmpeg/builder/decoder/Decoder.js';
import { DecoderFactory } from '@/ffmpeg/builder/decoder/DecoderFactory.js';
import { Encoder } from '@/ffmpeg/builder/encoder/Encoder.js';
import type { Encoder } from '@/ffmpeg/builder/encoder/Encoder.js';
import { DeinterlaceFilter } from '@/ffmpeg/builder/filter/DeinterlaceFilter.js';
import type { FilterOption } from '@/ffmpeg/builder/filter/FilterOption.js';
import { HardwareDownloadFilter } from '@/ffmpeg/builder/filter/HardwareDownloadFilter.js';
@@ -21,6 +21,7 @@ import {
PixelFormatNv12,
PixelFormatP010,
PixelFormats,
PixelFormatYuv420P,
PixelFormatYuv420P10Le,
PixelFormatYuva420P,
} from '@/ffmpeg/builder/format/PixelFormat.js';
@@ -164,29 +165,31 @@ export class QsvPipelineBuilder extends SoftwarePipelineBuilder {
currentState = this.decoder.nextState(currentState);
}
currentState = this.addFilterToVideoChain(
currentState,
new ResetPtsFilter(),
);
const setFrameRate =
this.context?.videoStream.getNumericFrameRateOrDefault() ?? 24;
currentState = this.addFilterToVideoChain(
currentState,
new SetFpsFilter(setFrameRate),
);
// Remove existing frame rate output option if the framerate we just
// set differs from the
if (
this.desiredState.frameRate &&
this.desiredState.frameRate !== setFrameRate
) {
const idx = this.pipelineSteps.findIndex(
(step) => step instanceof FrameRateOutputOption,
if (this.desiredState.videoFormat !== VideoFormats.Copy) {
currentState = this.addFilterToVideoChain(
currentState,
new ResetPtsFilter(),
);
if (idx !== -1) {
this.pipelineSteps.splice(idx, 1);
const setFrameRate =
this.context?.videoStream.getNumericFrameRateOrDefault() ?? 24;
currentState = this.addFilterToVideoChain(
currentState,
new SetFpsFilter(setFrameRate),
);
// Remove existing frame rate output option if the framerate we just
// set differs from the
if (
this.desiredState.frameRate &&
this.desiredState.frameRate !== setFrameRate
) {
const idx = this.pipelineSteps.findIndex(
(step) => step instanceof FrameRateOutputOption,
);
if (idx !== -1) {
this.pipelineSteps.splice(idx, 1);
}
}
}
@@ -334,70 +337,80 @@ export class QsvPipelineBuilder extends SoftwarePipelineBuilder {
step instanceof DeinterlaceQsvFilter,
);
const currentPixelFormat = currentState.pixelFormat;
let currentPixelFormat = currentState.pixelFormat;
if (
some(
this.videoInputSource.filterSteps,
(step) => !(step instanceof Encoder),
) &&
currentPixelFormat
) {
if (currentPixelFormat && currentPixelFormat.isUnknown()) {
const resolved =
currentPixelFormat.bitDepth === 10
? new PixelFormatP010()
: new PixelFormatNv12(new PixelFormatYuv420P());
currentState = currentState.update({ pixelFormat: resolved });
currentPixelFormat = resolved;
}
if (currentPixelFormat) {
let needsConversion = false;
if (currentPixelFormat.name === PixelFormats.NV12) {
needsConversion =
currentPixelFormat.unwrap().name !== targetPixelFormat.name;
if (!needsConversion) {
currentState = currentState.update({
pixelFormat: targetPixelFormat,
});
}
} else {
needsConversion = currentPixelFormat.name !== targetPixelFormat.name;
const unwrappedCurrent =
currentPixelFormat.toSoftwareFormat() ?? currentPixelFormat;
needsConversion = unwrappedCurrent.name !== targetPixelFormat.name;
if (!needsConversion) {
currentState = currentState.update({
pixelFormat: targetPixelFormat,
});
}
if (needsConversion) {
const filter = new QsvFormatFilter(currentPixelFormat);
const filterCtor =
currentState.frameDataLocation === FrameDataLocation.Hardware
? QsvFormatFilter
: PixelFormatFilter;
hasQsvFilter =
currentState.frameDataLocation === FrameDataLocation.Hardware;
const filter = new filterCtor(currentPixelFormat);
steps.push(filter);
currentState = filter.nextState(currentState);
if (currentPixelFormat.bitDepth === 8 && this.context.is10BitOutput) {
const tenbitFilter = new QsvFormatFilter(new PixelFormatP010());
const tenbitFilter = new filterCtor(new PixelFormatP010());
steps.push(tenbitFilter);
currentState = tenbitFilter.nextState(currentState);
}
hasQsvFilter = true;
}
}
if (hasQsvFilter) {
if (currentState.frameDataLocation === FrameDataLocation.Hardware) {
if (
currentState.pixelFormat?.bitDepth === 10 &&
pixelFormatToDownload?.name !== PixelFormats.YUV420P10LE
) {
pixelFormatToDownload = new PixelFormatYuv420P10Le();
currentState = currentState.update({
pixelFormat: pixelFormatToDownload,
});
} else if (
currentState.pixelFormat?.bitDepth === 8 &&
pixelFormatToDownload?.name !== PixelFormats.NV12
) {
pixelFormatToDownload = new PixelFormatNv12(pixelFormatToDownload);
currentState = currentState.update({
pixelFormat: pixelFormatToDownload,
});
}
// hasQsvFilter implies we're on hardware, but check anyway.
if (
hasQsvFilter &&
currentState.frameDataLocation === FrameDataLocation.Hardware
) {
if (
currentState.pixelFormat?.bitDepth === 10 &&
pixelFormatToDownload?.name !== PixelFormats.P010
) {
pixelFormatToDownload = new PixelFormatP010();
currentState = currentState.update({
pixelFormat: pixelFormatToDownload,
});
} else if (
currentState.pixelFormat?.bitDepth === 8 &&
pixelFormatToDownload?.name !== PixelFormats.NV12
) {
pixelFormatToDownload = new PixelFormatNv12(pixelFormatToDownload);
currentState = currentState.update({
pixelFormat: pixelFormatToDownload,
});
}
}
// If we're about to encode with software and we're in hardware,
// we'll need to download. We shouldn't have to do any more conversions
// at this point
if (
this.ffmpegState.encoderHwAccelMode === HardwareAccelerationMode.None &&
currentState.frameDataLocation === FrameDataLocation.Hardware
) {
pixelFormatToDownload = new PixelFormatNv12(pixelFormatToDownload);
// pixelFormatToDownload = new PixelFormatNv12(pixelFormatToDownload);
const hwDownloadFilter = new HardwareDownloadFilter(
currentState.update({ pixelFormat: pixelFormatToDownload }),
);
@@ -405,18 +418,47 @@ export class QsvPipelineBuilder extends SoftwarePipelineBuilder {
steps.push(hwDownloadFilter);
}
// If we're going to encode on hardware, but we're still in software,
// perform the final upload.
if (
this.ffmpegState.encoderHwAccelMode === HardwareAccelerationMode.Qsv &&
currentState.frameDataLocation === FrameDataLocation.Software
) {
const hwCompatFormat =
currentState.pixelFormat?.bitDepth === 10
? new PixelFormatP010()
: new PixelFormatNv12(new PixelFormatYuv420P());
if (currentState.pixelFormat?.name !== hwCompatFormat.name) {
const fmtFilter = new PixelFormatFilter(hwCompatFormat);
steps.push(fmtFilter);
currentState = fmtFilter.nextState(currentState);
}
steps.push(new HardwareUploadQsvFilter(64));
}
if (currentState.pixelFormat?.name !== targetPixelFormat.name) {
// Only emit -pix_fmt for software encoders; QSV encoders don't accept
// a -pix_fmt flag and it causes swscaler errors with hardware frames.
if (
currentState.pixelFormat?.name !== targetPixelFormat.name &&
this.ffmpegState.encoderHwAccelMode !== HardwareAccelerationMode.Qsv
) {
// TODO: Handle color params
this.pipelineSteps.push(new PixelFormatOutputOption(targetPixelFormat));
}
this.context.filterChain.pixelFormatFilterSteps = steps;
} else if (
this.ffmpegState.encoderHwAccelMode === HardwareAccelerationMode.Qsv &&
currentState.frameDataLocation === FrameDataLocation.Software
) {
// No explicit pixel format was requested but QSV needs hardware frames.
// This happens after a watermark overlay (which outputs software yuv420p).
const hwCompatFormat =
currentState.pixelFormat?.bitDepth === 10
? new PixelFormatP010()
: new PixelFormatNv12(new PixelFormatYuv420P());
steps.push(new PixelFormatFilter(hwCompatFormat));
steps.push(new HardwareUploadQsvFilter(64));
this.context.filterChain.pixelFormatFilterSteps = steps;
}
return currentState;
@@ -472,9 +514,10 @@ export class QsvPipelineBuilder extends SoftwarePipelineBuilder {
// Fades
}
if (this.desiredState.pixelFormat) {
const pf = this.desiredState.pixelFormat.unwrap();
const pf = (
this.desiredState.pixelFormat ?? currentState.pixelFormat
)?.unwrap();
if (pf && !pf.isUnknown()) {
// Overlay
this.context.filterChain.watermarkOverlayFilterSteps.push(
new OverlayWatermarkFilter(

BIN
server/src/testing/ffmpeg/fixtures/black_watermark.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 130 B