From 56de88057e7d085669b22e3548369a7b63a98c1a Mon Sep 17 00:00:00 2001 From: Simon Kirsten Date: Sun, 9 Aug 2026 17:39:06 +0200 Subject: [PATCH] feat: make I-frame max QP and encode quality level configurable EncodeConfig grows max_qp_i, capping the QP of I/IDR frames under rate control (H.264/H.265), and encode_quality_level, the Vulkan video encode quality level applied to the session parameters and the first-frame control command of all codecs. Both default to the previous hardcoded behavior. Capping the I-frame QP keeps the first GOP from visibly banding while the driver's rate control is still adapting to the content: the initial QP estimate on a cold encoder can land well above what a static, dark scene needs, and skip-coded P-frames then carry the banded keyframe until the next IDR. --- src/encoder/av1/record.rs | 4 ++-- src/encoder/av1/session_params.rs | 3 ++- src/encoder/h264/record.rs | 14 +++++++++++--- src/encoder/h264/session_params.rs | 3 ++- src/encoder/h265/record.rs | 14 +++++++++++--- src/encoder/h265/session_params.rs | 3 ++- src/encoder/mod.rs | 30 ++++++++++++++++++++++++++++++ 7 files changed, 60 insertions(+), 11 deletions(-) diff --git a/src/encoder/av1/record.rs b/src/encoder/av1/record.rs index 622a3d5..883c45f 100644 --- a/src/encoder/av1/record.rs +++ b/src/encoder/av1/record.rs @@ -358,8 +358,8 @@ impl Av1 { } if is_first_frame { - let mut quality_level_info = - vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); + let mut quality_level_info = vk::VideoEncodeQualityLevelInfoKHR::default() + .quality_level(common.config.encode_quality_level); let control_info = vk::VideoCodingControlInfoKHR::default() .flags( vk::VideoCodingControlFlagsKHR::RESET diff --git a/src/encoder/av1/session_params.rs b/src/encoder/av1/session_params.rs index 2e1448b..07c9407 100644 --- a/src/encoder/av1/session_params.rs +++ b/src/encoder/av1/session_params.rs @@ -151,7 +151,8 @@ impl Av1 { .std_decoder_model_info(&decoder_model_info) .std_operating_points(std::slice::from_ref(&operating_point)); - let mut quality_info = vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); + let mut quality_info = vk::VideoEncodeQualityLevelInfoKHR::default() + .quality_level(common.config.encode_quality_level); let session_params_create_info = vk::VideoSessionParametersCreateInfoKHR::default() .video_session(common.session) .push(&mut quality_info) diff --git a/src/encoder/h264/record.rs b/src/encoder/h264/record.rs index 04b00df..1340b43 100644 --- a/src/encoder/h264/record.rs +++ b/src/encoder/h264/record.rs @@ -431,13 +431,21 @@ impl H264 { // Rate control. let qp_bounds = if rc.is_disabled() { rc.qp as i32 } else { 18 }; let qp_bounds_max = if rc.is_disabled() { rc.qp as i32 } else { 42 }; + let qp_bounds_max_i = if rc.is_disabled() { + rc.qp as i32 + } else { + common + .config + .max_qp_i + .map_or(qp_bounds_max, |qp| (qp as i32).max(qp_bounds)) + }; let min_qp = vk::VideoEncodeH264QpKHR { qp_i: qp_bounds, qp_p: qp_bounds, qp_b: qp_bounds, }; let max_qp = vk::VideoEncodeH264QpKHR { - qp_i: qp_bounds_max, + qp_i: qp_bounds_max_i, qp_p: qp_bounds_max, qp_b: qp_bounds_max, }; @@ -496,8 +504,8 @@ impl H264 { // RESET + RATE_CONTROL + QUALITY_LEVEL in one control command on the first // frame (matches FFmpeg; required for AMD RADV). if is_first_frame { - let mut quality_level_info = - vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); + let mut quality_level_info = vk::VideoEncodeQualityLevelInfoKHR::default() + .quality_level(common.config.encode_quality_level); let control_info = vk::VideoCodingControlInfoKHR::default() .flags( vk::VideoCodingControlFlagsKHR::RESET diff --git a/src/encoder/h264/session_params.rs b/src/encoder/h264/session_params.rs index d20ccd7..ac9cbc0 100644 --- a/src/encoder/h264/session_params.rs +++ b/src/encoder/h264/session_params.rs @@ -174,7 +174,8 @@ impl H264 { .parameters_add_info(&h264_add_info); // Chain quality level info (required by AMD RADV; matches FFmpeg). - let mut quality_level_info = vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); + let mut quality_level_info = vk::VideoEncodeQualityLevelInfoKHR::default() + .quality_level(config.encode_quality_level); let params_create_info = vk::VideoSessionParametersCreateInfoKHR::default() .video_session(common.session) diff --git a/src/encoder/h265/record.rs b/src/encoder/h265/record.rs index 70d50c7..445161a 100644 --- a/src/encoder/h265/record.rs +++ b/src/encoder/h265/record.rs @@ -406,13 +406,21 @@ impl H265 { // Rate control. let qp_min = if rc.is_disabled() { rc.qp as i32 } else { 26 }; let qp_max = if rc.is_disabled() { rc.qp as i32 } else { 51 }; + let qp_max_i = if rc.is_disabled() { + rc.qp as i32 + } else { + common + .config + .max_qp_i + .map_or(qp_max, |qp| (qp as i32).max(qp_min)) + }; let min_qp = vk::VideoEncodeH265QpKHR { qp_i: qp_min, qp_p: qp_min, qp_b: qp_min, }; let max_qp = vk::VideoEncodeH265QpKHR { - qp_i: qp_max, + qp_i: qp_max_i, qp_p: qp_max, qp_b: qp_max, }; @@ -466,8 +474,8 @@ impl H265 { } if is_first_frame { - let mut quality_level_info = - vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); + let mut quality_level_info = vk::VideoEncodeQualityLevelInfoKHR::default() + .quality_level(common.config.encode_quality_level); let control_info = vk::VideoCodingControlInfoKHR::default() .flags( vk::VideoCodingControlFlagsKHR::RESET diff --git a/src/encoder/h265/session_params.rs b/src/encoder/h265/session_params.rs index 4deca2f..a7f9c38 100644 --- a/src/encoder/h265/session_params.rs +++ b/src/encoder/h265/session_params.rs @@ -333,7 +333,8 @@ impl H265 { .max_std_pps_count(1) .parameters_add_info(&h265_add_info); - let mut quality_level_info = vk::VideoEncodeQualityLevelInfoKHR::default().quality_level(0); + let mut quality_level_info = vk::VideoEncodeQualityLevelInfoKHR::default() + .quality_level(common.config.encode_quality_level); let params_create_info = vk::VideoSessionParametersCreateInfoKHR::default() .video_session(common.session) diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index c4a4497..6376fa0 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -329,6 +329,18 @@ pub struct EncodeConfig { /// P-frames. Setting it equal to `virtual_buffer_size_ms` gives /// IDR frames maximum headroom. pub initial_virtual_buffer_size_ms: u32, + /// Maximum QP for I/IDR frames in rate-controlled modes. + /// `None` uses the codec default (42 for H.264, 51 for H.265). + /// Capping it below the default forces keyframes to spend enough bits + /// even before rate control has adapted to the content, at the cost of + /// larger keyframes. Values below the codec's minimum QP bound are + /// raised to it. Ignored by AV1 and by CQP/disabled rate control. + pub max_qp_i: Option, + /// Vulkan video encode quality level + /// (`VkVideoEncodeQualityLevelInfoKHR`). Higher levels trade encoding + /// speed for quality. Must be below the device's `maxQualityLevels` + /// for the video profile. + pub encode_quality_level: u32, /// Color description for VUI signaling. /// Defaults to BT.709 (full-range) when `None`. pub color_description: Option, @@ -362,6 +374,8 @@ impl EncodeConfig { max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, + max_qp_i: None, + encode_quality_level: 0, color_description: None, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, @@ -390,6 +404,8 @@ impl EncodeConfig { max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, + max_qp_i: None, + encode_quality_level: 0, color_description: None, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, @@ -418,6 +434,8 @@ impl EncodeConfig { max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, + max_qp_i: None, + encode_quality_level: 0, color_description: None, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, @@ -501,6 +519,18 @@ impl EncodeConfig { self } + /// Set the maximum QP for I/IDR frames in rate-controlled modes. + pub fn with_max_qp_i(mut self, qp: u32) -> Self { + self.max_qp_i = Some(qp); + self + } + + /// Set the Vulkan video encode quality level. + pub fn with_encode_quality_level(mut self, level: u32) -> Self { + self.encode_quality_level = level; + self + } + /// Set the color description for VUI signaling. pub fn with_color_description(mut self, desc: ColorDescription) -> Self { self.color_description = Some(desc);