From a78b021111583364e0247e1a440b5e2206e00d98 Mon Sep 17 00:00:00 2001 From: Tymur Boiko Date: Mon, 20 Apr 2026 05:39:08 +0000 Subject: [PATCH 1/5] encode: fix regression in RC struct chaining MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 62c184cc changed line 1807 in VkVideoEncoder.cpp from ((VkBaseInStructure*)(m_beginRateControlInfo.pNext))->pNext = NULL; to: m_beginRateControlInfo.pNext = (VkBaseInStructure*)(&encodeFrameInfo->pControlCmdChain); , introducing undefined behaviour because &encodeFrameInfo->pControlCmdChain is the address of the pointer member itself —- a VkBaseInStructure** and not a Vulkan structure. Commit 1c603c14 fixed the undefined behaviour by setting the RHS to NULL but went one step too far, because that also ended up removing any chained rate control structures. This change fully reverts 1c603c14 and restores the contents on line 1807 to its pre-62c184cc form, albeit written a bit differently to fix the following warning from GCC: cast from type ‘const void*’ to type ‘VkBaseInStructure*’ casts away qualifiers [-Wcast-qual] Signed-off-by: Tymur Boiko --- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp index 93209af9..e935393b 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp @@ -1916,9 +1916,10 @@ VkResult VkVideoEncoder::RecordVideoCodingCmd(VkSharedBaseObjcontrolCmd}; vkDevCtx->CmdControlVideoCodingKHR(cmdBuf, &renderControlInfo); + assert(encodeFrameInfo->pControlCmdChain); + m_beginRateControlInfo = *(VkVideoEncodeRateControlInfoKHR*)encodeFrameInfo->pControlCmdChain; - // Do not walk the chain, otherwise we end up creating a loop here. - m_beginRateControlInfo.pNext = NULL; + const_cast(static_cast(m_beginRateControlInfo.pNext))->pNext = NULL; } if (m_videoMaintenance1FeaturesSupported) From 10599bf8c5a7e3fe868d39df0ceb311fe6ad9be2 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Mon, 20 Apr 2026 06:29:10 +0000 Subject: [PATCH 2/5] encode: simplify and rename VkVideoEncodeFrameInfo::GetType() This method currently assumes that the first element in the pNext chain of VkVideoEncodeInfoKHR will be a codec-specific structure for the picture info (e.g. VkVideoEncodeH264PictureInfoKHR). This assumption is problematic because: - it complicates the pNext chaining logic for other places in the code which need to insert one or more structures into the pNext chain for VkVideoEncodeInfoKHR. - making assumptions within the application about the order of structures in the pNext chain, although not prohibited by the Vulkan specification, may lead to similar assumptions elsewhere and complicate code writing and comprehension. This method aids VkVideoEncoderH264::GetEncodeFrameInfoH264() and similar methods in the other codec-specific classes in checking that the pointer to the base class points in fact to the actual codec-specific data structure needed, instead of relying on runtime type information (RTTI). This purpose is now fulfilled by changing GetType() into a virtual method which returns the codec operation, and which is overridden by derived classes. Additionally, GetType() has been renamed to GetCodecType() for clarity. Signed-off-by: Srinath Kumarapuram --- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h | 5 +---- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h | 6 +++++- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h | 6 +++++- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h | 6 +++++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h index dacc2929..c604df49 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h @@ -56,10 +56,7 @@ class VkVideoEncoder : public VkVideoRefCountBase { struct VkVideoEncodeFrameInfo : public VkVideoRefCountBase { - VkStructureType GetType() { - return (encodeInfo.pNext == nullptr) ? - VK_STRUCTURE_TYPE_VIDEO_ENCODE_INFO_KHR : reinterpret_cast(encodeInfo.pNext)->sType; - } + virtual VkVideoCodecOperationFlagBitsKHR GetCodecType() const = 0; VkVideoEncodeFrameInfo(const void* pNext = nullptr) : encodeInfo{ VK_STRUCTURE_TYPE_VIDEO_ENCODE_INFO_KHR, pNext} diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h index 968d83f4..2fc4566c 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h @@ -48,6 +48,10 @@ class VkVideoEncoderAV1 : public VkVideoEncoder VkVideoEncodeAV1RateControlInfoKHR rateControlInfoAV1; VkVideoEncodeAV1RateControlLayerInfoKHR rateControlLayersInfoAV1[1]; + VkVideoCodecOperationFlagBitsKHR GetCodecType() const override { + return VK_VIDEO_CODEC_OPERATION_ENCODE_AV1_BIT_KHR; + } + VkVideoEncodeFrameInfoAV1() : VkVideoEncodeFrameInfo(&pictureInfo) , pictureInfo{ VK_STRUCTURE_TYPE_VIDEO_ENCODE_AV1_PICTURE_INFO_KHR } @@ -184,7 +188,7 @@ class VkVideoEncoderAV1 : public VkVideoEncoder private: VkVideoEncodeFrameInfoAV1* GetEncodeFrameInfoAV1(VkSharedBaseObj& encodeFrameInfo) { - assert(VK_STRUCTURE_TYPE_VIDEO_ENCODE_AV1_PICTURE_INFO_KHR == encodeFrameInfo->GetType()); + assert(VK_VIDEO_CODEC_OPERATION_ENCODE_AV1_BIT_KHR == encodeFrameInfo->GetCodecType()); VkVideoEncodeFrameInfo* pEncodeFrameInfo = encodeFrameInfo; return (VkVideoEncodeFrameInfoAV1*)pEncodeFrameInfo; } diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h index 0eb79335..f5f7b4ce 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h @@ -48,6 +48,10 @@ class VkVideoEncoderH264 : public VkVideoEncoder { StdVideoEncodeH264RefListModEntry refList1ModOperations[MAX_REFFERENCES]; StdVideoEncodeH264RefPicMarkingEntry refPicMarkingEntry[MAX_MEM_MGMNT_CTRL_OPS_COMMANDS]; + VkVideoCodecOperationFlagBitsKHR GetCodecType() const override { + return VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR; + } + VkVideoEncodeFrameInfoH264() : VkVideoEncodeFrameInfo(&pictureInfo) , pictureInfo { VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_PICTURE_INFO_KHR } @@ -153,7 +157,7 @@ class VkVideoEncoderH264 : public VkVideoEncoder { private: VkVideoEncodeFrameInfoH264* GetEncodeFrameInfoH264(VkSharedBaseObj& encodeFrameInfo) { - assert(VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_PICTURE_INFO_KHR == encodeFrameInfo->GetType()); + assert(VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR == encodeFrameInfo->GetCodecType()); VkVideoEncodeFrameInfo* pEncodeFrameInfo = encodeFrameInfo; return (VkVideoEncodeFrameInfoH264*)pEncodeFrameInfo; } diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h index 6d829250..162d23e5 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h @@ -46,6 +46,10 @@ class VkVideoEncoderH265 : public VkVideoEncoder { StdVideoEncodeH265ReferenceInfo stdReferenceInfo[MAX_REFFERENCES]; VkVideoEncodeH265DpbSlotInfoKHR stdDpbSlotInfo[MAX_REFFERENCES]; + VkVideoCodecOperationFlagBitsKHR GetCodecType() const override { + return VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR; + } + VkVideoEncodeFrameInfoH265() : VkVideoEncodeFrameInfo(&pictureInfo) , pictureInfo { VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_PICTURE_INFO_KHR } @@ -148,7 +152,7 @@ class VkVideoEncoderH265 : public VkVideoEncoder { private: VkVideoEncodeFrameInfoH265* GetEncodeFrameInfoH265(VkSharedBaseObj& encodeFrameInfo) { - assert(VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_PICTURE_INFO_KHR == encodeFrameInfo->GetType()); + assert(VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR == encodeFrameInfo->GetCodecType()); VkVideoEncodeFrameInfo* pEncodeFrameInfo = encodeFrameInfo; return (VkVideoEncodeFrameInfoH265*)pEncodeFrameInfo; } From 7417426eb98888b2a802686bfff282a8068c4d99 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Mon, 20 Apr 2026 06:30:05 +0000 Subject: [PATCH 3/5] encode: improve reset of VkVideoEncodeInfoKHR.pNext chain Prior to this commit, the code in VkVideoEncodeFrameInfo::Reset() was written with the assumption that the first element in the pNext chain of VkVideoEncodeInfoKHR would be a codec-specific structure describing the picture info (e.g. VkVideoEncodeH264PictureInfoKHR). Correspondingly, the overridden versions of ::Reset() in the derived classes would simply do: pictureInfo.pNext = nullptr; to clear the pNext chain. Commit 16607feb9b86984b53e1747332c1ca386d7ecd2d regressed this by changing the pNext chaining behaviour in the ChainNextVkStruct() helper to add structures at the head of the pNext chain rather than at the tail. This meant that the codec-specific picture info structure was not guaranteed to be the first structure in the pNext chain of VkVideoEncodeInfoKHR, so the above assignment was not sufficient to clear the pNext chain. This change updates VkVideoEncodeFrameInfo::Reset() to clear the pNext of encodeInfo (i.e. VkVideoEncodeInfoKHR) and then have it point to codec-specific picture info structure in the overridden ::Reset() provided by the derived class. This change also restores the functionality of quantization maps which was broken by commit 16607feb9b86984b53e1747332c1ca386d7ecd2d. Signed-off-by: Srinath Kumarapuram --- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h | 3 ++- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h | 4 ++++ vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h | 3 +++ vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h | 3 +++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h index c604df49..7d0d99a9 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h @@ -250,7 +250,8 @@ class VkVideoEncoder : public VkVideoRefCountBase { virtual void Reset(bool releaseResources = true) { // Clear and check state assert(encodeInfo.sType == VK_STRUCTURE_TYPE_VIDEO_ENCODE_INFO_KHR); - assert(encodeInfo.pNext != nullptr); + + encodeInfo.pNext = nullptr; if ((frameInputOrderNum == (uint64_t)-1) && (frameEncodeInputOrderNum == (uint64_t)-1) && diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h index 2fc4566c..aa3b9c23 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h @@ -79,6 +79,10 @@ class VkVideoEncoderAV1 : public VkVideoEncoder // Reset the base first VkVideoEncodeFrameInfo::Reset(releaseResources); + + // After resetting the base structure parameters, start building the pNext chain again + assert(pictureInfo.sType == VK_STRUCTURE_TYPE_VIDEO_ENCODE_AV1_PICTURE_INFO_KHR); + encodeInfo.pNext = &pictureInfo; } virtual ~VkVideoEncodeFrameInfoAV1() { diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h index f5f7b4ce..2e8c41da 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h @@ -92,6 +92,9 @@ class VkVideoEncoderH264 : public VkVideoEncoder { // Reset the base first VkVideoEncodeFrameInfo::Reset(releaseResources); + // After resetting the base structure parameters, start building the pNext chain again + encodeInfo.pNext = &pictureInfo; + // Clear and check state assert(pictureInfo.sType == VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_PICTURE_INFO_KHR); assert(naluSliceInfo[0].sType == VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_NALU_SLICE_INFO_KHR); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h index 162d23e5..be7a2050 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h @@ -91,6 +91,9 @@ class VkVideoEncoderH265 : public VkVideoEncoder { // Reset the base first VkVideoEncodeFrameInfo::Reset(releaseResources); + // After resetting the base structure parameters, start building the pNext chain again + encodeInfo.pNext = &pictureInfo; + // Clear and check state assert(pictureInfo.sType == VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_PICTURE_INFO_KHR); assert(naluSliceSegmentInfo[0].sType == VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_NALU_SLICE_SEGMENT_INFO_KHR); From f87a519ee3c0c1b7d720406c6423b27660b14c4e Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Mon, 28 Jul 2025 13:11:20 +0530 Subject: [PATCH 4/5] encode: simplify pNext chaining for the inline query structure Previously, any code adding a structure to the pNext chain of VkVideoEncodeInfoKHR had to be careful to preserve a codec-specific picture as the first element in this chain, or restore it to that state as soon as possible, because code elsewhere assumed it or depended on it for correct operation. With the previous two commits, such assumptions have been eliminated, making pNext chaining for VkVideoEncodeInfoKHR much easier. Signed-off-by: Srinath Kumarapuram --- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp index e935393b..a220be15 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp @@ -1930,13 +1930,10 @@ VkResult VkVideoEncoder::RecordVideoCodingCmd(VkSharedBaseObjencodeInfo; - vk::ChainNextVkStruct(*pStruct, videoInlineQueryInfoKHR); - vkDevCtx->CmdEncodeVideoKHR(cmdBuf, &encodeFrameInfo->encodeInfo); + vk::ChainNextVkStruct(encodeFrameInfo->encodeInfo, videoInlineQueryInfoKHR); - // Remove the stack pointer from the chain, causes a use after free otherwise in GetEncodeFrameInfoH264 - encodeFrameInfo->encodeInfo.pNext = videoInlineQueryInfoKHR.pNext; + vkDevCtx->CmdEncodeVideoKHR(cmdBuf, &encodeFrameInfo->encodeInfo); } else { From ac910d36c7ce2a35585a6f128bce0586810833c3 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Mon, 27 Apr 2026 16:13:19 +0530 Subject: [PATCH 5/5] encode: tag overridding methods with the 'override' keyword The VkVideoEncodeFrameInfo struct contains a Reset() virtual method. Codec-specific structs derived from VkVideoEncodeFrameInfo contain a corresponding Reset() method which overrides VkVideoEncodeFrameInfo::Reset(), but the 'override' keyword was missing from the definitions of these overriding methods, which can cause compilers to complain about missing or inconsistent method overrides. --- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h | 2 +- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h | 2 +- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h index aa3b9c23..8c2b9680 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderAV1.h @@ -73,7 +73,7 @@ class VkVideoEncoderAV1 : public VkVideoEncoder pictureInfo.pStdPictureInfo = &stdPictureInfo; } - virtual void Reset(bool releaseResources = true) { + virtual void Reset(bool releaseResources = true) override { pictureInfo.pNext = nullptr; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h index 2e8c41da..5abf3947 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.h @@ -85,7 +85,7 @@ class VkVideoEncoderH264 : public VkVideoEncoder { stdDpbSlotInfo->pStdReferenceInfo = stdReferenceInfo; }; - virtual void Reset(bool releaseResources = true) { + virtual void Reset(bool releaseResources = true) override { pictureInfo.pNext = nullptr; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h index be7a2050..5faad859 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.h @@ -84,7 +84,7 @@ class VkVideoEncoderH265 : public VkVideoEncoder { stdDpbSlotInfo->pStdReferenceInfo = stdReferenceInfo; }; - virtual void Reset(bool releaseResources = true) { + virtual void Reset(bool releaseResources = true) override { pictureInfo.pNext = nullptr;