1 #ifndef _VKTVIDEOFRAMEBUFFER_HPP
2 #define _VKTVIDEOFRAMEBUFFER_HPP
3 /*------------------------------------------------------------------------
4 * Vulkan Conformance Tests
5 * ------------------------
6 *
7 * Copyright (c) 2023 The Khronos Group Inc.
8 *
9 * Licensed under the Apache License, Version 2.0 (the "License");
10 * you may not use this file except in compliance with the License.
11 * You may obtain a copy of the License at
12 *
13 *	  http://www.apache.org/licenses/LICENSE-2.0
14 *
15 * Unless required by applicable law or agreed to in writing, software
16 * distributed under the License is distributed on an "AS IS" BASIS,
17 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 * See the License for the specific language governing permissions and
19 * limitations under the License.
20 *
21 *//*!
22 * \file
23 * \brief Video framebuffer
24 *//*--------------------------------------------------------------------*/
25 /*
26 * Copyright 2020 NVIDIA Corporation.
27 *
28 * Licensed under the Apache License, Version 2.0 (the "License");
29 * you may not use this file except in compliance with the License.
30 * You may obtain a copy of the License at
31 *
32 *    http://www.apache.org/licenses/LICENSE-2.0
33 *
34 * Unless required by applicable law or agreed to in writing, software
35 * distributed under the License is distributed on an "AS IS" BASIS,
36 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
37 * See the License for the specific language governing permissions and
38 * limitations under the License.
39 */
40 #include "deDefs.hpp"
41 #include "vktVideoTestUtils.hpp"
42 #include "extNvidiaVideoParserIf.hpp"
43 
44 namespace vkt
45 {
46 namespace video
47 {
48 
49 using ImagePtr =de::MovePtr<ImageWithMemory>;
50 
51 class VkImageResource : public VkVideoRefCountBase
52 {
53 public:
54 	static VkResult Create(DeviceContext&					 vkDevCtx,
55 						   const VkImageCreateInfo*			 pImageCreateInfo,
56 						   VkSharedBaseObj<VkImageResource>& imageResource);
57 
58 	int32_t AddRef() override
59 	{
60 		return ++m_refCount;
61 	}
62 
63 	int32_t Release() override
64 	{
65 		deUint32 ret = --m_refCount;
66 		// Destroy the device if ref-count reaches zero
67 		if (ret == 0)
68 		{
69 			delete this;
70 		}
71 		return ret;
72 	}
73 
GetImage() const74 	VkImage GetImage() const
75 	{
76 		return m_imageWithMemory->get();
77 	}
78 
GetImageCreateInfo() const79 	const VkImageCreateInfo& GetImageCreateInfo() const
80 	{
81 		return m_imageCreateInfo;
82 	}
83 
84 private:
85 	std::atomic<int32_t>	m_refCount;
86 	const VkImageCreateInfo m_imageCreateInfo;
87 	ImagePtr				m_imageWithMemory;
88 
VkImageResource(DeviceContext& vkDevCtx, const VkImageCreateInfo* pImageCreateInfo)89 	VkImageResource(DeviceContext&							 vkDevCtx,
90 					const VkImageCreateInfo*				 pImageCreateInfo)
91 		: m_refCount(0), m_imageCreateInfo(*pImageCreateInfo)
92 	{
93 		m_imageWithMemory = ImagePtr(new ImageWithMemory(vkDevCtx.getDeviceDriver(), vkDevCtx.device, vkDevCtx.allocator(), *pImageCreateInfo, MemoryRequirement::Local));
94 	}
95 };
96 
97 class VkImageResourceView : public VkVideoRefCountBase
98 {
99 public:
100 	static VkResult Create(DeviceContext&						 vkDevCtx,
101 						   VkSharedBaseObj<VkImageResource>&	 imageResource,
102 						   VkImageSubresourceRange&				 imageSubresourceRange,
103 						   VkSharedBaseObj<VkImageResourceView>& imageResourceView);
104 
AddRef()105 	virtual int32_t AddRef()
106 	{
107 		return ++m_refCount;
108 	}
109 
Release()110 	virtual int32_t Release()
111 	{
112 		deUint32 ret = --m_refCount;
113 		// Destroy the device if ref-count reaches zero
114 		if (ret == 0)
115 		{
116 			delete this;
117 		}
118 		return ret;
119 	}
120 
operator VkImageView() const121 	operator VkImageView() const
122 	{
123 		return m_imageView;
124 	}
GetImageView() const125 	VkImageView GetImageView() const
126 	{
127 		return m_imageView;
128 	}
129 
GetImageResource()130 	const VkSharedBaseObj<VkImageResource>& GetImageResource()
131 	{
132 		return m_imageResource;
133 	}
134 
135 private:
136 	std::atomic<int32_t>			 m_refCount;
137 	DeviceContext&					 m_vkDevCtx;
138 	VkSharedBaseObj<VkImageResource> m_imageResource;
139 	VkImageView						 m_imageView;
140 
VkImageResourceView(DeviceContext& vkDevCtx, VkSharedBaseObj<VkImageResource>& imageResource, VkImageView imageView, VkImageSubresourceRange& )141 	VkImageResourceView(DeviceContext&					  vkDevCtx,
142 						VkSharedBaseObj<VkImageResource>& imageResource,
143 						VkImageView						  imageView,
144 						VkImageSubresourceRange&		  /*imageSubresourceRange*/)
145 		: m_refCount(0), m_vkDevCtx(vkDevCtx), m_imageResource(imageResource), m_imageView(imageView)
146 	{
147 	}
148 
149 	virtual ~VkImageResourceView();
150 };
151 
152 struct DecodedFrame
153 {
154 	int32_t								 pictureIndex;
155 	uint32_t							 imageLayerIndex; // The layer of a multi-layered images. Always "0" for single layered images
156 	int32_t								 displayWidth;
157 	int32_t								 displayHeight;
158 	VkSharedBaseObj<VkImageResourceView> decodedImageView;
159 	VkSharedBaseObj<VkImageResourceView> outputImageView;
160 	VkFence								 frameCompleteFence; // If valid, the fence is signaled when the decoder is done decoding the frame.
161 	VkFence								 frameConsumerDoneFence; // If valid, the fence is signaled when the consumer (graphics, compute or display) is done using the frame.
162 	VkSemaphore							 frameCompleteSemaphore; // If valid, the semaphore is signaled when the decoder is done decoding the frame.
163 	VkSemaphore							 frameConsumerDoneSemaphore; // If valid, the semaphore is signaled when the consumer (graphics, compute or display) is done using the frame.
164 	VkQueryPool							 queryPool; // queryPool handle used for the video queries.
165 	int32_t								 startQueryId; // query Id used for the this frame.
166 	deUint32							 numQueries; // usually one query per frame
167 	// If multiple queues are available, submittedVideoQueueIndex is the queue index that the video frame was submitted to.
168 	// if only one queue is available, submittedVideoQueueIndex will always have a value of "0".
169 	int32_t								 submittedVideoQueueIndex;
170 	uint64_t							 timestamp;
171 	deUint32							 hasConsummerSignalFence : 1;
172 	deUint32							 hasConsummerSignalSemaphore : 1;
173 	// For debugging
174 	int32_t								 decodeOrder;
175 	int32_t								 displayOrder;
176 
Resetvkt::video::DecodedFrame177 	void								 Reset()
178 	{
179 		pictureIndex				= -1;
180 		imageLayerIndex				= 0;
181 		displayWidth				= 0;
182 		displayHeight				= 0;
183 		decodedImageView			= nullptr;
184 		outputImageView				= nullptr;
185 		frameCompleteFence			= VK_NULL_HANDLE;
186 		frameConsumerDoneFence		= VK_NULL_HANDLE;
187 		frameCompleteSemaphore		= VK_NULL_HANDLE;
188 		frameConsumerDoneSemaphore	= VK_NULL_HANDLE;
189 		queryPool					= VK_NULL_HANDLE;
190 		startQueryId				= 0;
191 		numQueries					= 0;
192 		submittedVideoQueueIndex	= 0;
193 		timestamp					= 0;
194 		hasConsummerSignalFence		= false;
195 		hasConsummerSignalSemaphore = false;
196 		// For debugging
197 		decodeOrder					= 0;
198 		displayOrder				= 0;
199 	}
200 };
201 
202 struct DecodedFrameRelease
203 {
204 	int32_t			 pictureIndex;
205 	VkVideotimestamp timestamp;
206 	deUint32		 hasConsummerSignalFence : 1;
207 	deUint32		 hasConsummerSignalSemaphore : 1;
208 	// For debugging
209 	int32_t			 decodeOrder;
210 	int32_t			 displayOrder;
211 };
212 
213 class VulkanVideoFrameBuffer : public IVulkanVideoFrameBufferParserCb
214 {
215 public:
216 	// Synchronization
217 	struct FrameSynchronizationInfo
218 	{
219 		VkFence		frameCompleteFence{VK_NULL_HANDLE};
220 		VkSemaphore frameCompleteSemaphore{VK_NULL_HANDLE};
221 		VkFence		frameConsumerDoneFence{VK_NULL_HANDLE};
222 		VkSemaphore frameConsumerDoneSemaphore{VK_NULL_HANDLE};
223 		VkQueryPool queryPool{VK_NULL_HANDLE};
224 		int32_t		startQueryId;
225 		deUint32	numQueries;
226 		deUint32	hasFrameCompleteSignalFence : 1;
227 		deUint32	hasFrameCompleteSignalSemaphore : 1;
228 	};
229 
230 	struct ReferencedObjectsInfo
231 	{
232 
233 		// The bitstream Buffer
234 		const VkVideoRefCountBase* pBitstreamData;
235 		// PPS
236 		const VkVideoRefCountBase* pStdPps;
237 		// SPS
238 		const VkVideoRefCountBase* pStdSps;
239 		// VPS
240 		const VkVideoRefCountBase* pStdVps;
241 
ReferencedObjectsInfovkt::video::VulkanVideoFrameBuffer::ReferencedObjectsInfo242 		ReferencedObjectsInfo(const VkVideoRefCountBase* pBitstreamDataRef,
243 							  const VkVideoRefCountBase* pStdPpsRef,
244 							  const VkVideoRefCountBase* pStdSpsRef,
245 							  const VkVideoRefCountBase* pStdVpsRef = nullptr)
246 			: pBitstreamData(pBitstreamDataRef)
247 			, pStdPps(pStdPpsRef)
248 			, pStdSps(pStdSpsRef)
249 			, pStdVps(pStdVpsRef)
250 		{
251 		}
252 	};
253 
254 	struct PictureResourceInfo
255 	{
256 		VkImage		  image;
257 		VkFormat	  imageFormat;
258 		VkImageLayout currentImageLayout;
259 	};
260 
261 	virtual int32_t InitImagePool(const VkVideoProfileInfoKHR* pDecodeProfile,
262 								  deUint32					   numImages,
263 								  VkFormat					   dpbImageFormat,
264 								  VkFormat					   outImageFormat,
265 								  const VkExtent2D&			   codedExtent,
266 								  const VkExtent2D&			   maxImageExtent,
267 								  VkImageUsageFlags			   dpbImageUsage,
268 								  VkImageUsageFlags			   outImageUsage,
269 								  deUint32					   queueFamilyIndex,
270 								  bool						   useImageArray		  = false,
271 								  bool						   useImageViewArray	  = false,
272 								  bool						   useSeparateOutputImage = false,
273 								  bool						   useLinearOutput		  = false)																																																						= 0;
274 
275 	virtual int32_t QueuePictureForDecode(int8_t picId, VkParserDecodePictureInfo* pDecodePictureInfo, ReferencedObjectsInfo* pReferencedObjectsInfo, FrameSynchronizationInfo* pFrameSynchronizationInfo)																			= 0;
276 	virtual int32_t DequeueDecodedPicture(DecodedFrame* pDecodedFrame)																																																				= 0;
277 	virtual int32_t ReleaseDisplayedPicture(DecodedFrameRelease** pDecodedFramesRelease, deUint32 numFramesToRelease)																																								= 0;
278 	virtual int32_t GetDpbImageResourcesByIndex(deUint32 numResources, const int8_t* referenceSlotIndexes, VkVideoPictureResourceInfoKHR* pictureResources, PictureResourceInfo* pictureResourcesInfo, VkImageLayout newDpbImageLayerLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR) = 0;
279 	virtual int32_t GetCurrentImageResourceByIndex(int8_t						  referenceSlotIndex,
280 												   VkVideoPictureResourceInfoKHR* dpbPictureResource,
281 												   PictureResourceInfo*			  dpbPictureResourceInfo,
282 												   VkImageLayout				  newDpbImageLayerLayout	= VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR,
283 												   VkVideoPictureResourceInfoKHR* outputPictureResource		= nullptr,
284 												   PictureResourceInfo*			  outputPictureResourceInfo = nullptr,
285 												   VkImageLayout				  newOutputImageLayerLayout = VK_IMAGE_LAYOUT_MAX_ENUM)																																								= 0;
286 	virtual int32_t ReleaseImageResources(deUint32 numResources, const deUint32* indexes)																																															= 0;
287 	virtual int32_t SetPicNumInDecodeOrder(int32_t picId, int32_t picNumInDecodeOrder)																																																= 0;
288 	virtual int32_t SetPicNumInDisplayOrder(int32_t picId, int32_t picNumInDisplayOrder)																																															= 0;
289 	virtual size_t	GetSize()																																																														= 0;
290 	virtual size_t	GetDisplayedFrameCount() const																																																									= 0;
291 
~VulkanVideoFrameBuffer()292 	virtual ~VulkanVideoFrameBuffer()
293 	{
294 	}
295 
296 	static VkResult Create(DeviceContext*							devCtx,
297 						   bool										supportsQueries,
298 						   VkSharedBaseObj<VulkanVideoFrameBuffer>& vkVideoFrameBuffer);
299 };
300 
301 } // namespace video
302 } // namespace vkt
303 
304 #endif // _VKTVIDEOFRAMEBUFFER_HPP
305