1 /*-------------------------------------------------------------------------
2 * Vulkan CTS Framework
3 * --------------------
4 *
5 * Copyright (c) 2021 The Khronos Group Inc.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *-------------------------------------------------------------------------*/
20
21 #include <iostream>
22 #include <fstream>
23 #include <sstream>
24 #include <json/json.h>
25 #include "deCommandLine.hpp"
26 #include "deDirectoryIterator.hpp"
27 #include "tcuCommandLine.hpp"
28 #include "tcuPlatform.hpp"
29 #include "tcuTestContext.hpp"
30 #include "tcuResource.hpp"
31 #include "tcuTestLog.hpp"
32 #include "vkPlatform.hpp"
33 #include "vktTestCase.hpp"
34 #include "vksStructsVKSC.hpp"
35 #include "vksCacheBuilder.hpp"
36
37 namespace opt
38 {
39
40 DE_DECLARE_COMMAND_LINE_OPT(CompilerDataPath, std::string);
41 DE_DECLARE_COMMAND_LINE_OPT(CompilerOutputFile, std::string);
42 DE_DECLARE_COMMAND_LINE_OPT(LogFile, std::string);
43 DE_DECLARE_COMMAND_LINE_OPT(FilePrefix, std::string);
44
registerOptions(de::cmdline::Parser& parser)45 void registerOptions (de::cmdline::Parser& parser)
46 {
47 using de::cmdline::Option;
48 using de::cmdline::NamedValue;
49
50 parser << Option<CompilerDataPath> ("p", "path", "Offline pipeline data directory", "");
51 parser << Option<CompilerOutputFile> ("o", "out", "Output file with pipeline cache", "");
52 parser << Option<LogFile> ("l", "log", "Log file", "dummy.log");
53 parser << Option<FilePrefix> ("x", "prefix", "Prefix for input files", "");
54 }
55
56 }
57
58 enum PipelineType
59 {
60 PT_UNDEFINED_PIPELINE = 0,
61 PT_GRAPHICS_PIPELINE,
62 PT_COMPUTE_PIPELINE,
63 };
64
importFilesForExternalCompiler(vksc_server::VulkanPipelineCacheInput& input, const std::string& path, const std::string& filePrefix)65 void importFilesForExternalCompiler (vksc_server::VulkanPipelineCacheInput& input,
66 const std::string& path,
67 const std::string& filePrefix)
68 {
69 vksc_server::json::Context context;
70
71 for (de::DirectoryIterator iter(path); iter.hasItem(); iter.next())
72 {
73 const de::FilePath filePath = iter.getItem();
74 if (filePath.getType() != de::FilePath::TYPE_FILE)
75 continue;
76 if (filePath.getFileExtension() != "json")
77 continue;
78 if (!filePrefix.empty() && filePath.getBaseName().find(filePrefix) != 0)
79 continue;
80
81 std::string fileContents;
82 {
83 std::ifstream file(filePath.getPath());
84 std::stringstream buffer;
85 buffer << file.rdbuf();
86 fileContents = buffer.str();
87 }
88
89 Json::Value jsonRoot;
90 std::string errors;
91 bool parsingSuccessful = context.reader->parse(fileContents.c_str(), fileContents.c_str() + fileContents.size(), &jsonRoot, &errors);
92 if (!parsingSuccessful)
93 TCU_THROW(InternalError, (std::string("JSON parsing error. File ") + filePath.getPath() + " Error : " + errors).c_str());
94
95 // decide what pipeline type will be created later
96 PipelineType pipelineType = PT_UNDEFINED_PIPELINE;
97 if (jsonRoot.isMember("GraphicsPipelineState"))
98 pipelineType = PT_GRAPHICS_PIPELINE;
99 else if (jsonRoot.isMember("ComputePipelineState"))
100 pipelineType = PT_COMPUTE_PIPELINE;
101 if(pipelineType == PT_UNDEFINED_PIPELINE)
102 TCU_THROW(InternalError, (std::string("JSON - unknown pipeline. File ") + filePath.getPath()).c_str());
103
104 const Json::Value& jsonGraphicsPipelineState = jsonRoot["GraphicsPipelineState"];
105 const Json::Value& jsonComputePipelineState = jsonRoot["ComputePipelineState"];
106 const Json::Value& jsonPipelineState = (pipelineType == PT_GRAPHICS_PIPELINE ) ? jsonGraphicsPipelineState : jsonComputePipelineState;
107 vksc_server::VulkanJsonPipelineDescription pipelineDescription;
108
109 {
110 const Json::Value& jsonSamplerYcbcrConversions = jsonPipelineState["YcbcrSamplers"];
111 if (!jsonSamplerYcbcrConversions.isNull())
112 {
113 for (Json::ArrayIndex i = 0; i < jsonSamplerYcbcrConversions.size(); ++i)
114 {
115 const Json::Value::Members membersNames = jsonSamplerYcbcrConversions[i].getMemberNames();
116 const Json::Value& value = jsonSamplerYcbcrConversions[i][membersNames[0]];
117 deUint64 index;
118 std::istringstream(membersNames[0]) >> index;
119 input.samplerYcbcrConversions[vk::VkSamplerYcbcrConversion(index)] = std::string(fileContents.begin() + value.getOffsetStart(), fileContents.begin() + value.getOffsetLimit());
120 }
121 }
122
123 const Json::Value& jsonSamplers = jsonPipelineState["ImmutableSamplers"];
124 if (!jsonSamplers.isNull())
125 {
126 for (Json::ArrayIndex i = 0; i < jsonSamplers.size(); ++i)
127 {
128 const Json::Value::Members membersNames = jsonSamplers[i].getMemberNames();
129 const Json::Value& value = jsonSamplers[i][membersNames[0]];
130 deUint64 index;
131 std::istringstream(membersNames[0]) >> index;
132 input.samplers[vk::VkSampler(index)] = std::string(fileContents.begin() + value.getOffsetStart(), fileContents.begin() + value.getOffsetLimit());
133 }
134 }
135
136 const Json::Value& jsonDescriptorSetLayouts = jsonPipelineState["DescriptorSetLayouts"];
137 if (!jsonDescriptorSetLayouts.isNull())
138 {
139 for (Json::ArrayIndex i = 0; i < jsonDescriptorSetLayouts.size(); ++i)
140 {
141 const Json::Value::Members membersNames = jsonDescriptorSetLayouts[i].getMemberNames();
142 const Json::Value& value = jsonDescriptorSetLayouts[i][membersNames[0]];
143 deUint64 index;
144 std::istringstream(membersNames[0]) >> index;
145 input.descriptorSetLayouts[vk::VkDescriptorSetLayout(index)] = std::string(fileContents.begin() + value.getOffsetStart(), fileContents.begin() + value.getOffsetLimit());
146 }
147 }
148
149 deUint64 pipelineLayoutHandle = 0u;
150 deUint64 renderPassHandle = 0u;
151 std::map<std::string, deUint64> stages;
152
153 const Json::Value& jsonComputePipeline = jsonPipelineState["ComputePipeline"];
154 if (!jsonComputePipeline.isNull())
155 {
156 pipelineDescription.pipelineContents = std::string(fileContents.begin() + jsonComputePipeline.getOffsetStart(), fileContents.begin() + jsonComputePipeline.getOffsetLimit());
157 pipelineLayoutHandle = jsonComputePipeline["layout"].asUInt64();
158
159 const Json::Value& jsonStage = jsonComputePipeline["stage"];
160 stages[jsonStage["stage"].asString()] = jsonStage["module"].asUInt64();
161 }
162
163 const Json::Value& jsonGraphicsPipeline = jsonPipelineState["GraphicsPipeline"];
164 if (!jsonGraphicsPipeline.isNull())
165 {
166 pipelineDescription.pipelineContents = std::string(fileContents.begin() + jsonGraphicsPipeline.getOffsetStart(), fileContents.begin() + jsonGraphicsPipeline.getOffsetLimit());
167 pipelineLayoutHandle = jsonGraphicsPipeline["layout"].asUInt64();
168 renderPassHandle = jsonGraphicsPipeline["renderPass"].asUInt64();
169
170 const Json::Value& jsonStages = jsonGraphicsPipeline["pStages"];
171 for (Json::ArrayIndex i = 0; i < jsonStages.size(); ++i)
172 stages[jsonStages[i]["stage"].asString()] = jsonStages[i]["module"].asUInt64();
173 }
174
175 const Json::Value& jsonPipelineLayout = jsonPipelineState["PipelineLayout"];
176 if (!jsonPipelineLayout.isNull() && pipelineLayoutHandle != 0u)
177 {
178 input.pipelineLayouts[vk::VkPipelineLayout(pipelineLayoutHandle)] = std::string(fileContents.begin() + jsonPipelineLayout.getOffsetStart(), fileContents.begin() + jsonPipelineLayout.getOffsetLimit());
179 }
180
181 const Json::Value& jsonRenderPass = jsonPipelineState["Renderpass"];
182 if (!jsonRenderPass.isNull() && renderPassHandle != 0u)
183 {
184 input.renderPasses[vk::VkRenderPass(renderPassHandle)] = std::string(fileContents.begin() + jsonRenderPass.getOffsetStart(), fileContents.begin() + jsonRenderPass.getOffsetLimit());
185 }
186
187 const Json::Value& jsonRenderPass2 = jsonPipelineState["Renderpass2"];
188 if (!jsonRenderPass2.isNull() && renderPassHandle != 0u)
189 {
190 input.renderPasses[vk::VkRenderPass(renderPassHandle)] = std::string(fileContents.begin() + jsonRenderPass.getOffsetStart(), fileContents.begin() + jsonRenderPass.getOffsetLimit());
191 }
192
193 const Json::Value& jsonShaderFileNames = jsonPipelineState["ShaderFileNames"];
194 if (!jsonShaderFileNames.isNull())
195 {
196 for (Json::ArrayIndex i = 0; i < jsonShaderFileNames.size(); ++i)
197 {
198 std::string stageName = jsonShaderFileNames[i]["stage"].asString();
199 std::string fileName = jsonShaderFileNames[i]["filename"].asString();
200 auto it = stages.find(stageName);
201 if(it == end(stages))
202 TCU_THROW(InternalError, (std::string("JSON - missing shader stage. File ") + filePath.getPath()).c_str());
203
204 de::FilePath shaderPath (path);
205 shaderPath.join(de::FilePath(fileName));
206 std::ifstream iFile (shaderPath.getPath(), std::ios::in | std::ios::binary);
207 if(!iFile)
208 TCU_THROW(InternalError, (std::string("JSON - missing shader file ") + fileName + ". File " + filePath.getPath()).c_str());
209
210 auto fileBegin = iFile.tellg();
211 iFile.seekg(0, std::ios::end);
212 auto fileEnd = iFile.tellg();
213 iFile.seekg(0, std::ios::beg);
214 std::size_t fileSize = static_cast<std::size_t>(fileEnd - fileBegin);
215 std::vector<deUint8> shaderData (fileSize);
216
217 iFile.read(reinterpret_cast<char*>(shaderData.data()), fileSize);
218 if (iFile.fail())
219 TCU_THROW(InternalError, (std::string("JSON - error reading shader file ") + fileName + ". File " + filePath.getPath()).c_str());
220
221 vk::VkShaderModuleCreateInfo smCI
222 {
223 VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, // VkStructureType sType;
224 DE_NULL, // const void* pNext;
225 vk::VkShaderModuleCreateFlags(0u), // VkShaderModuleCreateFlags flags;
226 fileSize, // deUintptr codeSize;
227 reinterpret_cast<deUint32*>(shaderData.data()) // const deUint32* pCode;
228 };
229
230 input.shaderModules[vk::VkShaderModule(it->second)] = vksc_server::json::writeJSON_VkShaderModuleCreateInfo(smCI);
231 }
232 }
233
234 const Json::Value& jsonPhysicalDeviceFeatures = jsonPipelineState["PhysicalDeviceFeatures"];
235 if (!jsonPhysicalDeviceFeatures.isNull())
236 {
237 pipelineDescription.deviceFeatures = std::string(fileContents.begin() + jsonPhysicalDeviceFeatures.getOffsetStart(), fileContents.begin() + jsonPhysicalDeviceFeatures.getOffsetLimit());
238 }
239 }
240
241 const Json::Value& jsonEnabledExtensions = jsonRoot["EnabledExtensions"];
242 if (!jsonEnabledExtensions.isNull())
243 {
244 for (Json::ArrayIndex i = 0; i < jsonEnabledExtensions.size(); ++i)
245 pipelineDescription.deviceExtensions.push_back(jsonEnabledExtensions[i].asString());
246 }
247
248 const Json::Value& jsonPipelineUUID = jsonRoot["PipelineUUID"];
249 if (!jsonPipelineUUID.isNull())
250 {
251 pipelineDescription.id.sType = VK_STRUCTURE_TYPE_PIPELINE_OFFLINE_CREATE_INFO;
252 pipelineDescription.id.pNext = DE_NULL;
253 for (Json::ArrayIndex i = 0; i < jsonPipelineUUID.size(); ++i)
254 pipelineDescription.id.pipelineIdentifier[i] = deUint8(jsonPipelineUUID[i].asUInt());
255 pipelineDescription.id.matchControl = VK_PIPELINE_MATCH_CONTROL_APPLICATION_UUID_EXACT_MATCH;
256 pipelineDescription.id.poolEntrySize = 0u;
257 }
258 input.pipelines.push_back(pipelineDescription);
259 }
260 }
261
262 tcu::Platform* createPlatform(void);
263
main(int argc, char** argv)264 int main (int argc, char** argv)
265 {
266 de::cmdline::CommandLine cmdLine;
267
268 // Parse command line.
269 {
270 de::cmdline::Parser parser;
271 opt::registerOptions(parser);
272
273 if (!parser.parse(argc, argv, &cmdLine, std::cerr))
274 {
275 parser.help(std::cout);
276 return EXIT_FAILURE;
277 }
278 }
279
280 try
281 {
282 // load JSON files into VulkanPipelineCacheInput
283 vksc_server::VulkanPipelineCacheInput input;
284 importFilesForExternalCompiler(input, cmdLine.getOption<opt::CompilerDataPath>(), cmdLine.getOption<opt::FilePrefix>());
285
286 // create Vulkan instance
287 tcu::CommandLine cmdLineDummy {"--deqp-vk-device-id=0"};
288 tcu::DirArchive archive {""};
289 tcu::TestLog log { cmdLine.getOption<opt::LogFile>().c_str() }; log.supressLogging(true);
290 de::SharedPtr<tcu::Platform> platform {createPlatform()};
291 #ifdef DE_PLATFORM_USE_LIBRARY_TYPE
292 de::SharedPtr<vk::Library> library {platform->getVulkanPlatform().createLibrary(vk::Platform::LIBRARY_TYPE_VULKAN, DE_NULL)};
293 #else
294 de::SharedPtr<vk::Library> library {platform->getVulkanPlatform().createLibrary(DE_NULL)};
295 #endif
296 tcu::TestContext tcx {*platform, archive, log, cmdLineDummy, nullptr};
297 vk::BinaryCollection collection {};
298 vkt::Context context (tcx, library->getPlatformInterface(), collection, de::SharedPtr<vk::ResourceInterface>{new vk::ResourceInterfaceStandard{ tcx }});
299
300 // create pipeline cache
301 std::vector<deUint8> binary = vksc_server::buildPipelineCache(
302 input,
303 library->getPlatformInterface(),
304 context.getInstance(),
305 context.getInstanceInterface(),
306 context.getPhysicalDevice(),
307 context.getUniversalQueueFamilyIndex());
308
309 // write pipeline cache to output file
310 std::ofstream oFile (cmdLine.getOption<opt::CompilerOutputFile>().c_str(), std::ios::out | std::ios::binary);
311 if (!oFile)
312 TCU_THROW(InternalError, (std::string("Cannot create file : ") + cmdLine.getOption<opt::CompilerOutputFile>().c_str()));
313 oFile.write(reinterpret_cast<char*>(binary.data()), binary.size());
314 }
315 catch (const std::exception& e)
316 {
317 std::cout << e.what() << std::endl;
318 }
319
320 return EXIT_SUCCESS;
321 }
322