1# -*- coding: utf-8 -*-
2
3#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
23import os
24import sys
25import shutil
26import tarfile
27import zipfile
28import hashlib
29import argparse
30import subprocess
31import ssl
32import stat
33import platform
34
35scriptPath = os.path.join(os.path.dirname(__file__), "..", "scripts")
36sys.path.insert(0, scriptPath)
37
38from ctsbuild.common import *
39
40EXTERNAL_DIR	= os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
41
42SYSTEM_NAME		= platform.system()
43
44def computeChecksum (data):
45	return hashlib.sha256(data).hexdigest()
46
47def onReadonlyRemoveError (func, path, exc_info):
48	os.chmod(path, stat.S_IWRITE)
49	os.unlink(path)
50
51class Source:
52	def __init__(self, baseDir, extractDir):
53		self.baseDir		= baseDir
54		self.extractDir		= extractDir
55
56	def clean (self):
57		fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
58		# Remove read-only first
59		readonlydir = os.path.join(fullDstPath, ".git")
60		if os.path.exists(readonlydir):
61			shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError)
62		if os.path.exists(fullDstPath):
63			shutil.rmtree(fullDstPath, ignore_errors=False)
64
65class SourcePackage (Source):
66	def __init__(self, url, checksum, baseDir, extractDir = "src", postExtract=None):
67		Source.__init__(self, baseDir, extractDir)
68		self.url			= url
69		self.filename		= os.path.basename(self.url)
70		self.checksum		= checksum
71		self.archiveDir		= "packages"
72		self.postExtract	= postExtract
73
74	def clean (self):
75		Source.clean(self)
76		self.removeArchives()
77
78	def update (self, cmdProtocol = None, force = False):
79		if not self.isArchiveUpToDate():
80			self.fetchAndVerifyArchive()
81
82		if self.getExtractedChecksum() != self.checksum:
83			Source.clean(self)
84			self.extract()
85			self.storeExtractedChecksum(self.checksum)
86
87	def removeArchives (self):
88		archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
89		if os.path.exists(archiveDir):
90			shutil.rmtree(archiveDir, ignore_errors=False)
91
92	def isArchiveUpToDate (self):
93		archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
94		if os.path.exists(archiveFile):
95			return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
96		else:
97			return False
98
99	def getExtractedChecksumFilePath (self):
100		return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
101
102	def getExtractedChecksum (self):
103		extractedChecksumFile = self.getExtractedChecksumFilePath()
104
105		if os.path.exists(extractedChecksumFile):
106			return readFile(extractedChecksumFile)
107		else:
108			return None
109
110	def storeExtractedChecksum (self, checksum):
111		checksum_bytes = checksum.encode("utf-8")
112		writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
113
114	def connectToUrl (self, url):
115		result = None
116
117		if sys.version_info < (3, 0):
118			from urllib2 import urlopen
119		else:
120			from urllib.request import urlopen
121
122		if args.insecure:
123			print("Ignoring certificate checks")
124			ssl_context = ssl._create_unverified_context()
125			result = urlopen(url, context=ssl_context)
126		else:
127			result = urlopen(url)
128
129		return result
130
131	def fetchAndVerifyArchive (self):
132		print("Fetching %s" % self.url)
133
134		req			= self.connectToUrl(self.url)
135		data		= req.read()
136		checksum	= computeChecksum(data)
137		dstPath		= os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
138
139		if checksum != self.checksum:
140			raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
141
142		if not os.path.exists(os.path.dirname(dstPath)):
143			os.makedirs(os.path.dirname(dstPath))
144
145		writeBinaryFile(dstPath, data)
146
147	def extract (self):
148		print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
149
150		srcPath	= os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
151		tmpPath	= os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
152		dstPath	= os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
153
154		if self.filename.endswith(".zip"):
155			archive	= zipfile.ZipFile(srcPath)
156		else:
157			archive	= tarfile.open(srcPath)
158
159		if os.path.exists(tmpPath):
160			shutil.rmtree(tmpPath, ignore_errors=False)
161
162		os.mkdir(tmpPath)
163
164		archive.extractall(tmpPath)
165		archive.close()
166
167		extractedEntries = os.listdir(tmpPath)
168		if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
169			raise Exception("%s doesn't contain single top-level directory" % self.filename)
170
171		topLevelPath = os.path.join(tmpPath, extractedEntries[0])
172
173		if not os.path.exists(dstPath):
174			os.mkdir(dstPath)
175
176		for entry in os.listdir(topLevelPath):
177			if os.path.exists(os.path.join(dstPath, entry)):
178				raise Exception("%s exists already" % entry)
179
180			shutil.move(os.path.join(topLevelPath, entry), dstPath)
181
182		shutil.rmtree(tmpPath, ignore_errors=True)
183
184		if self.postExtract != None:
185			self.postExtract(dstPath)
186
187class SourceFile (Source):
188	def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
189		Source.__init__(self, baseDir, extractDir)
190		self.url			= url
191		self.filename		= filename
192		self.checksum		= checksum
193
194	def update (self, cmdProtocol = None, force = False):
195		if not self.isFileUpToDate():
196			Source.clean(self)
197			self.fetchAndVerifyFile()
198
199	def isFileUpToDate (self):
200		file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
201		if os.path.exists(file):
202			data = readFile(file)
203			return computeChecksum(data.encode('utf-8')) == self.checksum
204		else:
205			return False
206
207	def connectToUrl (self, url):
208		result = None
209
210		if sys.version_info < (3, 0):
211			from urllib2 import urlopen
212		else:
213			from urllib.request import urlopen
214
215		if args.insecure:
216			print("Ignoring certificate checks")
217			ssl_context = ssl._create_unverified_context()
218			result = urlopen(url, context=ssl_context)
219		else:
220			result = urlopen(url)
221
222		return result
223
224	def fetchAndVerifyFile (self):
225		print("Fetching %s" % self.url)
226
227		req			= self.connectToUrl(self.url)
228		data		= req.read()
229		checksum	= computeChecksum(data)
230		dstPath		= os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
231
232		if checksum != self.checksum:
233			raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
234
235		if not os.path.exists(os.path.dirname(dstPath)):
236			os.mkdir(os.path.dirname(dstPath))
237
238		writeBinaryFile(dstPath, data)
239
240class GitRepo (Source):
241	def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = [], patch = ""):
242		Source.__init__(self, baseDir, extractDir)
243		self.httpsUrl	= httpsUrl
244		self.sshUrl		= sshUrl
245		self.revision	= revision
246		self.removeTags	= removeTags
247		self.patch		= patch
248
249	def checkout(self, url, fullDstPath, force):
250		if not os.path.exists(os.path.join(fullDstPath, '.git')):
251			execute(["git", "clone", "--no-checkout", url, fullDstPath])
252
253		pushWorkingDir(fullDstPath)
254		try:
255			for tag in self.removeTags:
256				proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
257				(stdout, stderr) = proc.communicate()
258				if len(stdout) > 0:
259					execute(["git", "tag", "-d",tag])
260			force_arg = ['--force'] if force else []
261			execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
262			execute(["git", "checkout"] + force_arg + [self.revision])
263
264			if(self.patch != ""):
265				patchFile = os.path.join(EXTERNAL_DIR, self.patch)
266				execute(["git", "reset", "--hard", "HEAD"])
267				execute(["git", "apply", patchFile])
268		finally:
269			popWorkingDir()
270
271	def update (self, cmdProtocol, force = False):
272		fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
273		url         = self.httpsUrl
274		backupUrl   = self.sshUrl
275
276		# If url is none then start with ssh
277		if cmdProtocol == 'ssh' or url == None:
278			url       = self.sshUrl
279			backupUrl = self.httpsUrl
280
281		try:
282			self.checkout(url, fullDstPath, force)
283		except:
284			if backupUrl != None:
285				self.checkout(backupUrl, fullDstPath, force)
286
287def postExtractLibpng (path):
288	shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
289				os.path.join(path, "pnglibconf.h"))
290
291PACKAGES = [
292	SourcePackage(
293		"http://zlib.net/fossils/zlib-1.2.13.tar.gz",
294		"b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
295		"zlib"),
296	SourcePackage(
297		"http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
298		"c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
299		"libpng",
300		postExtract = postExtractLibpng),
301	SourceFile(
302		"https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
303		"renderdoc_app.h",
304		"e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
305		"renderdoc"),
306	GitRepo(
307		"https://github.com/KhronosGroup/SPIRV-Tools.git",
308		"git@github.com:KhronosGroup/SPIRV-Tools.git",
309		"bfc94f63a7adbcf8ae166f5f108ac9f69079efc0",
310		"spirv-tools"),
311	GitRepo(
312		"https://github.com/KhronosGroup/glslang.git",
313		None,
314		"c5117b328afc86e16edff6ed6afe0fe7872a7cf3",
315		"glslang",
316		removeTags = ["main-tot"]),
317	GitRepo(
318		"https://github.com/KhronosGroup/SPIRV-Headers.git",
319		"git@github.com:KhronosGroup/SPIRV-Headers.git",
320		"b8b9eb8640c8c0107ba580fbcb10f969022ca32c",
321		"spirv-headers"),
322	GitRepo(
323		"https://github.com/KhronosGroup/Vulkan-Docs.git",
324		"git@github.com:KhronosGroup/Vulkan-Docs.git",
325		"b9aad705f0d9e5e6734ac2ad671d5d1de57b05e0",
326		"vulkan-docs"),
327	GitRepo(
328		"https://github.com/google/amber.git",
329		None,
330		"933ecb4d6288675a92eb1650e0f52b1d7afe8273",
331		"amber"),
332	GitRepo(
333		"https://github.com/open-source-parsers/jsoncpp.git",
334		"git@github.com:open-source-parsers/jsoncpp.git",
335		"9059f5cad030ba11d37818847443a53918c327b1",
336		"jsoncpp"),
337	# NOTE: The samples application is not well suited to external
338	# integration, this fork contains the small fixes needed for use
339	# by the CTS.
340	GitRepo(
341		"https://github.com/Igalia/vk_video_samples.git",
342		"git@github.com:Igalia/vk_video_samples.git",
343		"138bbe048221d315962ddf8413aa6a08cc62a381",
344		"nvidia-video-samples"),
345	GitRepo(
346		"https://github.com/Igalia/ESExtractor.git",
347		"git@github.com:Igalia/ESExtractor.git",
348		"v0.3.3",
349		"ESExtractor"),
350]
351
352def parseArgs ():
353	versionsForInsecure = ((2,7,9), (3,4,3))
354	versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
355
356	parser = argparse.ArgumentParser(description = "Fetch external sources")
357	parser.add_argument('--clean', dest='clean', action='store_true', default=False,
358						help='Remove sources instead of fetching')
359	parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
360						help="Disable certificate check for external sources."
361						" Minimum python version required " + versionsForInsecureStr)
362	parser.add_argument('--protocol', dest='protocol', default='https', choices=['ssh', 'https'],
363						help="Select protocol to checkout git repositories.")
364	parser.add_argument('--force', dest='force', action='store_true', default=False,
365						help="Pass --force to git fetch and checkout commands")
366
367	args = parser.parse_args()
368
369	if args.insecure:
370		for versionItem in versionsForInsecure:
371			if (sys.version_info.major == versionItem[0]):
372				if sys.version_info < versionItem:
373					parser.error("For --insecure minimum required python version is " +
374								versionsForInsecureStr)
375				break;
376
377	return args
378
379def run(*popenargs, **kwargs):
380	process = subprocess.Popen(*popenargs, **kwargs)
381
382	try:
383		stdout, stderr = process.communicate(None)
384	except:
385		process.kill()
386		process.wait()
387		raise
388
389	retcode = process.poll()
390
391	if retcode:
392		raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr)
393
394	return retcode, stdout, stderr
395
396if __name__ == "__main__":
397	args = parseArgs()
398
399	for pkg in PACKAGES:
400		if args.clean:
401			pkg.clean()
402		else:
403			pkg.update(args.protocol, args.force)
404