1#!/usr/bin/env python3
2# coding: utf-8
3
4"""
5Copyright (c) 2024 Huawei Device Co., Ltd.
6Licensed under the Apache License, Version 2.0 (the "License");
7you may not use this file except in compliance with the License.
8You may obtain a copy of the License at
9
10    http://www.apache.org/licenses/LICENSE-2.0
11
12Unless required by applicable law or agreed to in awriting, software
13distributed under the License is distributed on an "AS IS" BASIS,
14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15See the License for the specific language governing permissions and
16limitations under the License.
17"""
18
19import argparse
20import re
21import subprocess
22import tarfile
23import datetime
24import gzip
25from urllib.parse import urlparse
26
27import httpx
28import json
29import os
30import requests
31import shutil
32import stat
33import sys
34import time
35import tqdm
36import yaml
37import zipfile
38
39
40def is_windows():
41    return sys.platform == 'win32' or sys.platform == 'cygwin'
42
43
44def is_mac():
45    return sys.platform == 'darwin'
46
47
48def parse_configs():
49    config_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../config.yaml')
50    with open(config_file_path, 'r', encoding='utf-8') as config_file:
51        configs = yaml.safe_load(config_file)
52    return configs
53
54
55def parse_file_name(download_url):
56    parsed_url = urlparse(download_url)
57    path = parsed_url.path
58    file_full_name = path.split("/")[-1]
59    file_name = file_full_name.split(".tar.gz")[0]
60    return file_name
61
62
63def convert_to_datetime(date_str):
64    date_format = '%Y%m%d%H%M%S'
65    date_time = datetime.datetime.strptime(date_str, date_format)
66    return date_time
67
68
69def get_download_url(task_name, image_date):
70    image_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S') if image_date is None else \
71        datetime.datetime.strptime(image_date, '%Y-%m-%d').strftime('%Y%m%d') + '235959'
72    last_hour = (convert_to_datetime(image_date) +
73                 datetime.timedelta(hours=-24)).strftime('%Y%m%d%H%M%S')
74    url = 'http://ci.openharmony.cn/api/daily_build/build/tasks'
75    downnload_job = {
76        'pageNum': 1,
77        'pageSize': 1000,
78        'startTime': '',
79        'endTime': '',
80        'projectName': 'openharmony',
81        'branch': 'master',
82        'component': '',
83        'deviceLevel': '',
84        'hardwareBoard': '',
85        'buildStatus': '',
86        'buildFailReason': '',
87        'testResult': '',
88    }
89    downnload_job['startTime'] = str(last_hour)
90    downnload_job['endTime'] = str(image_date)
91    post_result = requests.post(url, json=downnload_job)
92    post_data = json.loads(post_result.text)
93    download_url_suffix = ''
94    for ohos_sdk_list in post_data['data']['dailyBuildVos']:
95        try:
96            if get_remote_download_name(task_name) in ohos_sdk_list['obsPath']:
97                download_url_suffix = ohos_sdk_list['obsPath']
98                break
99        except BaseException as err:
100            print(err)
101    download_url = 'http://download.ci.openharmony.cn/' + download_url_suffix
102    return download_url
103
104
105def retry_after_download_failed(download_url, temp_file, temp_file_name, max_retries=3):
106    for i in range(max_retries):
107        try:
108            is_download_success = download(download_url, temp_file, temp_file_name)
109            if is_download_success:
110                return True
111        except Exception as e:
112            print(f"download failed! retrying... ({i + 1}/{max_retries})")
113            time.sleep(2)
114    return False
115
116
117def download_progress_bar(response, temp, temp_file_name):
118    total_length = int(response.headers.get("content-length"))
119    with tqdm.tqdm(total=total_length, unit="B", unit_scale=True) as pbar:
120        pbar.set_description(temp_file_name)
121        chunk_sum = 0
122        count = 0
123        for chunk in response.iter_bytes():
124            temp.write(chunk)
125            chunk_sum += len(chunk)
126            percentage = chunk_sum / total_length * 100
127            while str(percentage).startswith(str(count)):
128                count += 1
129            pbar.update(len(chunk))
130
131
132def supports_resume(download_url):
133    with httpx.Client() as client:
134        response = client.head(download_url)
135        return response.headers.get("Accept-Ranges") == "bytes"
136
137
138def download(download_url, temp_file, temp_file_name):
139    existing_size = 0
140    if os.path.exists(temp_file):
141        existing_size = os.path.getsize(temp_file)
142
143    headers = {}
144    if existing_size:
145        headers['Range'] = f'bytes={existing_size}-'
146
147    with httpx.Client() as client:
148        response = client.head(download_url)
149        total_file_size = int(response.headers['Content-Length'])
150        free_space = shutil.disk_usage(os.path.dirname(temp_file)).free
151        if free_space < total_file_size - existing_size:
152            print('Insufficient disk space; download has been halted.')
153            return False
154
155        if existing_size >= total_file_size:
156            print(f"{temp_file_name} has already been downloaded.")
157            return True
158
159        resume_supported = supports_resume(download_url)
160        # If the server supports resuming from breakpoints, it will continue to append modifications to the file.
161        mode = 'ab' if resume_supported else 'wb'
162        flags = os.O_WRONLY | os.O_CREAT | (os.O_APPEND if resume_supported else 0)
163        file_mode = stat.S_IWUSR | stat.S_IRUSR
164
165        with client.stream('GET', download_url, headers=headers) as response:
166            with os.fdopen(os.open(temp_file, flags, file_mode), mode) as temp:
167                download_progress_bar(response, temp, temp_file_name)
168
169    return True
170
171
172def check_gzip_file(file_path):
173    try:
174        with gzip.open(file_path, 'rb') as gzfile:
175            gzfile.read(1)
176    except Exception as e:
177        print(e)
178        return False
179    return True
180
181
182def check_zip_file(file_path):
183    try:
184        if zipfile.is_zipfile(file_path):
185            with zipfile.ZipFile(file_path, 'r') as zip_file:
186                return True
187        else:
188            return False
189    except Exception as e:
190        print(e)
191        return False
192    return True
193
194
195def get_remote_download_name(task_name):
196    if is_windows():
197        if 'sdk' in task_name.lower():
198            return 'ohos-sdk-full.tar.gz'
199        return 'dayu200.tar.gz'
200    elif is_mac():
201        if 'sdk' in task_name.lower():
202            return 'L2-SDK-MAC-FULL.tar.gz'
203        return 'dayu200.tar.gz'
204    else:
205        print('Unsuport platform to get sdk from daily build')
206        return ''
207
208
209def get_api_version(json_path):
210    with open(json_path, 'r') as uni:
211        uni_cont = uni.read()
212        uni_data = json.loads(uni_cont)
213        api_version = uni_data['apiVersion']
214    return api_version
215
216
217def copy_to_output_path(file_name, file_path, output_path_list):
218    update_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'update.py')
219    cmd = ['python', update_file_path]
220    if 'sdk' in file_name.lower():
221        sdk_temp_file = os.path.join(file_path, 'sdk_temp')
222        cmd.extend(['--sdkFilePath', sdk_temp_file])
223        cmd.extend(['--sdkOutputPath'])
224        for output_path in output_path_list:
225            cmd.extend([output_path])
226
227    if 'dayu' in file_name:
228        dayu_temp_file = os.path.join(file_path, 'dayu200_xts')
229        cmd.extend(['--dayuFilePath', dayu_temp_file])
230        cmd.extend(['--dayuOutputPath'])
231        for output_path in output_path_list:
232            cmd.extend([output_path])
233    print(cmd)
234    subprocess.run(cmd, shell=False)
235
236
237def get_the_unzip_file(download_url, save_path):
238    download_name = get_remote_download_name(download_url)
239    download_temp_file = os.path.join(save_path, download_name)
240    if not os.path.exists(save_path):
241        os.mkdir(save_path)
242    print(f'download {download_name} from {download_url}, please wait!!!')
243    success = retry_after_download_failed(download_url, download_temp_file, download_name)
244    if not success:
245        return False
246    if check_gzip_file(download_temp_file):
247        with tarfile.open(download_temp_file, 'r:gz') as tar:
248            print(f'Unpacking {download_temp_file}')
249            if 'dayu' in download_name:
250                save_path = os.path.join(save_path, 'dayu200_xts')
251            tar.extractall(save_path)
252            print(f'Decompression {download_temp_file} completed')
253    elif check_zip_file(download_temp_file):
254        with zipfile.ZipFile(download_temp_file, 'r') as zip_file:
255            print(f'Unpacking {download_temp_file}')
256            zip_file.extractall(save_path)
257            print(f'Decompression {download_temp_file} completed')
258    else:
259        print('The downloaded file is not a valid gzip or zip file.')
260
261    if 'sdk' in download_name:
262        unpack_sdk_file(save_path)
263
264    return True
265
266
267def unpack_sdk_file(path):
268    sdk_zip_path_list = [path, 'windows']
269    if is_mac():
270        sdk_zip_path_list = [path, 'sdk',
271                             'packages', 'ohos-sdk', 'darwin']
272    sdk_floder = os.path.join(path, 'sdk_temp')
273    sdk_zip_path = os.path.join(*sdk_zip_path_list)
274    for item in os.listdir(sdk_zip_path):
275        if item != '.DS_Store':
276            print(f'Unpacking {item}')
277            with zipfile.ZipFile(os.path.join(sdk_zip_path, item)) as zip_file:
278                zip_file.extractall(os.path.join(sdk_floder))
279            print(f'Decompression {item} completed')
280
281
282def get_task_config(file_name):
283    configs = parse_configs()
284    download_list = configs['download_list']
285    for item in download_list:
286        if file_name in item['name']:
287            url = item['url']
288            date = item['date']
289            output_path_list = item['output_path_list']
290    return url, date, output_path_list
291
292
293def delete_redundant_files(task_name, file_name, download_save_path, is_save):
294    if is_save:
295        date_time = re.search(r"\d{8}", file_name).group()
296        new_file_path = os.path.join(download_save_path, f'{date_time}-{task_name}')
297        if not os.path.exists(new_file_path):
298            temp_file_name = 'sdk_temp' if 'sdk' in task_name else 'dayu200_xts'
299            temp_file_path = os.path.join(download_save_path, temp_file_name)
300            os.rename(temp_file_path, new_file_path)
301    subdirs_and_files = [subdir_or_file for subdir_or_file in os.listdir(download_save_path)]
302    for subdir_or_file in subdirs_and_files:
303        if not subdir_or_file[0].isdigit():
304            path = os.path.join(download_save_path, subdir_or_file)
305            if os.path.isdir(path):
306                shutil.rmtree(path)
307            elif os.path.isfile(path):
308                os.remove(path)
309
310
311def write_download_url_to_txt(task_name, download_url):
312    download_url_txt = os.path.join(os.path.dirname(os.path.abspath(__file__)),
313                                    'download_url.txt')
314    flags = os.O_WRONLY | os.O_CREAT
315    mode = stat.S_IWUSR | stat.S_IRUSR
316    with os.fdopen(os.open(download_url_txt, flags, mode), 'a+', encoding='utf-8') as file:
317        file.write(f'{task_name}, {download_url}\n')
318
319
320def get_the_image(task_name, download_url, image_date, output_path_list):
321    configs = parse_configs()
322    is_save = configs.get('is_save')
323    download_save_path = configs.get('download_save_path')
324    if download_url == '':
325        download_url = get_download_url(task_name, image_date)
326        if download_url == 'http://download.ci.openharmony.cn/':
327            print('get download url failed')
328            return False
329    file_name = parse_file_name(download_url)
330    if output_path_list is None:
331        output_path_list = get_task_config(file_name)[2]
332    print(f'output_path_list: {output_path_list}')
333    success = get_the_unzip_file(download_url, download_save_path)
334    if not success:
335        print(f'get {task_name} unzip file failed')
336        return False
337
338    copy_to_output_path(file_name, download_save_path, output_path_list)
339    delete_redundant_files(task_name, file_name, download_save_path, is_save)
340    write_download_url_to_txt(task_name, download_url)
341
342    return True
343
344
345def clean_download_log():
346    download_url_txt = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'download_url.txt')
347    if os.path.exists(download_url_txt):
348        os.remove(download_url_txt)
349
350
351def parse_args():
352    parser = argparse.ArgumentParser()
353    parser.add_argument('--sdkUrl', type=str, dest='sdk_url', nargs='?', const='', default=None,
354                        help='specify which sdk you want to download')
355    parser.add_argument('--dayuUrl', type=str, dest='dayu_url', nargs='?', const='', default=None,
356                        help='specify which dayu200 you want to download')
357    parser.add_argument('--sdkDate', type=str, dest='sdk_date', default=None,
358                        help='specify which day you want to download the sdk')
359    parser.add_argument('--dayuDate', type=str, dest='dayu_date', default=None,
360                        help='specify which day you want to download the dayu')
361    parser.add_argument('--sdkPath', type=str, dest='sdk_path', default=None,
362                        nargs='+',
363                        help='specify where you want to store the sdk')
364    parser.add_argument('--dayuPath', type=str, dest='dayu_path', default=None,
365                        nargs='+',
366                        help='specify where you want to store the dayu200')
367
368    return parser.parse_args()
369
370
371def main():
372    clean_download_log()
373    arguments = parse_args()
374    if arguments.sdk_url is not None:
375        get_the_image('sdk', arguments.sdk_url, arguments.sdk_date, arguments.sdk_path)
376    else:
377        sdk_config = get_task_config('sdk')
378        get_the_image('sdk', sdk_config[0], sdk_config[1], sdk_config[2])
379    if arguments.dayu_url is not None:
380        get_the_image('dayu', arguments.dayu_url, arguments.dayu_date, arguments.dayu_path)
381    else:
382        dayu_config = get_task_config('dayu')
383        get_the_image('dayu', dayu_config[0], dayu_config[1], dayu_config[2])
384
385
386if __name__ == '__main__':
387    main()