1# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 2# Licensed under the Apache License, Version 2.0 (the "License"); 3# you may not use this file except in compliance with the License. 4# You may obtain a copy of the License at 5# 6# http://www.apache.org/licenses/LICENSE-2.0 7# 8# Unless required by applicable law or agreed to in writing, software 9# distributed under the License is distributed on an "AS IS" BASIS, 10# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11# See the License for the specific language governing permissions and 12# limitations under the License. 13# Building and running benchmarks in Panda Assembly from the Panda source tree. 14 15if(NOT DEFINED PANDA_ROOT) 16 message(FATAL_ERROR "Not a Panda build") 17endif() 18 19define_property(SOURCE 20 PROPERTY panda_benchmark_name 21 BRIEF_DOCS "Short benchmark name" 22 FULL_DOCS "Short benchmark name") 23define_property(SOURCE 24 PROPERTY panda_benchmark_vmb_name 25 BRIEF_DOCS "Short VMB name" 26 FULL_DOCS "Short VMB name") 27define_property(SOURCE 28 PROPERTY panda_benchmark_stack_limit 29 BRIEF_DOCS "Stack limit for this benchmark" 30 FULL_DOCS "Stack limit for this benchmark in bytes. Set 0 to use default value") 31define_property(SOURCE 32 PROPERTY panda_benchmark_run_interpreter 33 BRIEF_DOCS "Run this benchmark with interpreter?" 34 FULL_DOCS "Run this benchmark with interpreter?") 35define_property(SOURCE 36 PROPERTY panda_benchmark_run_enforce_jit_compiler 37 BRIEF_DOCS "Run this benchmark with enforced JIT compiler?" 38 FULL_DOCS "Run this benchmark with enforced JIT compiler?") 39define_property(SOURCE 40 PROPERTY panda_benchmark_run_gc_type 41 BRIEF_DOCS "Type of GC?" 42 FULL_DOCS "Type of GC?") 43define_property(SOURCE 44 PROPERTY panda_benchmark_run_aot 45 BRIEF_DOCS "Run with enabled AOT?" 46 FULL_DOCS "Run with enabled AOT?") 47define_property(SOURCE 48 PROPERTY panda_benchmark_run_default 49 BRIEF_DOCS "Run in default mode?" 50 FULL_DOCS "Run in default mode?") 51 52set(PANDA_BENCHMARKS) 53set(PANDA_BENCHMARKS_DIR "${CMAKE_CURRENT_LIST_DIR}") 54function(panda_add_benchmark 55 name vmb_name stack_limit interpreter enforce_jit_compiler gc_type aot default) 56 # Allow to write something like this: 64 * 1024 * 1024 57 math(EXPR stack_limit "${stack_limit}") 58 59 set(file_name "${PANDA_BENCHMARKS_DIR}/${name}.pa") 60 61 set_source_files_properties(${file_name} PROPERTIES 62 panda_benchmark_name "${name}" 63 panda_benchmark_vmb_name "${vmb_name}" 64 panda_benchmark_stack_limit "${stack_limit}" 65 panda_benchmark_run_interpreter ${interpreter} 66 panda_benchmark_run_enforce_jit_compiler ${enforce_jit_compiler} 67 panda_benchmark_run_gc_type ${gc_type} 68 panda_benchmark_run_aot ${aot} 69 panda_benchmark_run_default ${default}) 70 71 list(APPEND PANDA_BENCHMARKS ${file_name}) 72 set (PANDA_BENCHMARKS ${PANDA_BENCHMARKS} PARENT_SCOPE) 73endfunction() 74 75# Arguments: 76# * 1. Panda benchmark name 77# * 2. VMB benchmark name; if empty, not a part of VMB yet 78# * 3. Stack limit 79# * 4. Run with interpreter (without compiler at all)? 80# * 5. Run with enforced JIT compiler? 81# * 6. Gc type (default - for default gc) 82# * 7. Run in AOT mode? 83# * 8. Run in default mode? 84# TODO: Make all benchmarks into VMB 85# 1.name 2.vmb_name 3.stack_limit 4.interpreter 86# | | | | 5.enforce_jit_compiler 87# | | | | | 6.gc_type 88# | | | | | | 7.aot 89# | | | | | | | 8.default 90panda_add_benchmark("3d-morph" "Morph3d" 0 TRUE TRUE default TRUE TRUE) 91panda_add_benchmark("access-binary-trees" "AccessBinaryTrees" 0 TRUE TRUE default TRUE TRUE) 92panda_add_benchmark("access-fannkuch" "AccessFannkuch" 0 TRUE TRUE default TRUE TRUE) 93panda_add_benchmark("access-nbody" "AccessNBody" 0 TRUE TRUE default TRUE TRUE) 94panda_add_benchmark("access-nsieve" "AccessNSieve" 0 TRUE TRUE default TRUE TRUE) 95panda_add_benchmark("bitops-3bit-bits-in-byte" "Bitops3BitBitsInByte" 0 TRUE TRUE default TRUE TRUE) 96panda_add_benchmark("bitops-bits-in-byte" "BitopsBitsInByte" 0 TRUE TRUE default TRUE TRUE) 97panda_add_benchmark("bitops-bitwise-and" "BitopsBitwiseAnd" 0 TRUE TRUE default TRUE TRUE) 98panda_add_benchmark("bitops-nsieve-bits" "BitopsNSieveBits" 0 TRUE TRUE default TRUE TRUE) 99panda_add_benchmark("controlflow-recursive" "ControlFlowRecursive" "384 * 1024 * 1024" TRUE TRUE default TRUE TRUE) 100panda_add_benchmark("math-cordic" "MathCordic" 0 TRUE TRUE default TRUE TRUE) 101panda_add_benchmark("math-partial-sums" "MathPartialSums" 0 TRUE TRUE default TRUE TRUE) 102panda_add_benchmark("math-spectral-norm" "MathSpectralNorm" 0 TRUE TRUE default TRUE TRUE) 103 104add_custom_target(benchmarks-panda-assembly-interpreter 105 COMMENT "Running benchmarks in Panda Assembly in interpreter") 106add_custom_target(benchmarks-panda-assembly-enforce-jit-compiler 107 COMMENT "Running benchmarks in Panda Assembly with enforced JIT") 108add_custom_target(benchmarks-panda-assembly-aot 109 COMMENT "Running benchmarks in Panda Assembly with AOT") 110add_custom_target(benchmarks-panda-assembly-default 111 COMMENT "Running benchmarks in Panda Assembly") 112 113add_dependencies(benchmarks 114 benchmarks-panda-assembly-interpreter 115 benchmarks-panda-assembly-enforce-jit-compiler 116 benchmarks-panda-assembly-aot 117 benchmarks-panda-assembly-default 118) 119 120add_custom_target(benchmarks-panda-assembly-aot-stats 121 COMMAND python3 ${CMAKE_SOURCE_DIR}/scripts/extras/mem_usage_analysis.py ${COMPILER_STATS_DIR} --mode=default --output=${COMPILER_STATS_DIR}/../report.html 122 COMMENT "Gathering compiler's statistics in benchmarks-AOT") 123 124function(add_benchmark 125 common_target_basename name file interpreter enforce_jit_compiler stack_limit gc_type aot default) 126 127 if (stack_limit) 128 set(PANDA_RUN_PREFIX prlimit --stack=${stack_limit} ${PANDA_RUN_PREFIX}) 129 endif() 130 131 if (NOT "${gc_type}" STREQUAL "default") 132 set(ENABLE_GC_OPTION "--gc-type=${gc_type}") 133 endif() 134 135 set(source_language "core") 136 set(entry_point "_GLOBAL::main") 137 set(skip_build FALSE) 138 139 string(TOLOWER "${CMAKE_BUILD_TYPE}" cm_build_type) 140 if ("${cm_build_type}" STREQUAL "") 141 set(cm_build_type "debug") 142 endif() 143 if ("${cm_build_type}" STREQUAL "debug") 144 set(benchmark_timeout "5s") 145 endif() 146 if ("${cm_build_type}" STREQUAL "fastverify") 147 set(benchmark_timeout "5s") 148 endif() 149 150 set(subdir_prefix "benchmarks/${common_target_basename}") 151 152 if (interpreter) 153 panda_add_asm_file(TARGET "${name}-interpreter" 154 FILE "${file}" 155 LANGUAGE_CONTEXT "${source_language}" 156 SUBDIR "${subdir_prefix}" 157 ENTRY "${entry_point}" 158 TIMEOUT "${benchmark_timeout}" 159 SKIP_BUILD ${skip_build} 160 DEPENDS ${deps} 161 AOT_MODE FALSE 162 RUNTIME_OPTIONS ${ENABLE_GC_OPTION} "--compiler-enable-jit=false" "--limit-standard-alloc=true") 163 add_dependencies(${common_target_basename}-interpreter "${name}-interpreter") 164 endif() 165 166 set(CUR_COMPILER_OPTIONS "") 167 if (PANDA_TARGET_ARM32) 168 set(CUR_COMPILER_OPTIONS "--compiler-ignore-failures=true") 169 endif() 170 171 if(PANDA_COMPILER_ENABLE) 172 list(APPEND deps arkcompiler) 173 if (enforce_jit_compiler) 174 panda_add_asm_file(TARGET "${name}-enforce-jit-compiler" 175 FILE "${file}" 176 LANGUAGE_CONTEXT "${source_language}" 177 SUBDIR "${subdir_prefix}" 178 ENTRY "${entry_point}" 179 TIMEOUT "${benchmark_timeout}" 180 SKIP_BUILD ${skip_build} 181 DEPENDS ${deps} 182 AOT_MODE FALSE 183 COMPILER_OPTIONS ${CUR_COMPILER_OPTIONS} 184 RUNTIME_OPTIONS ${ENABLE_GC_OPTION} "--compiler-enable-jit=true" "--compiler-hotness-threshold=0" "--limit-standard-alloc=true") 185 add_dependencies(${common_target_basename}-enforce-jit-compiler "${name}-enforce-jit-compiler") 186 endif() 187 188 if (aot AND (PANDA_TARGET_ARM64 OR PANDA_TARGET_AMD64)) 189 panda_add_asm_file(TARGET "${name}-aot" 190 FILE "${file}" 191 LANGUAGE_CONTEXT "${source_language}" 192 SUBDIR "${subdir_prefix}" 193 ENTRY "${entry_point}" 194 TIMEOUT "${benchmark_timeout}" 195 SKIP_BUILD ${skip_build} 196 DEPENDS ${deps} 197 AOT_MODE TRUE 198 AOT_STATS TRUE 199 AOT_GC_OPTION ${ENABLE_GC_OPTION} 200 RUNTIME_OPTIONS ${ENABLE_GC_OPTION} "--compiler-enable-jit=true") 201 add_dependencies(${common_target_basename}-aot "${name}-aot") 202 add_dependencies(${common_target_basename}-aot-stats "${name}-aot-stats") 203 endif() 204 205 if (default) 206 panda_add_asm_file(TARGET "${name}-default" 207 FILE "${file}" 208 LANGUAGE_CONTEXT "${source_language}" 209 SUBDIR "${subdir_prefix}" 210 ENTRY "${entry_point}" 211 TIMEOUT "${benchmark_timeout}" 212 SKIP_BUILD ${skip_build} 213 DEPENDS ${deps} 214 AOT_MODE FALSE 215 COMPILER_OPTIONS ${CUR_COMPILER_OPTIONS} 216 RUNTIME_OPTIONS ${ENABLE_GC_OPTION} "--compiler-enable-jit=true") 217 add_dependencies(${common_target_basename}-default "${name}-default") 218 endif() 219 endif() 220endfunction() 221 222foreach(benchmark ${PANDA_BENCHMARKS}) 223 get_source_file_property(name "${benchmark}" panda_benchmark_name) 224 get_source_file_property(interpreter "${benchmark}" panda_benchmark_run_interpreter) 225 get_source_file_property(gc_type "${benchmark}" panda_benchmark_run_gc_type) 226 get_source_file_property(enforce_jit_compiler "${benchmark}" panda_benchmark_run_enforce_jit_compiler) 227 get_source_file_property(stack_limit "${benchmark}" panda_benchmark_stack_limit) 228 get_source_file_property(aot "${benchmark}" panda_benchmark_run_aot) 229 get_source_file_property(default "${benchmark}" panda_benchmark_run_default) 230 231 add_benchmark(benchmarks-panda-assembly "${name}" "${benchmark}" 232 ${interpreter} 233 ${enforce_jit_compiler} 234 ${stack_limit} 235 ${gc_type} 236 ${aot} 237 ${default}) 238endforeach() 239 240# Build-time fix-up for applying most recent changes in Panda Assembly 241# benchmarks to the source tree of VM Benchmarks aka VMB. To be used in CI only. 242# Currently VMB owns almost all benchmarks and infrastructure for running them, 243# but Panda Assembly benchmarks are maintained within Panda source tree. 244# This function generates two shell scripts which copy all necessary files 245# between the source trees. Dedicated targets are created for 246# invoking the scripts. Path to the root of VMB repository is expected to be 247# passed via VMB_REPO_ROOT environment variable during build time. 248# Targets: 249# * update-vmb: Update all benchmarks. 250# * update-vmb-jit: Update all benchmarks which can be JIT-compiled. 251# Benchmarks that cannot be JIT-compiled will be 252# copied as VMBenchmarkName.pa.disabled. 253function(panda_update_vmb) 254 set(vmb_benchmark_path "\$VMB_REPO_ROOT/benchmarks/components/SimpleLanguage/SimpleLanguage") 255 set(script_name_all "${CMAKE_CURRENT_BINARY_DIR}/update-vmb.sh") 256 set(script_name_jit "${CMAKE_CURRENT_BINARY_DIR}/update-vmb-jit.sh") 257 # NB! Please do not use ';' character in the body of the update script. 258 set(script_all 259 "#!/bin/sh" 260 "set -e" 261 "[ -d \"\$VMB_REPO_ROOT\" ] || exit 1" 262 ) 263 set(script_jit 264 "#!/bin/sh" 265 "set -e" 266 "[ -d \"\$VMB_REPO_ROOT\" ] || exit 1" 267 ) 268 269 foreach(benchmark ${PANDA_BENCHMARKS}) 270 get_source_file_property(vmb_name "${benchmark}" panda_benchmark_vmb_name) 271 get_source_file_property(enforce_jit_compiler "${benchmark}" panda_benchmark_run_enforce_jit_compiler) 272 273 if(NOT "${vmb_name}" STREQUAL "") 274 list(APPEND script_all 275 "cp \"${benchmark}\" \"${vmb_benchmark_path}/${vmb_name}/pa/${vmb_name}.pa\"") 276 277 if(enforce_jit_compiler) 278 list(APPEND script_jit 279 "cp \"${benchmark}\" \"${vmb_benchmark_path}/${vmb_name}/pa/${vmb_name}.pa\"") 280 else() 281 list(APPEND script_jit 282 "cp \"${benchmark}\" \"${vmb_benchmark_path}/${vmb_name}/pa/${vmb_name}.pa.disabled\"") 283 list(APPEND script_jit 284 "rm -f \"${vmb_benchmark_path}/${vmb_name}/pa/${vmb_name}.pa\"") 285 endif() 286 endif() 287 endforeach() 288 289 string(REPLACE ";" "\n" script_all "${script_all}") 290 string(REPLACE ";" "\n" script_jit "${script_jit}") 291 file(GENERATE OUTPUT ${script_name_all} CONTENT "${script_all}") 292 file(GENERATE OUTPUT ${script_name_jit} CONTENT "${script_jit}") 293 add_custom_target(update-vmb 294 COMMENT "Update VMB, all benchmarks" 295 COMMAND sh ${script_name_all} 296 ) 297 add_custom_target(update-vmb-jit 298 COMMENT "Update VMB, only JITtable benchmarks" 299 COMMAND sh ${script_name_jit} 300 ) 301endfunction() 302 303panda_update_vmb() 304