1#!/usr/bin/env ruby
2
3# Copyright (c) 2021-2024 Huawei Device Co., Ltd.
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16include_relative 'common.irt'
17
18$available_regs = $temps_mask + :arg0 + :arg1
19# there are 2 temp registers used in the case of arm64 (instead of 3 for
20# x86_64 and arm32) that's why one callee-saved register is added to the mask
21if Options.arch == :arm64
22  $available_regs = $available_regs + :callee1
23end
24
25AllocateObjectTlabValidation = {
26  spills_count_max: { default: 2, arm32: 9999 },  # TODO(msherstennikov): set to 0 once regalloc supports smart temps
27  code_size_max: { arm64: 116, x86_64: 477, arm32: 9999 }
28  # TODO(msherstennikov): revert back code size values, once regalloc supports smart temps
29  # code_size_max: { arm64: 100, x86_64: 125, arm32: 9999 }
30}
31
32AllocateArrayTlabValidation = {
33  spills_count_max: { default: 2, arm32: 9999 },  # TODO(msherstennikov): set to 0 once regalloc supports smart temps
34  code_size_max: { arm64: 136, x86_64: 476, arm32: 9999 }
35  # TODO(msherstennikov): revert back code size values, once regalloc supports smart temps
36  # code_size_max: { arm64: 128, x86_64: 145, arm32: 9999 }
37}
38
39function(:AllocateObjectTlab,
40         params: {klass: 'ref', size: 'word'},
41         regmap: $full_regmap,
42         mode: [:FastPath],
43         regalloc_set: $available_regs,
44         validate: AllocateObjectTlabValidation) {
45
46  if Options.arch == :arm32
47    Intrinsic(:UNREACHABLE).Terminator.void
48    ReturnVoid().void
49    next
50  end
51
52  # Load pointer to the TLAB from TLS
53  tlab_ptr := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr
54
55  # Load pointer to the start address of free memory in the TLAB
56  start := LoadI(tlab_ptr).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
57
58  # Load pointer to the end address of free memory in the TLAB
59  tls_end := LoadI(tlab_ptr).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr
60
61  tls_size := Sub(tls_end, start).word
62
63  If(tls_size, size).LT.Unlikely.b {
64    ep_offset = get_entrypoint_offset("CREATE_OBJECT_BY_CLASS_SLOW_PATH")
65    Intrinsic(:SLOW_PATH_ENTRY, klass, size).AddImm(ep_offset).MethodAsImm("CreateObjectByClassBridge").Terminator.ptr
66    Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG
67  } Else {
68    Intrinsic(:WRITE_TLAB_STATS_SAFE, start, size, Cast(-1).u64).void if defines.DEBUG
69    if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__
70      call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, start, size).void
71    end
72    new_start := Add(start, size).ptr
73    StoreI(start, klass).Imm(Constants::OBJECT_CLASS_OFFSET).ref
74    addr := Add(tlab_ptr, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
75    StoreI(addr, new_start).Imm(0).Volatile.ptr
76    Return(start).ptr
77  }
78}
79
80# Allow three temps, two arguments registers and one callee saved register
81$available_regs = $available_regs + :callee0
82
83def AllocateArrayTlab(element_size)
84  function("AllocateArrayTlab#{element_size}".to_sym,
85           params: {klass: 'ref', elements_num: 'word'},
86           regmap: $full_regmap,
87           mode: [:FastPath],
88           regalloc_set: $available_regs,
89           validate: AllocateArrayTlabValidation) {
90    if Options.arch == :arm32
91      ReturnVoid().void
92      next
93    end
94    elements_num := And(elements_num, "0x00000000ffffffff").word
95    if element_size == 8
96      size := elements_num
97    elsif element_size == 16
98      size := Shl(elements_num, 1).word
99    elsif element_size == 32
100      size := Shl(elements_num, 2).word
101    elsif element_size == 64
102      size := Shl(elements_num, 3).word
103    else
104      raise "Wrong element size #{element_size}"
105    end
106    # Add sizeof(Array) and do align
107    size := Add(size, "DEFAULT_ALIGNMENT_IN_BYTES - 1 + CORETYPES_ARRAY_CLASS_SIZE").word
108    size := And(size, "(~(DEFAULT_ALIGNMENT_IN_BYTES - 1))").word
109
110    # Load pointer to the TLAB from TLS
111    tlab_ptr := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr
112
113    # Load pointer to the start address of free memory in the TLAB
114    start := LoadI(tlab_ptr).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
115
116    # Load pointer to the end address of free memory in the TLAB
117    tls_end := LoadI(tlab_ptr).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr
118
119    tls_size := Sub(tls_end, start).word
120
121    If(tls_size, size).LT.Unlikely.b {
122      ep_offset = get_entrypoint_offset("CREATE_ARRAY_SLOW_PATH")
123      Intrinsic(:SLOW_PATH_ENTRY, klass, elements_num).AddImm(ep_offset).MethodAsImm("CreateArrayBridge").Terminator.ptr
124      Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG
125    } Else {
126      Intrinsic(:WRITE_TLAB_STATS_SAFE, start, size, Cast(-1).u64).void if defines.DEBUG
127      if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__
128        call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, start, size).void
129      end
130      new_start := Add(start, size).ptr
131      StoreI(start, klass).Imm(Constants::OBJECT_CLASS_OFFSET).ref
132      StoreI(start, elements_num).Imm(Constants::ARRAY_LENGTH_OFFSET).word
133      addr := Add(tlab_ptr, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
134      StoreI(addr, new_start).Imm(0).Volatile.ptr
135      Return(start).ptr
136    }
137  }
138end
139
140AllocateArrayTlab(8)
141AllocateArrayTlab(16)
142AllocateArrayTlab(32)
143AllocateArrayTlab(64)
144