1#!/usr/bin/env ruby
2
3# Copyright (c) 2021-2024 Huawei Device Co., Ltd.
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16include_relative 'common.irt'
17
18function(:EmptyPostWriteBarrier,
19        params: {},
20        regmap: $full_regmap,
21        regalloc_set: RegMask.new,
22        mode: [:FastPath]) {
23    if Options.arch == :arm32
24        Intrinsic(:UNREACHABLE).Terminator.void
25        next
26    end
27    ReturnVoid().void
28}
29
30def PostInterGenerationalBarrier(obj_num)
31    if obj_num == 0
32        params = {mem: 'ref_uint'}
33        mask = RegMask.new($full_regmap, :arg0, :tmp0, :tmp1)
34    elsif obj_num == 1
35        params = {mem: 'ref_uint', offset: 'word', obj1: 'ref_uint'}
36        mask = RegMask.new($full_regmap, :arg0, :arg1, :tmp0)
37    elsif obj_num == 2
38        params = {mem: 'ref_uint', offset: 'word', obj1: 'ref_uint', obj2: 'ref_uint'}
39        mask = RegMask.new($full_regmap, :arg0, :arg1, :arg2)
40    else
41        raise "Wrong obj_num #{obj_num}"
42    end
43
44    function("PostInterGenerationalBarrier#{obj_num}".to_sym,
45             params: params,
46             regmap: $full_regmap,
47             regalloc_set: mask,
48             mode: [:FastPath]) {
49        if Options.arch == :arm32
50            Intrinsic(:UNREACHABLE).Terminator.void
51            next
52        end
53
54        min_addr := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_MIN_ADDR_OFFSET).word
55        cards := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_ADDR_OFFSET).ptr
56
57        mem_word := Cast(mem).SrcType(Constants::REF_UINT).word
58        card_offset := ShrI(Sub(mem_word, min_addr).word).Imm(Constants::CARD_TABLE_CARD_BITS).word
59        card := Add(cards, card_offset).ptr
60        StoreI(card, Constants::CARD_DIRTY_VALUE).Imm(Constants::CARD_VALUE_OFFSET).u8
61        ReturnVoid().void
62   }
63end
64
65# G1 PostWrite barrier
66# - Checks if mem and obj are in different regions
67# - Checks if GC Card for mem is not marked, then marks it
68# - Pushes Card to G1 LockFreeBuffer concurrently, GC consumer thread will fetch the Card
69def PostInterRegionBarrierMarkSingleFast()
70    params = {mem: 'ref_uint', offset: 'word', obj1: 'ref_uint'}
71    mask = RegMask.new($full_regmap, :arg0, :arg1, :tmp0)
72
73    function("PostInterRegionBarrierMarkSingleFast".to_sym,
74             params: params,
75             regmap: $full_regmap,
76             regalloc_set: mask,
77             mode: [:FastPath]) {
78        if Options.arch == :arm32
79            Intrinsic(:UNREACHABLE).Terminator.void
80            next
81        end
82
83        If(obj1, 0).EQ.Unlikely {
84            Goto(:Done)
85        }
86
87        If(ShrI(Xor(mem, obj1).ref_uint).Imm(Constants::REGION_SIZE_BIT).ref_uint, 0).EQ.Likely {
88            Goto(:Done)
89        }
90
91        ref_addr := Add(mem, offset).ref_uint
92        LiveOut(ref_addr).DstReg(regmap[:arg0]).ref_uint
93        ep_offset = get_entrypoint_offset("POST_INTER_REGION_BARRIER_SLOW")
94        Intrinsic(:TAIL_CALL).AddImm(ep_offset).MethodAsImm("PostInterRegionBarrierSlow").Terminator.void
95
96    Label(:Done)
97        ReturnVoid().void
98    }
99end
100
101# G1 PostWrite barrier for StorePair instruction.
102# Similar to the previous, but:
103# - Checks if the 2nd object's card should be marked;
104# - If not, checks the 1st object, marks it's card if neccessary and exits.
105# - Otherwise, if 1st and 2nd object's cards are the same (i.e.
106# -     2nd store addr isn't aligned on a beggining of a card), marks the card and exits.
107# - Otherwise, marks the 2nd object's card as well as the 1st obj's card.
108def PostInterRegionBarrierMarkPairFast()
109    params = {mem: 'ref_uint', offset: 'word', obj1: 'ref_uint', obj2: 'ref_uint'}
110    mask = RegMask.new($full_regmap, :arg0, :arg1, :arg2)
111    if Options.arch == :x86_64
112        mask += :tmp0
113    end
114
115    function("PostInterRegionBarrierMarkPairFast".to_sym,
116             params: params,
117             regmap: $full_regmap,
118             regalloc_set: mask,
119             mode: [:FastPath]) {
120        if Options.arch == :arm32
121            Intrinsic(:UNREACHABLE).Terminator.void
122            next
123        end
124
125        
126        ref_addr_obj1 := Add(mem, offset).ref_uint
127        
128        If(obj2, 0).EQ.Unlikely {
129            Goto(:Check1)
130        }
131        ref_addr_obj2 := AddI(ref_addr_obj1.ref_uint).Imm(Constants::REFERENCE_TYPE_SIZE).ref_uint
132
133        If(ShrI(Xor(mem, obj2).ref_uint).Imm(Constants::REGION_SIZE_BIT).ref_uint, 0).NE.Unlikely {
134            If(AndI(ref_addr_obj2).Imm(Constants::CARD_ALIGNMENT_MASK).ref_uint, 0).EQ.Unlikely {
135                LiveOut(ref_addr_obj2).DstReg(regmap[:arg0]).ref_uint
136                ep_offset = get_entrypoint_offset("POST_INTER_REGION_BARRIER_TWO_CARDS_SLOW")
137                Intrinsic(:TAIL_CALL).AddImm(ep_offset).MethodAsImm("PostInterRegionBarrierTwoCardsSlow").Terminator.void
138            }
139            # No need to sub REFERENCE_TYPE_SIZE as the card will be the same:
140            Goto(:Call1)
141        }
142    
143    Label(:Check1)
144        If(obj1, 0).EQ.Unlikely {
145            Goto(:Done)
146        }
147    
148        If(ShrI(Xor(mem, obj1).ref_uint).Imm(Constants::REGION_SIZE_BIT).ref_uint, 0).EQ.Likely {
149            Goto(:Done)
150        }
151
152    Label(:Call1)
153        phi := Phi(ref_addr_obj2, ref_addr_obj1).ref_uint
154        LiveOut(phi).DstReg(regmap[:arg0]).ref_uint
155        ep_offset = get_entrypoint_offset("POST_INTER_REGION_BARRIER_SLOW")
156        Intrinsic(:TAIL_CALL).AddImm(ep_offset).MethodAsImm("PostInterRegionBarrierSlow").Terminator.void
157
158    Label(:Done)
159        ReturnVoid().void
160    }
161end
162
163scoped_macro(:push_dirty_card_to_buffer) do |cards, min_addr, mem|
164    mem_word := Cast(mem).SrcType(Constants::REF_UINT).word
165    card_offset := ShrI(Sub(mem_word, min_addr).word).Imm(Constants::CARD_TABLE_CARD_BITS).word
166    card := Add(cards, card_offset).ptr
167    card_value := LoadI(card).Imm(Constants::CARD_VALUE_OFFSET).u8
168
169    # it is expected that young card cannot be enqueued
170    # so we can avoid card_value & CARD_STATUS_MASK operation
171    is_card_young := Compare(card_value, Constants::CARD_YOUNG_VALUE).EQ.b
172    IfImm(is_card_young).Imm(0).NE.b {
173        Goto(:Done)
174    }
175
176    # StoreLoad barrier is required to guarantee order of previous reference store and card load
177    Intrinsic(:DATA_MEMORY_BARRIER_FULL).void
178    card_value := LoadI(card).Imm(Constants::CARD_VALUE_OFFSET).u8
179    card_status := AndI(card_value).Imm(Constants::CARD_STATUS_MASK).u8
180
181    is_card_clear := Compare(card_status, Constants::CARD_CLEAR_VALUE).EQ.b
182    IfImm(is_card_clear).Imm(0).EQ.Unlikely.b {
183        Goto(:Done)
184    }
185    Intrinsic(:ATOMIC_BYTE_OR, card, Constants::CARD_MARKED_VALUE).void
186
187    hot_bit := AndI(card_value).Imm(Constants::CARD_HOT_FLAG).u8
188    If(hot_bit, Constants::CARD_HOT_FLAG).EQ.b {
189        Goto(:Done)
190    }
191
192    buffer := LoadI(%tr).Imm(Constants::MANAGED_THREAD_G1_POST_BARRIER_BUFFER_OFFSET).ptr
193    buffer_data := AddI(buffer).Imm(Constants::G1_LOCK_BUFFER_DATA_OFFSET).ptr
194    tail_index := LoadI(buffer).Imm(Constants::G1_LOCK_BUFFER_TAIL_OFFSET).word
195    tail_offset := ShlI(tail_index).Imm(Constants::POINTER_LOG_SIZE).word
196    next_tail_index := AndI(AddI(tail_index).Imm(1).word).Imm(Constants::G1_LOCK_BUFFER_SIZE_MASK).word
197
198    Label(:TryPushLoop)
199    head_index := LoadI(buffer).Imm(Constants::G1_LOCK_BUFFER_HEAD_OFFSET).Volatile.word
200    If(next_tail_index, head_index).EQ {
201        Goto(:TryPushLoop)
202    }
203
204    Store(buffer_data, tail_offset, card).ptr
205    StoreI(buffer, next_tail_index).Imm(Constants::G1_LOCK_BUFFER_TAIL_OFFSET).Volatile.word
206
207    Label(:Done)
208end
209
210def PostInterRegionBarrierSlow()
211    if Options.arch == :arm64
212        mask = RegMask.new($full_regmap, :arg0, :callee0, :callee1, :callee2, :tmp0, :tmp1)
213    elsif Options.arch == :x86_64
214        mask = RegMask.new($full_regmap, :arg0, :callee0, :caller0, :caller3, :tmp0, :tmp1)
215    else
216        mask = $panda_mask
217    end
218
219    function(:PostInterRegionBarrierSlow,
220            params: {mem: 'ref_uint'},
221            regmap: $full_regmap,
222            regalloc_set: mask,
223            mode: [:FastPath]) {
224        if Options.arch == :arm32
225            Intrinsic(:UNREACHABLE).Terminator.void
226            next
227        end
228
229        min_addr := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_MIN_ADDR_OFFSET).word
230        cards := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_ADDR_OFFSET).ptr
231        push_dirty_card_to_buffer(cards, min_addr, mem)
232        ReturnVoid().void
233    }
234end
235
236def PostInterRegionBarrierTwoCardsSlow()
237    if Options.arch == :arm64
238        mask = RegMask.new($full_regmap, :arg0, :callee0, :callee1, :callee2, :tmp0, :tmp1)
239    elsif Options.arch == :x86_64
240        mask = RegMask.new($full_regmap, :arg0, :callee0, :caller0, :caller3, :tmp0, :tmp1)
241    else
242        mask = $panda_mask
243    end
244
245    function(:PostInterRegionBarrierTwoCardsSlow,
246            params: {mem: 'ref_uint'},
247            regmap: $full_regmap,
248            regalloc_set: mask,
249            mode: [:FastPath]) {
250        if Options.arch == :arm32
251            Intrinsic(:UNREACHABLE).Terminator.void
252            next
253        end
254
255        min_addr := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_MIN_ADDR_OFFSET).word
256        cards := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_ADDR_OFFSET).ptr
257        # `mem` here is a pointer to the 2nd slot of the pair
258        push_dirty_card_to_buffer(cards, min_addr, mem)
259        push_dirty_card_to_buffer(cards, min_addr, SubI(mem).Imm(Constants::REFERENCE_TYPE_SIZE).ref_uint)
260        ReturnVoid().void
261    }
262end
263
264PostInterRegionBarrierSlow()
265PostInterRegionBarrierTwoCardsSlow()
266
267PostInterGenerationalBarrier(0)
268PostInterGenerationalBarrier(1)
269PostInterGenerationalBarrier(2)
270
271PostInterRegionBarrierMarkSingleFast()
272PostInterRegionBarrierMarkPairFast()
273