162306a36Sopenharmony_ci// SPDX-License-Identifier: MIT
262306a36Sopenharmony_ci/*
362306a36Sopenharmony_ci * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst)
462306a36Sopenharmony_ci *
562306a36Sopenharmony_ci * Based on bo.c which bears the following copyright notice,
662306a36Sopenharmony_ci * but is dual licensed:
762306a36Sopenharmony_ci *
862306a36Sopenharmony_ci * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
962306a36Sopenharmony_ci * All Rights Reserved.
1062306a36Sopenharmony_ci *
1162306a36Sopenharmony_ci * Permission is hereby granted, free of charge, to any person obtaining a
1262306a36Sopenharmony_ci * copy of this software and associated documentation files (the
1362306a36Sopenharmony_ci * "Software"), to deal in the Software without restriction, including
1462306a36Sopenharmony_ci * without limitation the rights to use, copy, modify, merge, publish,
1562306a36Sopenharmony_ci * distribute, sub license, and/or sell copies of the Software, and to
1662306a36Sopenharmony_ci * permit persons to whom the Software is furnished to do so, subject to
1762306a36Sopenharmony_ci * the following conditions:
1862306a36Sopenharmony_ci *
1962306a36Sopenharmony_ci * The above copyright notice and this permission notice (including the
2062306a36Sopenharmony_ci * next paragraph) shall be included in all copies or substantial portions
2162306a36Sopenharmony_ci * of the Software.
2262306a36Sopenharmony_ci *
2362306a36Sopenharmony_ci * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
2462306a36Sopenharmony_ci * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
2562306a36Sopenharmony_ci * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
2662306a36Sopenharmony_ci * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
2762306a36Sopenharmony_ci * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
2862306a36Sopenharmony_ci * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
2962306a36Sopenharmony_ci * USE OR OTHER DEALINGS IN THE SOFTWARE.
3062306a36Sopenharmony_ci *
3162306a36Sopenharmony_ci **************************************************************************/
3262306a36Sopenharmony_ci/*
3362306a36Sopenharmony_ci * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
3462306a36Sopenharmony_ci */
3562306a36Sopenharmony_ci
3662306a36Sopenharmony_ci#include <linux/dma-resv.h>
3762306a36Sopenharmony_ci#include <linux/dma-fence-array.h>
3862306a36Sopenharmony_ci#include <linux/export.h>
3962306a36Sopenharmony_ci#include <linux/mm.h>
4062306a36Sopenharmony_ci#include <linux/sched/mm.h>
4162306a36Sopenharmony_ci#include <linux/mmu_notifier.h>
4262306a36Sopenharmony_ci#include <linux/seq_file.h>
4362306a36Sopenharmony_ci
4462306a36Sopenharmony_ci/**
4562306a36Sopenharmony_ci * DOC: Reservation Object Overview
4662306a36Sopenharmony_ci *
4762306a36Sopenharmony_ci * The reservation object provides a mechanism to manage a container of
4862306a36Sopenharmony_ci * dma_fence object associated with a resource. A reservation object
4962306a36Sopenharmony_ci * can have any number of fences attaches to it. Each fence carries an usage
5062306a36Sopenharmony_ci * parameter determining how the operation represented by the fence is using the
5162306a36Sopenharmony_ci * resource. The RCU mechanism is used to protect read access to fences from
5262306a36Sopenharmony_ci * locked write-side updates.
5362306a36Sopenharmony_ci *
5462306a36Sopenharmony_ci * See struct dma_resv for more details.
5562306a36Sopenharmony_ci */
5662306a36Sopenharmony_ci
5762306a36Sopenharmony_ciDEFINE_WD_CLASS(reservation_ww_class);
5862306a36Sopenharmony_ciEXPORT_SYMBOL(reservation_ww_class);
5962306a36Sopenharmony_ci
6062306a36Sopenharmony_ci/* Mask for the lower fence pointer bits */
6162306a36Sopenharmony_ci#define DMA_RESV_LIST_MASK	0x3
6262306a36Sopenharmony_ci
6362306a36Sopenharmony_cistruct dma_resv_list {
6462306a36Sopenharmony_ci	struct rcu_head rcu;
6562306a36Sopenharmony_ci	u32 num_fences, max_fences;
6662306a36Sopenharmony_ci	struct dma_fence __rcu *table[];
6762306a36Sopenharmony_ci};
6862306a36Sopenharmony_ci
6962306a36Sopenharmony_ci/* Extract the fence and usage flags from an RCU protected entry in the list. */
7062306a36Sopenharmony_cistatic void dma_resv_list_entry(struct dma_resv_list *list, unsigned int index,
7162306a36Sopenharmony_ci				struct dma_resv *resv, struct dma_fence **fence,
7262306a36Sopenharmony_ci				enum dma_resv_usage *usage)
7362306a36Sopenharmony_ci{
7462306a36Sopenharmony_ci	long tmp;
7562306a36Sopenharmony_ci
7662306a36Sopenharmony_ci	tmp = (long)rcu_dereference_check(list->table[index],
7762306a36Sopenharmony_ci					  resv ? dma_resv_held(resv) : true);
7862306a36Sopenharmony_ci	*fence = (struct dma_fence *)(tmp & ~DMA_RESV_LIST_MASK);
7962306a36Sopenharmony_ci	if (usage)
8062306a36Sopenharmony_ci		*usage = tmp & DMA_RESV_LIST_MASK;
8162306a36Sopenharmony_ci}
8262306a36Sopenharmony_ci
8362306a36Sopenharmony_ci/* Set the fence and usage flags at the specific index in the list. */
8462306a36Sopenharmony_cistatic void dma_resv_list_set(struct dma_resv_list *list,
8562306a36Sopenharmony_ci			      unsigned int index,
8662306a36Sopenharmony_ci			      struct dma_fence *fence,
8762306a36Sopenharmony_ci			      enum dma_resv_usage usage)
8862306a36Sopenharmony_ci{
8962306a36Sopenharmony_ci	long tmp = ((long)fence) | usage;
9062306a36Sopenharmony_ci
9162306a36Sopenharmony_ci	RCU_INIT_POINTER(list->table[index], (struct dma_fence *)tmp);
9262306a36Sopenharmony_ci}
9362306a36Sopenharmony_ci
9462306a36Sopenharmony_ci/*
9562306a36Sopenharmony_ci * Allocate a new dma_resv_list and make sure to correctly initialize
9662306a36Sopenharmony_ci * max_fences.
9762306a36Sopenharmony_ci */
9862306a36Sopenharmony_cistatic struct dma_resv_list *dma_resv_list_alloc(unsigned int max_fences)
9962306a36Sopenharmony_ci{
10062306a36Sopenharmony_ci	struct dma_resv_list *list;
10162306a36Sopenharmony_ci	size_t size;
10262306a36Sopenharmony_ci
10362306a36Sopenharmony_ci	/* Round up to the next kmalloc bucket size. */
10462306a36Sopenharmony_ci	size = kmalloc_size_roundup(struct_size(list, table, max_fences));
10562306a36Sopenharmony_ci
10662306a36Sopenharmony_ci	list = kmalloc(size, GFP_KERNEL);
10762306a36Sopenharmony_ci	if (!list)
10862306a36Sopenharmony_ci		return NULL;
10962306a36Sopenharmony_ci
11062306a36Sopenharmony_ci	/* Given the resulting bucket size, recalculated max_fences. */
11162306a36Sopenharmony_ci	list->max_fences = (size - offsetof(typeof(*list), table)) /
11262306a36Sopenharmony_ci		sizeof(*list->table);
11362306a36Sopenharmony_ci
11462306a36Sopenharmony_ci	return list;
11562306a36Sopenharmony_ci}
11662306a36Sopenharmony_ci
11762306a36Sopenharmony_ci/* Free a dma_resv_list and make sure to drop all references. */
11862306a36Sopenharmony_cistatic void dma_resv_list_free(struct dma_resv_list *list)
11962306a36Sopenharmony_ci{
12062306a36Sopenharmony_ci	unsigned int i;
12162306a36Sopenharmony_ci
12262306a36Sopenharmony_ci	if (!list)
12362306a36Sopenharmony_ci		return;
12462306a36Sopenharmony_ci
12562306a36Sopenharmony_ci	for (i = 0; i < list->num_fences; ++i) {
12662306a36Sopenharmony_ci		struct dma_fence *fence;
12762306a36Sopenharmony_ci
12862306a36Sopenharmony_ci		dma_resv_list_entry(list, i, NULL, &fence, NULL);
12962306a36Sopenharmony_ci		dma_fence_put(fence);
13062306a36Sopenharmony_ci	}
13162306a36Sopenharmony_ci	kfree_rcu(list, rcu);
13262306a36Sopenharmony_ci}
13362306a36Sopenharmony_ci
13462306a36Sopenharmony_ci/**
13562306a36Sopenharmony_ci * dma_resv_init - initialize a reservation object
13662306a36Sopenharmony_ci * @obj: the reservation object
13762306a36Sopenharmony_ci */
13862306a36Sopenharmony_civoid dma_resv_init(struct dma_resv *obj)
13962306a36Sopenharmony_ci{
14062306a36Sopenharmony_ci	ww_mutex_init(&obj->lock, &reservation_ww_class);
14162306a36Sopenharmony_ci
14262306a36Sopenharmony_ci	RCU_INIT_POINTER(obj->fences, NULL);
14362306a36Sopenharmony_ci}
14462306a36Sopenharmony_ciEXPORT_SYMBOL(dma_resv_init);
14562306a36Sopenharmony_ci
14662306a36Sopenharmony_ci/**
14762306a36Sopenharmony_ci * dma_resv_fini - destroys a reservation object
14862306a36Sopenharmony_ci * @obj: the reservation object
14962306a36Sopenharmony_ci */
15062306a36Sopenharmony_civoid dma_resv_fini(struct dma_resv *obj)
15162306a36Sopenharmony_ci{
15262306a36Sopenharmony_ci	/*
15362306a36Sopenharmony_ci	 * This object should be dead and all references must have
15462306a36Sopenharmony_ci	 * been released to it, so no need to be protected with rcu.
15562306a36Sopenharmony_ci	 */
15662306a36Sopenharmony_ci	dma_resv_list_free(rcu_dereference_protected(obj->fences, true));
15762306a36Sopenharmony_ci	ww_mutex_destroy(&obj->lock);
15862306a36Sopenharmony_ci}
15962306a36Sopenharmony_ciEXPORT_SYMBOL(dma_resv_fini);
16062306a36Sopenharmony_ci
16162306a36Sopenharmony_ci/* Dereference the fences while ensuring RCU rules */
16262306a36Sopenharmony_cistatic inline struct dma_resv_list *dma_resv_fences_list(struct dma_resv *obj)
16362306a36Sopenharmony_ci{
16462306a36Sopenharmony_ci	return rcu_dereference_check(obj->fences, dma_resv_held(obj));
16562306a36Sopenharmony_ci}
16662306a36Sopenharmony_ci
16762306a36Sopenharmony_ci/**
16862306a36Sopenharmony_ci * dma_resv_reserve_fences - Reserve space to add fences to a dma_resv object.
16962306a36Sopenharmony_ci * @obj: reservation object
17062306a36Sopenharmony_ci * @num_fences: number of fences we want to add
17162306a36Sopenharmony_ci *
17262306a36Sopenharmony_ci * Should be called before dma_resv_add_fence().  Must be called with @obj
17362306a36Sopenharmony_ci * locked through dma_resv_lock().
17462306a36Sopenharmony_ci *
17562306a36Sopenharmony_ci * Note that the preallocated slots need to be re-reserved if @obj is unlocked
17662306a36Sopenharmony_ci * at any time before calling dma_resv_add_fence(). This is validated when
17762306a36Sopenharmony_ci * CONFIG_DEBUG_MUTEXES is enabled.
17862306a36Sopenharmony_ci *
17962306a36Sopenharmony_ci * RETURNS
18062306a36Sopenharmony_ci * Zero for success, or -errno
18162306a36Sopenharmony_ci */
18262306a36Sopenharmony_ciint dma_resv_reserve_fences(struct dma_resv *obj, unsigned int num_fences)
18362306a36Sopenharmony_ci{
18462306a36Sopenharmony_ci	struct dma_resv_list *old, *new;
18562306a36Sopenharmony_ci	unsigned int i, j, k, max;
18662306a36Sopenharmony_ci
18762306a36Sopenharmony_ci	dma_resv_assert_held(obj);
18862306a36Sopenharmony_ci
18962306a36Sopenharmony_ci	old = dma_resv_fences_list(obj);
19062306a36Sopenharmony_ci	if (old && old->max_fences) {
19162306a36Sopenharmony_ci		if ((old->num_fences + num_fences) <= old->max_fences)
19262306a36Sopenharmony_ci			return 0;
19362306a36Sopenharmony_ci		max = max(old->num_fences + num_fences, old->max_fences * 2);
19462306a36Sopenharmony_ci	} else {
19562306a36Sopenharmony_ci		max = max(4ul, roundup_pow_of_two(num_fences));
19662306a36Sopenharmony_ci	}
19762306a36Sopenharmony_ci
19862306a36Sopenharmony_ci	new = dma_resv_list_alloc(max);
19962306a36Sopenharmony_ci	if (!new)
20062306a36Sopenharmony_ci		return -ENOMEM;
20162306a36Sopenharmony_ci
20262306a36Sopenharmony_ci	/*
20362306a36Sopenharmony_ci	 * no need to bump fence refcounts, rcu_read access
20462306a36Sopenharmony_ci	 * requires the use of kref_get_unless_zero, and the
20562306a36Sopenharmony_ci	 * references from the old struct are carried over to
20662306a36Sopenharmony_ci	 * the new.
20762306a36Sopenharmony_ci	 */
20862306a36Sopenharmony_ci	for (i = 0, j = 0, k = max; i < (old ? old->num_fences : 0); ++i) {
20962306a36Sopenharmony_ci		enum dma_resv_usage usage;
21062306a36Sopenharmony_ci		struct dma_fence *fence;
21162306a36Sopenharmony_ci
21262306a36Sopenharmony_ci		dma_resv_list_entry(old, i, obj, &fence, &usage);
21362306a36Sopenharmony_ci		if (dma_fence_is_signaled(fence))
21462306a36Sopenharmony_ci			RCU_INIT_POINTER(new->table[--k], fence);
21562306a36Sopenharmony_ci		else
21662306a36Sopenharmony_ci			dma_resv_list_set(new, j++, fence, usage);
21762306a36Sopenharmony_ci	}
21862306a36Sopenharmony_ci	new->num_fences = j;
21962306a36Sopenharmony_ci
22062306a36Sopenharmony_ci	/*
22162306a36Sopenharmony_ci	 * We are not changing the effective set of fences here so can
22262306a36Sopenharmony_ci	 * merely update the pointer to the new array; both existing
22362306a36Sopenharmony_ci	 * readers and new readers will see exactly the same set of
22462306a36Sopenharmony_ci	 * active (unsignaled) fences. Individual fences and the
22562306a36Sopenharmony_ci	 * old array are protected by RCU and so will not vanish under
22662306a36Sopenharmony_ci	 * the gaze of the rcu_read_lock() readers.
22762306a36Sopenharmony_ci	 */
22862306a36Sopenharmony_ci	rcu_assign_pointer(obj->fences, new);
22962306a36Sopenharmony_ci
23062306a36Sopenharmony_ci	if (!old)
23162306a36Sopenharmony_ci		return 0;
23262306a36Sopenharmony_ci
23362306a36Sopenharmony_ci	/* Drop the references to the signaled fences */
23462306a36Sopenharmony_ci	for (i = k; i < max; ++i) {
23562306a36Sopenharmony_ci		struct dma_fence *fence;
23662306a36Sopenharmony_ci
23762306a36Sopenharmony_ci		fence = rcu_dereference_protected(new->table[i],
23862306a36Sopenharmony_ci						  dma_resv_held(obj));
23962306a36Sopenharmony_ci		dma_fence_put(fence);
24062306a36Sopenharmony_ci	}
24162306a36Sopenharmony_ci	kfree_rcu(old, rcu);
24262306a36Sopenharmony_ci
24362306a36Sopenharmony_ci	return 0;
24462306a36Sopenharmony_ci}
24562306a36Sopenharmony_ciEXPORT_SYMBOL(dma_resv_reserve_fences);
24662306a36Sopenharmony_ci
24762306a36Sopenharmony_ci#ifdef CONFIG_DEBUG_MUTEXES
24862306a36Sopenharmony_ci/**
24962306a36Sopenharmony_ci * dma_resv_reset_max_fences - reset fences for debugging
25062306a36Sopenharmony_ci * @obj: the dma_resv object to reset
25162306a36Sopenharmony_ci *
25262306a36Sopenharmony_ci * Reset the number of pre-reserved fence slots to test that drivers do
25362306a36Sopenharmony_ci * correct slot allocation using dma_resv_reserve_fences(). See also
25462306a36Sopenharmony_ci * &dma_resv_list.max_fences.
25562306a36Sopenharmony_ci */
25662306a36Sopenharmony_civoid dma_resv_reset_max_fences(struct dma_resv *obj)
25762306a36Sopenharmony_ci{
25862306a36Sopenharmony_ci	struct dma_resv_list *fences = dma_resv_fences_list(obj);
25962306a36Sopenharmony_ci
26062306a36Sopenharmony_ci	dma_resv_assert_held(obj);
26162306a36Sopenharmony_ci
26262306a36Sopenharmony_ci	/* Test fence slot reservation */
26362306a36Sopenharmony_ci	if (fences)
26462306a36Sopenharmony_ci		fences->max_fences = fences->num_fences;
26562306a36Sopenharmony_ci}
26662306a36Sopenharmony_ciEXPORT_SYMBOL(dma_resv_reset_max_fences);
26762306a36Sopenharmony_ci#endif
26862306a36Sopenharmony_ci
26962306a36Sopenharmony_ci/**
27062306a36Sopenharmony_ci * dma_resv_add_fence - Add a fence to the dma_resv obj
27162306a36Sopenharmony_ci * @obj: the reservation object
27262306a36Sopenharmony_ci * @fence: the fence to add
27362306a36Sopenharmony_ci * @usage: how the fence is used, see enum dma_resv_usage
27462306a36Sopenharmony_ci *
27562306a36Sopenharmony_ci * Add a fence to a slot, @obj must be locked with dma_resv_lock(), and
27662306a36Sopenharmony_ci * dma_resv_reserve_fences() has been called.
27762306a36Sopenharmony_ci *
27862306a36Sopenharmony_ci * See also &dma_resv.fence for a discussion of the semantics.
27962306a36Sopenharmony_ci */
28062306a36Sopenharmony_civoid dma_resv_add_fence(struct dma_resv *obj, struct dma_fence *fence,
28162306a36Sopenharmony_ci			enum dma_resv_usage usage)
28262306a36Sopenharmony_ci{
28362306a36Sopenharmony_ci	struct dma_resv_list *fobj;
28462306a36Sopenharmony_ci	struct dma_fence *old;
28562306a36Sopenharmony_ci	unsigned int i, count;
28662306a36Sopenharmony_ci
28762306a36Sopenharmony_ci	dma_fence_get(fence);
28862306a36Sopenharmony_ci
28962306a36Sopenharmony_ci	dma_resv_assert_held(obj);
29062306a36Sopenharmony_ci
29162306a36Sopenharmony_ci	/* Drivers should not add containers here, instead add each fence
29262306a36Sopenharmony_ci	 * individually.
29362306a36Sopenharmony_ci	 */
29462306a36Sopenharmony_ci	WARN_ON(dma_fence_is_container(fence));
29562306a36Sopenharmony_ci
29662306a36Sopenharmony_ci	fobj = dma_resv_fences_list(obj);
29762306a36Sopenharmony_ci	count = fobj->num_fences;
29862306a36Sopenharmony_ci
29962306a36Sopenharmony_ci	for (i = 0; i < count; ++i) {
30062306a36Sopenharmony_ci		enum dma_resv_usage old_usage;
30162306a36Sopenharmony_ci
30262306a36Sopenharmony_ci		dma_resv_list_entry(fobj, i, obj, &old, &old_usage);
30362306a36Sopenharmony_ci		if ((old->context == fence->context && old_usage >= usage &&
30462306a36Sopenharmony_ci		     dma_fence_is_later_or_same(fence, old)) ||
30562306a36Sopenharmony_ci		    dma_fence_is_signaled(old)) {
30662306a36Sopenharmony_ci			dma_resv_list_set(fobj, i, fence, usage);
30762306a36Sopenharmony_ci			dma_fence_put(old);
30862306a36Sopenharmony_ci			return;
30962306a36Sopenharmony_ci		}
31062306a36Sopenharmony_ci	}
31162306a36Sopenharmony_ci
31262306a36Sopenharmony_ci	BUG_ON(fobj->num_fences >= fobj->max_fences);
31362306a36Sopenharmony_ci	count++;
31462306a36Sopenharmony_ci
31562306a36Sopenharmony_ci	dma_resv_list_set(fobj, i, fence, usage);
31662306a36Sopenharmony_ci	/* pointer update must be visible before we extend the num_fences */
31762306a36Sopenharmony_ci	smp_store_mb(fobj->num_fences, count);
31862306a36Sopenharmony_ci}
31962306a36Sopenharmony_ciEXPORT_SYMBOL(dma_resv_add_fence);
32062306a36Sopenharmony_ci
32162306a36Sopenharmony_ci/**
32262306a36Sopenharmony_ci * dma_resv_replace_fences - replace fences in the dma_resv obj
32362306a36Sopenharmony_ci * @obj: the reservation object
32462306a36Sopenharmony_ci * @context: the context of the fences to replace
32562306a36Sopenharmony_ci * @replacement: the new fence to use instead
32662306a36Sopenharmony_ci * @usage: how the new fence is used, see enum dma_resv_usage
32762306a36Sopenharmony_ci *
32862306a36Sopenharmony_ci * Replace fences with a specified context with a new fence. Only valid if the
32962306a36Sopenharmony_ci * operation represented by the original fence has no longer access to the
33062306a36Sopenharmony_ci * resources represented by the dma_resv object when the new fence completes.
33162306a36Sopenharmony_ci *
33262306a36Sopenharmony_ci * And example for using this is replacing a preemption fence with a page table
33362306a36Sopenharmony_ci * update fence which makes the resource inaccessible.
33462306a36Sopenharmony_ci */
33562306a36Sopenharmony_civoid dma_resv_replace_fences(struct dma_resv *obj, uint64_t context,
33662306a36Sopenharmony_ci			     struct dma_fence *replacement,
33762306a36Sopenharmony_ci			     enum dma_resv_usage usage)
33862306a36Sopenharmony_ci{
33962306a36Sopenharmony_ci	struct dma_resv_list *list;
34062306a36Sopenharmony_ci	unsigned int i;
34162306a36Sopenharmony_ci
34262306a36Sopenharmony_ci	dma_resv_assert_held(obj);
34362306a36Sopenharmony_ci
34462306a36Sopenharmony_ci	list = dma_resv_fences_list(obj);
34562306a36Sopenharmony_ci	for (i = 0; list && i < list->num_fences; ++i) {
34662306a36Sopenharmony_ci		struct dma_fence *old;
34762306a36Sopenharmony_ci
34862306a36Sopenharmony_ci		dma_resv_list_entry(list, i, obj, &old, NULL);
34962306a36Sopenharmony_ci		if (old->context != context)
35062306a36Sopenharmony_ci			continue;
35162306a36Sopenharmony_ci
35262306a36Sopenharmony_ci		dma_resv_list_set(list, i, dma_fence_get(replacement), usage);
35362306a36Sopenharmony_ci		dma_fence_put(old);
35462306a36Sopenharmony_ci	}
35562306a36Sopenharmony_ci}
35662306a36Sopenharmony_ciEXPORT_SYMBOL(dma_resv_replace_fences);
35762306a36Sopenharmony_ci
35862306a36Sopenharmony_ci/* Restart the unlocked iteration by initializing the cursor object. */
35962306a36Sopenharmony_cistatic void dma_resv_iter_restart_unlocked(struct dma_resv_iter *cursor)
36062306a36Sopenharmony_ci{
36162306a36Sopenharmony_ci	cursor->index = 0;
36262306a36Sopenharmony_ci	cursor->num_fences = 0;
36362306a36Sopenharmony_ci	cursor->fences = dma_resv_fences_list(cursor->obj);
36462306a36Sopenharmony_ci	if (cursor->fences)
36562306a36Sopenharmony_ci		cursor->num_fences = cursor->fences->num_fences;
36662306a36Sopenharmony_ci	cursor->is_restarted = true;
36762306a36Sopenharmony_ci}
36862306a36Sopenharmony_ci
36962306a36Sopenharmony_ci/* Walk to the next not signaled fence and grab a reference to it */
37062306a36Sopenharmony_cistatic void dma_resv_iter_walk_unlocked(struct dma_resv_iter *cursor)
37162306a36Sopenharmony_ci{
37262306a36Sopenharmony_ci	if (!cursor->fences)
37362306a36Sopenharmony_ci		return;
37462306a36Sopenharmony_ci
37562306a36Sopenharmony_ci	do {
37662306a36Sopenharmony_ci		/* Drop the reference from the previous round */
37762306a36Sopenharmony_ci		dma_fence_put(cursor->fence);
37862306a36Sopenharmony_ci
37962306a36Sopenharmony_ci		if (cursor->index >= cursor->num_fences) {
38062306a36Sopenharmony_ci			cursor->fence = NULL;
38162306a36Sopenharmony_ci			break;
38262306a36Sopenharmony_ci
38362306a36Sopenharmony_ci		}
38462306a36Sopenharmony_ci
38562306a36Sopenharmony_ci		dma_resv_list_entry(cursor->fences, cursor->index++,
38662306a36Sopenharmony_ci				    cursor->obj, &cursor->fence,
38762306a36Sopenharmony_ci				    &cursor->fence_usage);
38862306a36Sopenharmony_ci		cursor->fence = dma_fence_get_rcu(cursor->fence);
38962306a36Sopenharmony_ci		if (!cursor->fence) {
39062306a36Sopenharmony_ci			dma_resv_iter_restart_unlocked(cursor);
39162306a36Sopenharmony_ci			continue;
39262306a36Sopenharmony_ci		}
39362306a36Sopenharmony_ci
39462306a36Sopenharmony_ci		if (!dma_fence_is_signaled(cursor->fence) &&
39562306a36Sopenharmony_ci		    cursor->usage >= cursor->fence_usage)
39662306a36Sopenharmony_ci			break;
39762306a36Sopenharmony_ci	} while (true);
39862306a36Sopenharmony_ci}
39962306a36Sopenharmony_ci
40062306a36Sopenharmony_ci/**
40162306a36Sopenharmony_ci * dma_resv_iter_first_unlocked - first fence in an unlocked dma_resv obj.
40262306a36Sopenharmony_ci * @cursor: the cursor with the current position
40362306a36Sopenharmony_ci *
40462306a36Sopenharmony_ci * Subsequent fences are iterated with dma_resv_iter_next_unlocked().
40562306a36Sopenharmony_ci *
40662306a36Sopenharmony_ci * Beware that the iterator can be restarted.  Code which accumulates statistics
40762306a36Sopenharmony_ci * or similar needs to check for this with dma_resv_iter_is_restarted(). For
40862306a36Sopenharmony_ci * this reason prefer the locked dma_resv_iter_first() whenver possible.
40962306a36Sopenharmony_ci *
41062306a36Sopenharmony_ci * Returns the first fence from an unlocked dma_resv obj.
41162306a36Sopenharmony_ci */
41262306a36Sopenharmony_cistruct dma_fence *dma_resv_iter_first_unlocked(struct dma_resv_iter *cursor)
41362306a36Sopenharmony_ci{
41462306a36Sopenharmony_ci	rcu_read_lock();
41562306a36Sopenharmony_ci	do {
41662306a36Sopenharmony_ci		dma_resv_iter_restart_unlocked(cursor);
41762306a36Sopenharmony_ci		dma_resv_iter_walk_unlocked(cursor);
41862306a36Sopenharmony_ci	} while (dma_resv_fences_list(cursor->obj) != cursor->fences);
41962306a36Sopenharmony_ci	rcu_read_unlock();
42062306a36Sopenharmony_ci
42162306a36Sopenharmony_ci	return cursor->fence;
42262306a36Sopenharmony_ci}
42362306a36Sopenharmony_ciEXPORT_SYMBOL(dma_resv_iter_first_unlocked);
42462306a36Sopenharmony_ci
42562306a36Sopenharmony_ci/**
42662306a36Sopenharmony_ci * dma_resv_iter_next_unlocked - next fence in an unlocked dma_resv obj.
42762306a36Sopenharmony_ci * @cursor: the cursor with the current position
42862306a36Sopenharmony_ci *
42962306a36Sopenharmony_ci * Beware that the iterator can be restarted.  Code which accumulates statistics
43062306a36Sopenharmony_ci * or similar needs to check for this with dma_resv_iter_is_restarted(). For
43162306a36Sopenharmony_ci * this reason prefer the locked dma_resv_iter_next() whenver possible.
43262306a36Sopenharmony_ci *
43362306a36Sopenharmony_ci * Returns the next fence from an unlocked dma_resv obj.
43462306a36Sopenharmony_ci */
43562306a36Sopenharmony_cistruct dma_fence *dma_resv_iter_next_unlocked(struct dma_resv_iter *cursor)
43662306a36Sopenharmony_ci{
43762306a36Sopenharmony_ci	bool restart;
43862306a36Sopenharmony_ci
43962306a36Sopenharmony_ci	rcu_read_lock();
44062306a36Sopenharmony_ci	cursor->is_restarted = false;
44162306a36Sopenharmony_ci	restart = dma_resv_fences_list(cursor->obj) != cursor->fences;
44262306a36Sopenharmony_ci	do {
44362306a36Sopenharmony_ci		if (restart)
44462306a36Sopenharmony_ci			dma_resv_iter_restart_unlocked(cursor);
44562306a36Sopenharmony_ci		dma_resv_iter_walk_unlocked(cursor);
44662306a36Sopenharmony_ci		restart = true;
44762306a36Sopenharmony_ci	} while (dma_resv_fences_list(cursor->obj) != cursor->fences);
44862306a36Sopenharmony_ci	rcu_read_unlock();
44962306a36Sopenharmony_ci
45062306a36Sopenharmony_ci	return cursor->fence;
45162306a36Sopenharmony_ci}
45262306a36Sopenharmony_ciEXPORT_SYMBOL(dma_resv_iter_next_unlocked);
45362306a36Sopenharmony_ci
45462306a36Sopenharmony_ci/**
45562306a36Sopenharmony_ci * dma_resv_iter_first - first fence from a locked dma_resv object
45662306a36Sopenharmony_ci * @cursor: cursor to record the current position
45762306a36Sopenharmony_ci *
45862306a36Sopenharmony_ci * Subsequent fences are iterated with dma_resv_iter_next_unlocked().
45962306a36Sopenharmony_ci *
46062306a36Sopenharmony_ci * Return the first fence in the dma_resv object while holding the
46162306a36Sopenharmony_ci * &dma_resv.lock.
46262306a36Sopenharmony_ci */
46362306a36Sopenharmony_cistruct dma_fence *dma_resv_iter_first(struct dma_resv_iter *cursor)
46462306a36Sopenharmony_ci{
46562306a36Sopenharmony_ci	struct dma_fence *fence;
46662306a36Sopenharmony_ci
46762306a36Sopenharmony_ci	dma_resv_assert_held(cursor->obj);
46862306a36Sopenharmony_ci
46962306a36Sopenharmony_ci	cursor->index = 0;
47062306a36Sopenharmony_ci	cursor->fences = dma_resv_fences_list(cursor->obj);
47162306a36Sopenharmony_ci
47262306a36Sopenharmony_ci	fence = dma_resv_iter_next(cursor);
47362306a36Sopenharmony_ci	cursor->is_restarted = true;
47462306a36Sopenharmony_ci	return fence;
47562306a36Sopenharmony_ci}
47662306a36Sopenharmony_ciEXPORT_SYMBOL_GPL(dma_resv_iter_first);
47762306a36Sopenharmony_ci
47862306a36Sopenharmony_ci/**
47962306a36Sopenharmony_ci * dma_resv_iter_next - next fence from a locked dma_resv object
48062306a36Sopenharmony_ci * @cursor: cursor to record the current position
48162306a36Sopenharmony_ci *
48262306a36Sopenharmony_ci * Return the next fences from the dma_resv object while holding the
48362306a36Sopenharmony_ci * &dma_resv.lock.
48462306a36Sopenharmony_ci */
48562306a36Sopenharmony_cistruct dma_fence *dma_resv_iter_next(struct dma_resv_iter *cursor)
48662306a36Sopenharmony_ci{
48762306a36Sopenharmony_ci	struct dma_fence *fence;
48862306a36Sopenharmony_ci
48962306a36Sopenharmony_ci	dma_resv_assert_held(cursor->obj);
49062306a36Sopenharmony_ci
49162306a36Sopenharmony_ci	cursor->is_restarted = false;
49262306a36Sopenharmony_ci
49362306a36Sopenharmony_ci	do {
49462306a36Sopenharmony_ci		if (!cursor->fences ||
49562306a36Sopenharmony_ci		    cursor->index >= cursor->fences->num_fences)
49662306a36Sopenharmony_ci			return NULL;
49762306a36Sopenharmony_ci
49862306a36Sopenharmony_ci		dma_resv_list_entry(cursor->fences, cursor->index++,
49962306a36Sopenharmony_ci				    cursor->obj, &fence, &cursor->fence_usage);
50062306a36Sopenharmony_ci	} while (cursor->fence_usage > cursor->usage);
50162306a36Sopenharmony_ci
50262306a36Sopenharmony_ci	return fence;
50362306a36Sopenharmony_ci}
50462306a36Sopenharmony_ciEXPORT_SYMBOL_GPL(dma_resv_iter_next);
50562306a36Sopenharmony_ci
50662306a36Sopenharmony_ci/**
50762306a36Sopenharmony_ci * dma_resv_copy_fences - Copy all fences from src to dst.
50862306a36Sopenharmony_ci * @dst: the destination reservation object
50962306a36Sopenharmony_ci * @src: the source reservation object
51062306a36Sopenharmony_ci *
51162306a36Sopenharmony_ci * Copy all fences from src to dst. dst-lock must be held.
51262306a36Sopenharmony_ci */
51362306a36Sopenharmony_ciint dma_resv_copy_fences(struct dma_resv *dst, struct dma_resv *src)
51462306a36Sopenharmony_ci{
51562306a36Sopenharmony_ci	struct dma_resv_iter cursor;
51662306a36Sopenharmony_ci	struct dma_resv_list *list;
51762306a36Sopenharmony_ci	struct dma_fence *f;
51862306a36Sopenharmony_ci
51962306a36Sopenharmony_ci	dma_resv_assert_held(dst);
52062306a36Sopenharmony_ci
52162306a36Sopenharmony_ci	list = NULL;
52262306a36Sopenharmony_ci
52362306a36Sopenharmony_ci	dma_resv_iter_begin(&cursor, src, DMA_RESV_USAGE_BOOKKEEP);
52462306a36Sopenharmony_ci	dma_resv_for_each_fence_unlocked(&cursor, f) {
52562306a36Sopenharmony_ci
52662306a36Sopenharmony_ci		if (dma_resv_iter_is_restarted(&cursor)) {
52762306a36Sopenharmony_ci			dma_resv_list_free(list);
52862306a36Sopenharmony_ci
52962306a36Sopenharmony_ci			list = dma_resv_list_alloc(cursor.num_fences);
53062306a36Sopenharmony_ci			if (!list) {
53162306a36Sopenharmony_ci				dma_resv_iter_end(&cursor);
53262306a36Sopenharmony_ci				return -ENOMEM;
53362306a36Sopenharmony_ci			}
53462306a36Sopenharmony_ci			list->num_fences = 0;
53562306a36Sopenharmony_ci		}
53662306a36Sopenharmony_ci
53762306a36Sopenharmony_ci		dma_fence_get(f);
53862306a36Sopenharmony_ci		dma_resv_list_set(list, list->num_fences++, f,
53962306a36Sopenharmony_ci				  dma_resv_iter_usage(&cursor));
54062306a36Sopenharmony_ci	}
54162306a36Sopenharmony_ci	dma_resv_iter_end(&cursor);
54262306a36Sopenharmony_ci
54362306a36Sopenharmony_ci	list = rcu_replace_pointer(dst->fences, list, dma_resv_held(dst));
54462306a36Sopenharmony_ci	dma_resv_list_free(list);
54562306a36Sopenharmony_ci	return 0;
54662306a36Sopenharmony_ci}
54762306a36Sopenharmony_ciEXPORT_SYMBOL(dma_resv_copy_fences);
54862306a36Sopenharmony_ci
54962306a36Sopenharmony_ci/**
55062306a36Sopenharmony_ci * dma_resv_get_fences - Get an object's fences
55162306a36Sopenharmony_ci * fences without update side lock held
55262306a36Sopenharmony_ci * @obj: the reservation object
55362306a36Sopenharmony_ci * @usage: controls which fences to include, see enum dma_resv_usage.
55462306a36Sopenharmony_ci * @num_fences: the number of fences returned
55562306a36Sopenharmony_ci * @fences: the array of fence ptrs returned (array is krealloc'd to the
55662306a36Sopenharmony_ci * required size, and must be freed by caller)
55762306a36Sopenharmony_ci *
55862306a36Sopenharmony_ci * Retrieve all fences from the reservation object.
55962306a36Sopenharmony_ci * Returns either zero or -ENOMEM.
56062306a36Sopenharmony_ci */
56162306a36Sopenharmony_ciint dma_resv_get_fences(struct dma_resv *obj, enum dma_resv_usage usage,
56262306a36Sopenharmony_ci			unsigned int *num_fences, struct dma_fence ***fences)
56362306a36Sopenharmony_ci{
56462306a36Sopenharmony_ci	struct dma_resv_iter cursor;
56562306a36Sopenharmony_ci	struct dma_fence *fence;
56662306a36Sopenharmony_ci
56762306a36Sopenharmony_ci	*num_fences = 0;
56862306a36Sopenharmony_ci	*fences = NULL;
56962306a36Sopenharmony_ci
57062306a36Sopenharmony_ci	dma_resv_iter_begin(&cursor, obj, usage);
57162306a36Sopenharmony_ci	dma_resv_for_each_fence_unlocked(&cursor, fence) {
57262306a36Sopenharmony_ci
57362306a36Sopenharmony_ci		if (dma_resv_iter_is_restarted(&cursor)) {
57462306a36Sopenharmony_ci			struct dma_fence **new_fences;
57562306a36Sopenharmony_ci			unsigned int count;
57662306a36Sopenharmony_ci
57762306a36Sopenharmony_ci			while (*num_fences)
57862306a36Sopenharmony_ci				dma_fence_put((*fences)[--(*num_fences)]);
57962306a36Sopenharmony_ci
58062306a36Sopenharmony_ci			count = cursor.num_fences + 1;
58162306a36Sopenharmony_ci
58262306a36Sopenharmony_ci			/* Eventually re-allocate the array */
58362306a36Sopenharmony_ci			new_fences = krealloc_array(*fences, count,
58462306a36Sopenharmony_ci						    sizeof(void *),
58562306a36Sopenharmony_ci						    GFP_KERNEL);
58662306a36Sopenharmony_ci			if (count && !new_fences) {
58762306a36Sopenharmony_ci				kfree(*fences);
58862306a36Sopenharmony_ci				*fences = NULL;
58962306a36Sopenharmony_ci				*num_fences = 0;
59062306a36Sopenharmony_ci				dma_resv_iter_end(&cursor);
59162306a36Sopenharmony_ci				return -ENOMEM;
59262306a36Sopenharmony_ci			}
59362306a36Sopenharmony_ci			*fences = new_fences;
59462306a36Sopenharmony_ci		}
59562306a36Sopenharmony_ci
59662306a36Sopenharmony_ci		(*fences)[(*num_fences)++] = dma_fence_get(fence);
59762306a36Sopenharmony_ci	}
59862306a36Sopenharmony_ci	dma_resv_iter_end(&cursor);
59962306a36Sopenharmony_ci
60062306a36Sopenharmony_ci	return 0;
60162306a36Sopenharmony_ci}
60262306a36Sopenharmony_ciEXPORT_SYMBOL_GPL(dma_resv_get_fences);
60362306a36Sopenharmony_ci
60462306a36Sopenharmony_ci/**
60562306a36Sopenharmony_ci * dma_resv_get_singleton - Get a single fence for all the fences
60662306a36Sopenharmony_ci * @obj: the reservation object
60762306a36Sopenharmony_ci * @usage: controls which fences to include, see enum dma_resv_usage.
60862306a36Sopenharmony_ci * @fence: the resulting fence
60962306a36Sopenharmony_ci *
61062306a36Sopenharmony_ci * Get a single fence representing all the fences inside the resv object.
61162306a36Sopenharmony_ci * Returns either 0 for success or -ENOMEM.
61262306a36Sopenharmony_ci *
61362306a36Sopenharmony_ci * Warning: This can't be used like this when adding the fence back to the resv
61462306a36Sopenharmony_ci * object since that can lead to stack corruption when finalizing the
61562306a36Sopenharmony_ci * dma_fence_array.
61662306a36Sopenharmony_ci *
61762306a36Sopenharmony_ci * Returns 0 on success and negative error values on failure.
61862306a36Sopenharmony_ci */
61962306a36Sopenharmony_ciint dma_resv_get_singleton(struct dma_resv *obj, enum dma_resv_usage usage,
62062306a36Sopenharmony_ci			   struct dma_fence **fence)
62162306a36Sopenharmony_ci{
62262306a36Sopenharmony_ci	struct dma_fence_array *array;
62362306a36Sopenharmony_ci	struct dma_fence **fences;
62462306a36Sopenharmony_ci	unsigned count;
62562306a36Sopenharmony_ci	int r;
62662306a36Sopenharmony_ci
62762306a36Sopenharmony_ci	r = dma_resv_get_fences(obj, usage, &count, &fences);
62862306a36Sopenharmony_ci        if (r)
62962306a36Sopenharmony_ci		return r;
63062306a36Sopenharmony_ci
63162306a36Sopenharmony_ci	if (count == 0) {
63262306a36Sopenharmony_ci		*fence = NULL;
63362306a36Sopenharmony_ci		return 0;
63462306a36Sopenharmony_ci	}
63562306a36Sopenharmony_ci
63662306a36Sopenharmony_ci	if (count == 1) {
63762306a36Sopenharmony_ci		*fence = fences[0];
63862306a36Sopenharmony_ci		kfree(fences);
63962306a36Sopenharmony_ci		return 0;
64062306a36Sopenharmony_ci	}
64162306a36Sopenharmony_ci
64262306a36Sopenharmony_ci	array = dma_fence_array_create(count, fences,
64362306a36Sopenharmony_ci				       dma_fence_context_alloc(1),
64462306a36Sopenharmony_ci				       1, false);
64562306a36Sopenharmony_ci	if (!array) {
64662306a36Sopenharmony_ci		while (count--)
64762306a36Sopenharmony_ci			dma_fence_put(fences[count]);
64862306a36Sopenharmony_ci		kfree(fences);
64962306a36Sopenharmony_ci		return -ENOMEM;
65062306a36Sopenharmony_ci	}
65162306a36Sopenharmony_ci
65262306a36Sopenharmony_ci	*fence = &array->base;
65362306a36Sopenharmony_ci	return 0;
65462306a36Sopenharmony_ci}
65562306a36Sopenharmony_ciEXPORT_SYMBOL_GPL(dma_resv_get_singleton);
65662306a36Sopenharmony_ci
65762306a36Sopenharmony_ci/**
65862306a36Sopenharmony_ci * dma_resv_wait_timeout - Wait on reservation's objects fences
65962306a36Sopenharmony_ci * @obj: the reservation object
66062306a36Sopenharmony_ci * @usage: controls which fences to include, see enum dma_resv_usage.
66162306a36Sopenharmony_ci * @intr: if true, do interruptible wait
66262306a36Sopenharmony_ci * @timeout: timeout value in jiffies or zero to return immediately
66362306a36Sopenharmony_ci *
66462306a36Sopenharmony_ci * Callers are not required to hold specific locks, but maybe hold
66562306a36Sopenharmony_ci * dma_resv_lock() already
66662306a36Sopenharmony_ci * RETURNS
66762306a36Sopenharmony_ci * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or
66862306a36Sopenharmony_ci * greater than zero on success.
66962306a36Sopenharmony_ci */
67062306a36Sopenharmony_cilong dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage,
67162306a36Sopenharmony_ci			   bool intr, unsigned long timeout)
67262306a36Sopenharmony_ci{
67362306a36Sopenharmony_ci	long ret = timeout ? timeout : 1;
67462306a36Sopenharmony_ci	struct dma_resv_iter cursor;
67562306a36Sopenharmony_ci	struct dma_fence *fence;
67662306a36Sopenharmony_ci
67762306a36Sopenharmony_ci	dma_resv_iter_begin(&cursor, obj, usage);
67862306a36Sopenharmony_ci	dma_resv_for_each_fence_unlocked(&cursor, fence) {
67962306a36Sopenharmony_ci
68062306a36Sopenharmony_ci		ret = dma_fence_wait_timeout(fence, intr, ret);
68162306a36Sopenharmony_ci		if (ret <= 0) {
68262306a36Sopenharmony_ci			dma_resv_iter_end(&cursor);
68362306a36Sopenharmony_ci			return ret;
68462306a36Sopenharmony_ci		}
68562306a36Sopenharmony_ci	}
68662306a36Sopenharmony_ci	dma_resv_iter_end(&cursor);
68762306a36Sopenharmony_ci
68862306a36Sopenharmony_ci	return ret;
68962306a36Sopenharmony_ci}
69062306a36Sopenharmony_ciEXPORT_SYMBOL_GPL(dma_resv_wait_timeout);
69162306a36Sopenharmony_ci
69262306a36Sopenharmony_ci/**
69362306a36Sopenharmony_ci * dma_resv_set_deadline - Set a deadline on reservation's objects fences
69462306a36Sopenharmony_ci * @obj: the reservation object
69562306a36Sopenharmony_ci * @usage: controls which fences to include, see enum dma_resv_usage.
69662306a36Sopenharmony_ci * @deadline: the requested deadline (MONOTONIC)
69762306a36Sopenharmony_ci *
69862306a36Sopenharmony_ci * May be called without holding the dma_resv lock.  Sets @deadline on
69962306a36Sopenharmony_ci * all fences filtered by @usage.
70062306a36Sopenharmony_ci */
70162306a36Sopenharmony_civoid dma_resv_set_deadline(struct dma_resv *obj, enum dma_resv_usage usage,
70262306a36Sopenharmony_ci			   ktime_t deadline)
70362306a36Sopenharmony_ci{
70462306a36Sopenharmony_ci	struct dma_resv_iter cursor;
70562306a36Sopenharmony_ci	struct dma_fence *fence;
70662306a36Sopenharmony_ci
70762306a36Sopenharmony_ci	dma_resv_iter_begin(&cursor, obj, usage);
70862306a36Sopenharmony_ci	dma_resv_for_each_fence_unlocked(&cursor, fence) {
70962306a36Sopenharmony_ci		dma_fence_set_deadline(fence, deadline);
71062306a36Sopenharmony_ci	}
71162306a36Sopenharmony_ci	dma_resv_iter_end(&cursor);
71262306a36Sopenharmony_ci}
71362306a36Sopenharmony_ciEXPORT_SYMBOL_GPL(dma_resv_set_deadline);
71462306a36Sopenharmony_ci
71562306a36Sopenharmony_ci/**
71662306a36Sopenharmony_ci * dma_resv_test_signaled - Test if a reservation object's fences have been
71762306a36Sopenharmony_ci * signaled.
71862306a36Sopenharmony_ci * @obj: the reservation object
71962306a36Sopenharmony_ci * @usage: controls which fences to include, see enum dma_resv_usage.
72062306a36Sopenharmony_ci *
72162306a36Sopenharmony_ci * Callers are not required to hold specific locks, but maybe hold
72262306a36Sopenharmony_ci * dma_resv_lock() already.
72362306a36Sopenharmony_ci *
72462306a36Sopenharmony_ci * RETURNS
72562306a36Sopenharmony_ci *
72662306a36Sopenharmony_ci * True if all fences signaled, else false.
72762306a36Sopenharmony_ci */
72862306a36Sopenharmony_cibool dma_resv_test_signaled(struct dma_resv *obj, enum dma_resv_usage usage)
72962306a36Sopenharmony_ci{
73062306a36Sopenharmony_ci	struct dma_resv_iter cursor;
73162306a36Sopenharmony_ci	struct dma_fence *fence;
73262306a36Sopenharmony_ci
73362306a36Sopenharmony_ci	dma_resv_iter_begin(&cursor, obj, usage);
73462306a36Sopenharmony_ci	dma_resv_for_each_fence_unlocked(&cursor, fence) {
73562306a36Sopenharmony_ci		dma_resv_iter_end(&cursor);
73662306a36Sopenharmony_ci		return false;
73762306a36Sopenharmony_ci	}
73862306a36Sopenharmony_ci	dma_resv_iter_end(&cursor);
73962306a36Sopenharmony_ci	return true;
74062306a36Sopenharmony_ci}
74162306a36Sopenharmony_ciEXPORT_SYMBOL_GPL(dma_resv_test_signaled);
74262306a36Sopenharmony_ci
74362306a36Sopenharmony_ci/**
74462306a36Sopenharmony_ci * dma_resv_describe - Dump description of the resv object into seq_file
74562306a36Sopenharmony_ci * @obj: the reservation object
74662306a36Sopenharmony_ci * @seq: the seq_file to dump the description into
74762306a36Sopenharmony_ci *
74862306a36Sopenharmony_ci * Dump a textual description of the fences inside an dma_resv object into the
74962306a36Sopenharmony_ci * seq_file.
75062306a36Sopenharmony_ci */
75162306a36Sopenharmony_civoid dma_resv_describe(struct dma_resv *obj, struct seq_file *seq)
75262306a36Sopenharmony_ci{
75362306a36Sopenharmony_ci	static const char *usage[] = { "kernel", "write", "read", "bookkeep" };
75462306a36Sopenharmony_ci	struct dma_resv_iter cursor;
75562306a36Sopenharmony_ci	struct dma_fence *fence;
75662306a36Sopenharmony_ci
75762306a36Sopenharmony_ci	dma_resv_for_each_fence(&cursor, obj, DMA_RESV_USAGE_READ, fence) {
75862306a36Sopenharmony_ci		seq_printf(seq, "\t%s fence:",
75962306a36Sopenharmony_ci			   usage[dma_resv_iter_usage(&cursor)]);
76062306a36Sopenharmony_ci		dma_fence_describe(fence, seq);
76162306a36Sopenharmony_ci	}
76262306a36Sopenharmony_ci}
76362306a36Sopenharmony_ciEXPORT_SYMBOL_GPL(dma_resv_describe);
76462306a36Sopenharmony_ci
76562306a36Sopenharmony_ci#if IS_ENABLED(CONFIG_LOCKDEP)
76662306a36Sopenharmony_cistatic int __init dma_resv_lockdep(void)
76762306a36Sopenharmony_ci{
76862306a36Sopenharmony_ci	struct mm_struct *mm = mm_alloc();
76962306a36Sopenharmony_ci	struct ww_acquire_ctx ctx;
77062306a36Sopenharmony_ci	struct dma_resv obj;
77162306a36Sopenharmony_ci	struct address_space mapping;
77262306a36Sopenharmony_ci	int ret;
77362306a36Sopenharmony_ci
77462306a36Sopenharmony_ci	if (!mm)
77562306a36Sopenharmony_ci		return -ENOMEM;
77662306a36Sopenharmony_ci
77762306a36Sopenharmony_ci	dma_resv_init(&obj);
77862306a36Sopenharmony_ci	address_space_init_once(&mapping);
77962306a36Sopenharmony_ci
78062306a36Sopenharmony_ci	mmap_read_lock(mm);
78162306a36Sopenharmony_ci	ww_acquire_init(&ctx, &reservation_ww_class);
78262306a36Sopenharmony_ci	ret = dma_resv_lock(&obj, &ctx);
78362306a36Sopenharmony_ci	if (ret == -EDEADLK)
78462306a36Sopenharmony_ci		dma_resv_lock_slow(&obj, &ctx);
78562306a36Sopenharmony_ci	fs_reclaim_acquire(GFP_KERNEL);
78662306a36Sopenharmony_ci	/* for unmap_mapping_range on trylocked buffer objects in shrinkers */
78762306a36Sopenharmony_ci	i_mmap_lock_write(&mapping);
78862306a36Sopenharmony_ci	i_mmap_unlock_write(&mapping);
78962306a36Sopenharmony_ci#ifdef CONFIG_MMU_NOTIFIER
79062306a36Sopenharmony_ci	lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
79162306a36Sopenharmony_ci	__dma_fence_might_wait();
79262306a36Sopenharmony_ci	lock_map_release(&__mmu_notifier_invalidate_range_start_map);
79362306a36Sopenharmony_ci#else
79462306a36Sopenharmony_ci	__dma_fence_might_wait();
79562306a36Sopenharmony_ci#endif
79662306a36Sopenharmony_ci	fs_reclaim_release(GFP_KERNEL);
79762306a36Sopenharmony_ci	ww_mutex_unlock(&obj.lock);
79862306a36Sopenharmony_ci	ww_acquire_fini(&ctx);
79962306a36Sopenharmony_ci	mmap_read_unlock(mm);
80062306a36Sopenharmony_ci
80162306a36Sopenharmony_ci	mmput(mm);
80262306a36Sopenharmony_ci
80362306a36Sopenharmony_ci	return 0;
80462306a36Sopenharmony_ci}
80562306a36Sopenharmony_cisubsys_initcall(dma_resv_lockdep);
80662306a36Sopenharmony_ci#endif
807