162306a36Sopenharmony_ci// SPDX-License-Identifier: GPL-2.0 OR MIT
262306a36Sopenharmony_ci/**************************************************************************
362306a36Sopenharmony_ci *
462306a36Sopenharmony_ci * Copyright 2014-2023 VMware, Inc., Palo Alto, CA., USA
562306a36Sopenharmony_ci *
662306a36Sopenharmony_ci * Permission is hereby granted, free of charge, to any person obtaining a
762306a36Sopenharmony_ci * copy of this software and associated documentation files (the
862306a36Sopenharmony_ci * "Software"), to deal in the Software without restriction, including
962306a36Sopenharmony_ci * without limitation the rights to use, copy, modify, merge, publish,
1062306a36Sopenharmony_ci * distribute, sub license, and/or sell copies of the Software, and to
1162306a36Sopenharmony_ci * permit persons to whom the Software is furnished to do so, subject to
1262306a36Sopenharmony_ci * the following conditions:
1362306a36Sopenharmony_ci *
1462306a36Sopenharmony_ci * The above copyright notice and this permission notice (including the
1562306a36Sopenharmony_ci * next paragraph) shall be included in all copies or substantial portions
1662306a36Sopenharmony_ci * of the Software.
1762306a36Sopenharmony_ci *
1862306a36Sopenharmony_ci * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1962306a36Sopenharmony_ci * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
2062306a36Sopenharmony_ci * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
2162306a36Sopenharmony_ci * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
2262306a36Sopenharmony_ci * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
2362306a36Sopenharmony_ci * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
2462306a36Sopenharmony_ci * USE OR OTHER DEALINGS IN THE SOFTWARE.
2562306a36Sopenharmony_ci *
2662306a36Sopenharmony_ci **************************************************************************/
2762306a36Sopenharmony_ci/*
2862306a36Sopenharmony_ci * Treat context OTables as resources to make use of the resource
2962306a36Sopenharmony_ci * backing MOB eviction mechanism, that is used to read back the COTable
3062306a36Sopenharmony_ci * whenever the backing MOB is evicted.
3162306a36Sopenharmony_ci */
3262306a36Sopenharmony_ci
3362306a36Sopenharmony_ci#include "vmwgfx_bo.h"
3462306a36Sopenharmony_ci#include "vmwgfx_drv.h"
3562306a36Sopenharmony_ci#include "vmwgfx_mksstat.h"
3662306a36Sopenharmony_ci#include "vmwgfx_resource_priv.h"
3762306a36Sopenharmony_ci#include "vmwgfx_so.h"
3862306a36Sopenharmony_ci
3962306a36Sopenharmony_ci#include <drm/ttm/ttm_placement.h>
4062306a36Sopenharmony_ci
4162306a36Sopenharmony_ci/**
4262306a36Sopenharmony_ci * struct vmw_cotable - Context Object Table resource
4362306a36Sopenharmony_ci *
4462306a36Sopenharmony_ci * @res: struct vmw_resource we are deriving from.
4562306a36Sopenharmony_ci * @ctx: non-refcounted pointer to the owning context.
4662306a36Sopenharmony_ci * @size_read_back: Size of data read back during eviction.
4762306a36Sopenharmony_ci * @seen_entries: Seen entries in command stream for this cotable.
4862306a36Sopenharmony_ci * @type: The cotable type.
4962306a36Sopenharmony_ci * @scrubbed: Whether the cotable has been scrubbed.
5062306a36Sopenharmony_ci * @resource_list: List of resources in the cotable.
5162306a36Sopenharmony_ci */
5262306a36Sopenharmony_cistruct vmw_cotable {
5362306a36Sopenharmony_ci	struct vmw_resource res;
5462306a36Sopenharmony_ci	struct vmw_resource *ctx;
5562306a36Sopenharmony_ci	size_t size_read_back;
5662306a36Sopenharmony_ci	int seen_entries;
5762306a36Sopenharmony_ci	u32 type;
5862306a36Sopenharmony_ci	bool scrubbed;
5962306a36Sopenharmony_ci	struct list_head resource_list;
6062306a36Sopenharmony_ci};
6162306a36Sopenharmony_ci
6262306a36Sopenharmony_ci/**
6362306a36Sopenharmony_ci * struct vmw_cotable_info - Static info about cotable types
6462306a36Sopenharmony_ci *
6562306a36Sopenharmony_ci * @min_initial_entries: Min number of initial intries at cotable allocation
6662306a36Sopenharmony_ci * for this cotable type.
6762306a36Sopenharmony_ci * @size: Size of each entry.
6862306a36Sopenharmony_ci * @unbind_func: Unbind call-back function.
6962306a36Sopenharmony_ci */
7062306a36Sopenharmony_cistruct vmw_cotable_info {
7162306a36Sopenharmony_ci	u32 min_initial_entries;
7262306a36Sopenharmony_ci	u32 size;
7362306a36Sopenharmony_ci	void (*unbind_func)(struct vmw_private *, struct list_head *,
7462306a36Sopenharmony_ci			    bool);
7562306a36Sopenharmony_ci};
7662306a36Sopenharmony_ci
7762306a36Sopenharmony_ci
7862306a36Sopenharmony_ci/*
7962306a36Sopenharmony_ci * Getting the initial size right is difficult because it all depends
8062306a36Sopenharmony_ci * on what the userspace is doing. The sizes will be aligned up to
8162306a36Sopenharmony_ci * a PAGE_SIZE so we just want to make sure that for majority of apps
8262306a36Sopenharmony_ci * the initial number of entries doesn't require an immediate resize.
8362306a36Sopenharmony_ci * For all cotables except SVGACOTableDXElementLayoutEntry and
8462306a36Sopenharmony_ci * SVGACOTableDXBlendStateEntry the initial number of entries fits
8562306a36Sopenharmony_ci * within the PAGE_SIZE. For SVGACOTableDXElementLayoutEntry and
8662306a36Sopenharmony_ci * SVGACOTableDXBlendStateEntry we want to reserve two pages,
8762306a36Sopenharmony_ci * because that's what all apps will require initially.
8862306a36Sopenharmony_ci */
8962306a36Sopenharmony_cistatic const struct vmw_cotable_info co_info[] = {
9062306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXRTViewEntry), &vmw_view_cotable_list_destroy},
9162306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXDSViewEntry), &vmw_view_cotable_list_destroy},
9262306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXSRViewEntry), &vmw_view_cotable_list_destroy},
9362306a36Sopenharmony_ci	{PAGE_SIZE/sizeof(SVGACOTableDXElementLayoutEntry) + 1, sizeof(SVGACOTableDXElementLayoutEntry), NULL},
9462306a36Sopenharmony_ci	{PAGE_SIZE/sizeof(SVGACOTableDXBlendStateEntry) + 1, sizeof(SVGACOTableDXBlendStateEntry), NULL},
9562306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXDepthStencilEntry), NULL},
9662306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXRasterizerStateEntry), NULL},
9762306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXSamplerEntry), NULL},
9862306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXStreamOutputEntry), &vmw_dx_streamoutput_cotable_list_scrub},
9962306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXQueryEntry), NULL},
10062306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXShaderEntry), &vmw_dx_shader_cotable_list_scrub},
10162306a36Sopenharmony_ci	{1, sizeof(SVGACOTableDXUAViewEntry), &vmw_view_cotable_list_destroy}
10262306a36Sopenharmony_ci};
10362306a36Sopenharmony_ci
10462306a36Sopenharmony_ci/*
10562306a36Sopenharmony_ci * Cotables with bindings that we remove must be scrubbed first,
10662306a36Sopenharmony_ci * otherwise, the device will swap in an invalid context when we remove
10762306a36Sopenharmony_ci * bindings before scrubbing a cotable...
10862306a36Sopenharmony_ci */
10962306a36Sopenharmony_ciconst SVGACOTableType vmw_cotable_scrub_order[] = {
11062306a36Sopenharmony_ci	SVGA_COTABLE_RTVIEW,
11162306a36Sopenharmony_ci	SVGA_COTABLE_DSVIEW,
11262306a36Sopenharmony_ci	SVGA_COTABLE_SRVIEW,
11362306a36Sopenharmony_ci	SVGA_COTABLE_DXSHADER,
11462306a36Sopenharmony_ci	SVGA_COTABLE_ELEMENTLAYOUT,
11562306a36Sopenharmony_ci	SVGA_COTABLE_BLENDSTATE,
11662306a36Sopenharmony_ci	SVGA_COTABLE_DEPTHSTENCIL,
11762306a36Sopenharmony_ci	SVGA_COTABLE_RASTERIZERSTATE,
11862306a36Sopenharmony_ci	SVGA_COTABLE_SAMPLER,
11962306a36Sopenharmony_ci	SVGA_COTABLE_STREAMOUTPUT,
12062306a36Sopenharmony_ci	SVGA_COTABLE_DXQUERY,
12162306a36Sopenharmony_ci	SVGA_COTABLE_UAVIEW,
12262306a36Sopenharmony_ci};
12362306a36Sopenharmony_ci
12462306a36Sopenharmony_cistatic int vmw_cotable_bind(struct vmw_resource *res,
12562306a36Sopenharmony_ci			    struct ttm_validate_buffer *val_buf);
12662306a36Sopenharmony_cistatic int vmw_cotable_unbind(struct vmw_resource *res,
12762306a36Sopenharmony_ci			      bool readback,
12862306a36Sopenharmony_ci			      struct ttm_validate_buffer *val_buf);
12962306a36Sopenharmony_cistatic int vmw_cotable_create(struct vmw_resource *res);
13062306a36Sopenharmony_cistatic int vmw_cotable_destroy(struct vmw_resource *res);
13162306a36Sopenharmony_ci
13262306a36Sopenharmony_cistatic const struct vmw_res_func vmw_cotable_func = {
13362306a36Sopenharmony_ci	.res_type = vmw_res_cotable,
13462306a36Sopenharmony_ci	.needs_guest_memory = true,
13562306a36Sopenharmony_ci	.may_evict = true,
13662306a36Sopenharmony_ci	.prio = 3,
13762306a36Sopenharmony_ci	.dirty_prio = 3,
13862306a36Sopenharmony_ci	.type_name = "context guest backed object tables",
13962306a36Sopenharmony_ci	.domain = VMW_BO_DOMAIN_MOB,
14062306a36Sopenharmony_ci	.busy_domain = VMW_BO_DOMAIN_MOB,
14162306a36Sopenharmony_ci	.create = vmw_cotable_create,
14262306a36Sopenharmony_ci	.destroy = vmw_cotable_destroy,
14362306a36Sopenharmony_ci	.bind = vmw_cotable_bind,
14462306a36Sopenharmony_ci	.unbind = vmw_cotable_unbind,
14562306a36Sopenharmony_ci};
14662306a36Sopenharmony_ci
14762306a36Sopenharmony_ci/**
14862306a36Sopenharmony_ci * vmw_cotable - Convert a struct vmw_resource pointer to a struct
14962306a36Sopenharmony_ci * vmw_cotable pointer
15062306a36Sopenharmony_ci *
15162306a36Sopenharmony_ci * @res: Pointer to the resource.
15262306a36Sopenharmony_ci */
15362306a36Sopenharmony_cistatic struct vmw_cotable *vmw_cotable(struct vmw_resource *res)
15462306a36Sopenharmony_ci{
15562306a36Sopenharmony_ci	return container_of(res, struct vmw_cotable, res);
15662306a36Sopenharmony_ci}
15762306a36Sopenharmony_ci
15862306a36Sopenharmony_ci/**
15962306a36Sopenharmony_ci * vmw_cotable_destroy - Cotable resource destroy callback
16062306a36Sopenharmony_ci *
16162306a36Sopenharmony_ci * @res: Pointer to the cotable resource.
16262306a36Sopenharmony_ci *
16362306a36Sopenharmony_ci * There is no device cotable destroy command, so this function only
16462306a36Sopenharmony_ci * makes sure that the resource id is set to invalid.
16562306a36Sopenharmony_ci */
16662306a36Sopenharmony_cistatic int vmw_cotable_destroy(struct vmw_resource *res)
16762306a36Sopenharmony_ci{
16862306a36Sopenharmony_ci	res->id = -1;
16962306a36Sopenharmony_ci	return 0;
17062306a36Sopenharmony_ci}
17162306a36Sopenharmony_ci
17262306a36Sopenharmony_ci/**
17362306a36Sopenharmony_ci * vmw_cotable_unscrub - Undo a cotable unscrub operation
17462306a36Sopenharmony_ci *
17562306a36Sopenharmony_ci * @res: Pointer to the cotable resource
17662306a36Sopenharmony_ci *
17762306a36Sopenharmony_ci * This function issues commands to (re)bind the cotable to
17862306a36Sopenharmony_ci * its backing mob, which needs to be validated and reserved at this point.
17962306a36Sopenharmony_ci * This is identical to bind() except the function interface looks different.
18062306a36Sopenharmony_ci */
18162306a36Sopenharmony_cistatic int vmw_cotable_unscrub(struct vmw_resource *res)
18262306a36Sopenharmony_ci{
18362306a36Sopenharmony_ci	struct vmw_cotable *vcotbl = vmw_cotable(res);
18462306a36Sopenharmony_ci	struct vmw_private *dev_priv = res->dev_priv;
18562306a36Sopenharmony_ci	struct ttm_buffer_object *bo = &res->guest_memory_bo->tbo;
18662306a36Sopenharmony_ci	struct {
18762306a36Sopenharmony_ci		SVGA3dCmdHeader header;
18862306a36Sopenharmony_ci		SVGA3dCmdDXSetCOTable body;
18962306a36Sopenharmony_ci	} *cmd;
19062306a36Sopenharmony_ci
19162306a36Sopenharmony_ci	WARN_ON_ONCE(bo->resource->mem_type != VMW_PL_MOB);
19262306a36Sopenharmony_ci	dma_resv_assert_held(bo->base.resv);
19362306a36Sopenharmony_ci
19462306a36Sopenharmony_ci	cmd = VMW_CMD_RESERVE(dev_priv, sizeof(*cmd));
19562306a36Sopenharmony_ci	if (!cmd)
19662306a36Sopenharmony_ci		return -ENOMEM;
19762306a36Sopenharmony_ci
19862306a36Sopenharmony_ci	WARN_ON(vcotbl->ctx->id == SVGA3D_INVALID_ID);
19962306a36Sopenharmony_ci	WARN_ON(bo->resource->mem_type != VMW_PL_MOB);
20062306a36Sopenharmony_ci	cmd->header.id = SVGA_3D_CMD_DX_SET_COTABLE;
20162306a36Sopenharmony_ci	cmd->header.size = sizeof(cmd->body);
20262306a36Sopenharmony_ci	cmd->body.cid = vcotbl->ctx->id;
20362306a36Sopenharmony_ci	cmd->body.type = vcotbl->type;
20462306a36Sopenharmony_ci	cmd->body.mobid = bo->resource->start;
20562306a36Sopenharmony_ci	cmd->body.validSizeInBytes = vcotbl->size_read_back;
20662306a36Sopenharmony_ci
20762306a36Sopenharmony_ci	vmw_cmd_commit_flush(dev_priv, sizeof(*cmd));
20862306a36Sopenharmony_ci	vcotbl->scrubbed = false;
20962306a36Sopenharmony_ci
21062306a36Sopenharmony_ci	return 0;
21162306a36Sopenharmony_ci}
21262306a36Sopenharmony_ci
21362306a36Sopenharmony_ci/**
21462306a36Sopenharmony_ci * vmw_cotable_bind - Undo a cotable unscrub operation
21562306a36Sopenharmony_ci *
21662306a36Sopenharmony_ci * @res: Pointer to the cotable resource
21762306a36Sopenharmony_ci * @val_buf: Pointer to a struct ttm_validate_buffer prepared by the caller
21862306a36Sopenharmony_ci * for convenience / fencing.
21962306a36Sopenharmony_ci *
22062306a36Sopenharmony_ci * This function issues commands to (re)bind the cotable to
22162306a36Sopenharmony_ci * its backing mob, which needs to be validated and reserved at this point.
22262306a36Sopenharmony_ci */
22362306a36Sopenharmony_cistatic int vmw_cotable_bind(struct vmw_resource *res,
22462306a36Sopenharmony_ci			    struct ttm_validate_buffer *val_buf)
22562306a36Sopenharmony_ci{
22662306a36Sopenharmony_ci	/*
22762306a36Sopenharmony_ci	 * The create() callback may have changed @res->backup without
22862306a36Sopenharmony_ci	 * the caller noticing, and with val_buf->bo still pointing to
22962306a36Sopenharmony_ci	 * the old backup buffer. Although hackish, and not used currently,
23062306a36Sopenharmony_ci	 * take the opportunity to correct the value here so that it's not
23162306a36Sopenharmony_ci	 * misused in the future.
23262306a36Sopenharmony_ci	 */
23362306a36Sopenharmony_ci	val_buf->bo = &res->guest_memory_bo->tbo;
23462306a36Sopenharmony_ci
23562306a36Sopenharmony_ci	return vmw_cotable_unscrub(res);
23662306a36Sopenharmony_ci}
23762306a36Sopenharmony_ci
23862306a36Sopenharmony_ci/**
23962306a36Sopenharmony_ci * vmw_cotable_scrub - Scrub the cotable from the device.
24062306a36Sopenharmony_ci *
24162306a36Sopenharmony_ci * @res: Pointer to the cotable resource.
24262306a36Sopenharmony_ci * @readback: Whether initiate a readback of the cotable data to the backup
24362306a36Sopenharmony_ci * buffer.
24462306a36Sopenharmony_ci *
24562306a36Sopenharmony_ci * In some situations (context swapouts) it might be desirable to make the
24662306a36Sopenharmony_ci * device forget about the cotable without performing a full unbind. A full
24762306a36Sopenharmony_ci * unbind requires reserved backup buffers and it might not be possible to
24862306a36Sopenharmony_ci * reserve them due to locking order violation issues. The vmw_cotable_scrub
24962306a36Sopenharmony_ci * function implements a partial unbind() without that requirement but with the
25062306a36Sopenharmony_ci * following restrictions.
25162306a36Sopenharmony_ci * 1) Before the cotable is again used by the GPU, vmw_cotable_unscrub() must
25262306a36Sopenharmony_ci *    be called.
25362306a36Sopenharmony_ci * 2) Before the cotable backing buffer is used by the CPU, or during the
25462306a36Sopenharmony_ci *    resource destruction, vmw_cotable_unbind() must be called.
25562306a36Sopenharmony_ci */
25662306a36Sopenharmony_ciint vmw_cotable_scrub(struct vmw_resource *res, bool readback)
25762306a36Sopenharmony_ci{
25862306a36Sopenharmony_ci	struct vmw_cotable *vcotbl = vmw_cotable(res);
25962306a36Sopenharmony_ci	struct vmw_private *dev_priv = res->dev_priv;
26062306a36Sopenharmony_ci	size_t submit_size;
26162306a36Sopenharmony_ci
26262306a36Sopenharmony_ci	struct {
26362306a36Sopenharmony_ci		SVGA3dCmdHeader header;
26462306a36Sopenharmony_ci		SVGA3dCmdDXReadbackCOTable body;
26562306a36Sopenharmony_ci	} *cmd0;
26662306a36Sopenharmony_ci	struct {
26762306a36Sopenharmony_ci		SVGA3dCmdHeader header;
26862306a36Sopenharmony_ci		SVGA3dCmdDXSetCOTable body;
26962306a36Sopenharmony_ci	} *cmd1;
27062306a36Sopenharmony_ci
27162306a36Sopenharmony_ci	if (vcotbl->scrubbed)
27262306a36Sopenharmony_ci		return 0;
27362306a36Sopenharmony_ci
27462306a36Sopenharmony_ci	if (co_info[vcotbl->type].unbind_func)
27562306a36Sopenharmony_ci		co_info[vcotbl->type].unbind_func(dev_priv,
27662306a36Sopenharmony_ci						  &vcotbl->resource_list,
27762306a36Sopenharmony_ci						  readback);
27862306a36Sopenharmony_ci	submit_size = sizeof(*cmd1);
27962306a36Sopenharmony_ci	if (readback)
28062306a36Sopenharmony_ci		submit_size += sizeof(*cmd0);
28162306a36Sopenharmony_ci
28262306a36Sopenharmony_ci	cmd1 = VMW_CMD_RESERVE(dev_priv, submit_size);
28362306a36Sopenharmony_ci	if (!cmd1)
28462306a36Sopenharmony_ci		return -ENOMEM;
28562306a36Sopenharmony_ci
28662306a36Sopenharmony_ci	vcotbl->size_read_back = 0;
28762306a36Sopenharmony_ci	if (readback) {
28862306a36Sopenharmony_ci		cmd0 = (void *) cmd1;
28962306a36Sopenharmony_ci		cmd0->header.id = SVGA_3D_CMD_DX_READBACK_COTABLE;
29062306a36Sopenharmony_ci		cmd0->header.size = sizeof(cmd0->body);
29162306a36Sopenharmony_ci		cmd0->body.cid = vcotbl->ctx->id;
29262306a36Sopenharmony_ci		cmd0->body.type = vcotbl->type;
29362306a36Sopenharmony_ci		cmd1 = (void *) &cmd0[1];
29462306a36Sopenharmony_ci		vcotbl->size_read_back = res->guest_memory_size;
29562306a36Sopenharmony_ci	}
29662306a36Sopenharmony_ci	cmd1->header.id = SVGA_3D_CMD_DX_SET_COTABLE;
29762306a36Sopenharmony_ci	cmd1->header.size = sizeof(cmd1->body);
29862306a36Sopenharmony_ci	cmd1->body.cid = vcotbl->ctx->id;
29962306a36Sopenharmony_ci	cmd1->body.type = vcotbl->type;
30062306a36Sopenharmony_ci	cmd1->body.mobid = SVGA3D_INVALID_ID;
30162306a36Sopenharmony_ci	cmd1->body.validSizeInBytes = 0;
30262306a36Sopenharmony_ci	vmw_cmd_commit_flush(dev_priv, submit_size);
30362306a36Sopenharmony_ci	vcotbl->scrubbed = true;
30462306a36Sopenharmony_ci
30562306a36Sopenharmony_ci	/* Trigger a create() on next validate. */
30662306a36Sopenharmony_ci	res->id = -1;
30762306a36Sopenharmony_ci
30862306a36Sopenharmony_ci	return 0;
30962306a36Sopenharmony_ci}
31062306a36Sopenharmony_ci
31162306a36Sopenharmony_ci/**
31262306a36Sopenharmony_ci * vmw_cotable_unbind - Cotable resource unbind callback
31362306a36Sopenharmony_ci *
31462306a36Sopenharmony_ci * @res: Pointer to the cotable resource.
31562306a36Sopenharmony_ci * @readback: Whether to read back cotable data to the backup buffer.
31662306a36Sopenharmony_ci * @val_buf: Pointer to a struct ttm_validate_buffer prepared by the caller
31762306a36Sopenharmony_ci * for convenience / fencing.
31862306a36Sopenharmony_ci *
31962306a36Sopenharmony_ci * Unbinds the cotable from the device and fences the backup buffer.
32062306a36Sopenharmony_ci */
32162306a36Sopenharmony_cistatic int vmw_cotable_unbind(struct vmw_resource *res,
32262306a36Sopenharmony_ci			      bool readback,
32362306a36Sopenharmony_ci			      struct ttm_validate_buffer *val_buf)
32462306a36Sopenharmony_ci{
32562306a36Sopenharmony_ci	struct vmw_cotable *vcotbl = vmw_cotable(res);
32662306a36Sopenharmony_ci	struct vmw_private *dev_priv = res->dev_priv;
32762306a36Sopenharmony_ci	struct ttm_buffer_object *bo = val_buf->bo;
32862306a36Sopenharmony_ci	struct vmw_fence_obj *fence;
32962306a36Sopenharmony_ci
33062306a36Sopenharmony_ci	if (!vmw_resource_mob_attached(res))
33162306a36Sopenharmony_ci		return 0;
33262306a36Sopenharmony_ci
33362306a36Sopenharmony_ci	WARN_ON_ONCE(bo->resource->mem_type != VMW_PL_MOB);
33462306a36Sopenharmony_ci	dma_resv_assert_held(bo->base.resv);
33562306a36Sopenharmony_ci
33662306a36Sopenharmony_ci	mutex_lock(&dev_priv->binding_mutex);
33762306a36Sopenharmony_ci	if (!vcotbl->scrubbed)
33862306a36Sopenharmony_ci		vmw_dx_context_scrub_cotables(vcotbl->ctx, readback);
33962306a36Sopenharmony_ci	mutex_unlock(&dev_priv->binding_mutex);
34062306a36Sopenharmony_ci	(void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
34162306a36Sopenharmony_ci	vmw_bo_fence_single(bo, fence);
34262306a36Sopenharmony_ci	if (likely(fence != NULL))
34362306a36Sopenharmony_ci		vmw_fence_obj_unreference(&fence);
34462306a36Sopenharmony_ci
34562306a36Sopenharmony_ci	return 0;
34662306a36Sopenharmony_ci}
34762306a36Sopenharmony_ci
34862306a36Sopenharmony_ci/**
34962306a36Sopenharmony_ci * vmw_cotable_readback - Read back a cotable without unbinding.
35062306a36Sopenharmony_ci *
35162306a36Sopenharmony_ci * @res: The cotable resource.
35262306a36Sopenharmony_ci *
35362306a36Sopenharmony_ci * Reads back a cotable to its backing mob without scrubbing the MOB from
35462306a36Sopenharmony_ci * the cotable. The MOB is fenced for subsequent CPU access.
35562306a36Sopenharmony_ci */
35662306a36Sopenharmony_cistatic int vmw_cotable_readback(struct vmw_resource *res)
35762306a36Sopenharmony_ci{
35862306a36Sopenharmony_ci	struct vmw_cotable *vcotbl = vmw_cotable(res);
35962306a36Sopenharmony_ci	struct vmw_private *dev_priv = res->dev_priv;
36062306a36Sopenharmony_ci
36162306a36Sopenharmony_ci	struct {
36262306a36Sopenharmony_ci		SVGA3dCmdHeader header;
36362306a36Sopenharmony_ci		SVGA3dCmdDXReadbackCOTable body;
36462306a36Sopenharmony_ci	} *cmd;
36562306a36Sopenharmony_ci	struct vmw_fence_obj *fence;
36662306a36Sopenharmony_ci
36762306a36Sopenharmony_ci	if (!vcotbl->scrubbed) {
36862306a36Sopenharmony_ci		cmd = VMW_CMD_RESERVE(dev_priv, sizeof(*cmd));
36962306a36Sopenharmony_ci		if (!cmd)
37062306a36Sopenharmony_ci			return -ENOMEM;
37162306a36Sopenharmony_ci
37262306a36Sopenharmony_ci		cmd->header.id = SVGA_3D_CMD_DX_READBACK_COTABLE;
37362306a36Sopenharmony_ci		cmd->header.size = sizeof(cmd->body);
37462306a36Sopenharmony_ci		cmd->body.cid = vcotbl->ctx->id;
37562306a36Sopenharmony_ci		cmd->body.type = vcotbl->type;
37662306a36Sopenharmony_ci		vcotbl->size_read_back = res->guest_memory_size;
37762306a36Sopenharmony_ci		vmw_cmd_commit(dev_priv, sizeof(*cmd));
37862306a36Sopenharmony_ci	}
37962306a36Sopenharmony_ci
38062306a36Sopenharmony_ci	(void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
38162306a36Sopenharmony_ci	vmw_bo_fence_single(&res->guest_memory_bo->tbo, fence);
38262306a36Sopenharmony_ci	vmw_fence_obj_unreference(&fence);
38362306a36Sopenharmony_ci
38462306a36Sopenharmony_ci	return 0;
38562306a36Sopenharmony_ci}
38662306a36Sopenharmony_ci
38762306a36Sopenharmony_ci/**
38862306a36Sopenharmony_ci * vmw_cotable_resize - Resize a cotable.
38962306a36Sopenharmony_ci *
39062306a36Sopenharmony_ci * @res: The cotable resource.
39162306a36Sopenharmony_ci * @new_size: The new size.
39262306a36Sopenharmony_ci *
39362306a36Sopenharmony_ci * Resizes a cotable and binds the new backup buffer.
39462306a36Sopenharmony_ci * On failure the cotable is left intact.
39562306a36Sopenharmony_ci * Important! This function may not fail once the MOB switch has been
39662306a36Sopenharmony_ci * committed to hardware. That would put the device context in an
39762306a36Sopenharmony_ci * invalid state which we can't currently recover from.
39862306a36Sopenharmony_ci */
39962306a36Sopenharmony_cistatic int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
40062306a36Sopenharmony_ci{
40162306a36Sopenharmony_ci	struct ttm_operation_ctx ctx = { false, false };
40262306a36Sopenharmony_ci	struct vmw_private *dev_priv = res->dev_priv;
40362306a36Sopenharmony_ci	struct vmw_cotable *vcotbl = vmw_cotable(res);
40462306a36Sopenharmony_ci	struct vmw_bo *buf, *old_buf = res->guest_memory_bo;
40562306a36Sopenharmony_ci	struct ttm_buffer_object *bo, *old_bo = &res->guest_memory_bo->tbo;
40662306a36Sopenharmony_ci	size_t old_size = res->guest_memory_size;
40762306a36Sopenharmony_ci	size_t old_size_read_back = vcotbl->size_read_back;
40862306a36Sopenharmony_ci	size_t cur_size_read_back;
40962306a36Sopenharmony_ci	struct ttm_bo_kmap_obj old_map, new_map;
41062306a36Sopenharmony_ci	int ret;
41162306a36Sopenharmony_ci	size_t i;
41262306a36Sopenharmony_ci	struct vmw_bo_params bo_params = {
41362306a36Sopenharmony_ci		.domain = VMW_BO_DOMAIN_MOB,
41462306a36Sopenharmony_ci		.busy_domain = VMW_BO_DOMAIN_MOB,
41562306a36Sopenharmony_ci		.bo_type = ttm_bo_type_device,
41662306a36Sopenharmony_ci		.size = new_size,
41762306a36Sopenharmony_ci		.pin = true
41862306a36Sopenharmony_ci	};
41962306a36Sopenharmony_ci
42062306a36Sopenharmony_ci	MKS_STAT_TIME_DECL(MKSSTAT_KERN_COTABLE_RESIZE);
42162306a36Sopenharmony_ci	MKS_STAT_TIME_PUSH(MKSSTAT_KERN_COTABLE_RESIZE);
42262306a36Sopenharmony_ci
42362306a36Sopenharmony_ci	ret = vmw_cotable_readback(res);
42462306a36Sopenharmony_ci	if (ret)
42562306a36Sopenharmony_ci		goto out_done;
42662306a36Sopenharmony_ci
42762306a36Sopenharmony_ci	cur_size_read_back = vcotbl->size_read_back;
42862306a36Sopenharmony_ci	vcotbl->size_read_back = old_size_read_back;
42962306a36Sopenharmony_ci
43062306a36Sopenharmony_ci	/*
43162306a36Sopenharmony_ci	 * While device is processing, Allocate and reserve a buffer object
43262306a36Sopenharmony_ci	 * for the new COTable. Initially pin the buffer object to make sure
43362306a36Sopenharmony_ci	 * we can use tryreserve without failure.
43462306a36Sopenharmony_ci	 */
43562306a36Sopenharmony_ci	ret = vmw_gem_object_create(dev_priv, &bo_params, &buf);
43662306a36Sopenharmony_ci	if (ret) {
43762306a36Sopenharmony_ci		DRM_ERROR("Failed initializing new cotable MOB.\n");
43862306a36Sopenharmony_ci		goto out_done;
43962306a36Sopenharmony_ci	}
44062306a36Sopenharmony_ci
44162306a36Sopenharmony_ci	bo = &buf->tbo;
44262306a36Sopenharmony_ci	WARN_ON_ONCE(ttm_bo_reserve(bo, false, true, NULL));
44362306a36Sopenharmony_ci
44462306a36Sopenharmony_ci	ret = ttm_bo_wait(old_bo, false, false);
44562306a36Sopenharmony_ci	if (unlikely(ret != 0)) {
44662306a36Sopenharmony_ci		DRM_ERROR("Failed waiting for cotable unbind.\n");
44762306a36Sopenharmony_ci		goto out_wait;
44862306a36Sopenharmony_ci	}
44962306a36Sopenharmony_ci
45062306a36Sopenharmony_ci	/*
45162306a36Sopenharmony_ci	 * Do a page by page copy of COTables. This eliminates slow vmap()s.
45262306a36Sopenharmony_ci	 * This should really be a TTM utility.
45362306a36Sopenharmony_ci	 */
45462306a36Sopenharmony_ci	for (i = 0; i < PFN_UP(old_bo->resource->size); ++i) {
45562306a36Sopenharmony_ci		bool dummy;
45662306a36Sopenharmony_ci
45762306a36Sopenharmony_ci		ret = ttm_bo_kmap(old_bo, i, 1, &old_map);
45862306a36Sopenharmony_ci		if (unlikely(ret != 0)) {
45962306a36Sopenharmony_ci			DRM_ERROR("Failed mapping old COTable on resize.\n");
46062306a36Sopenharmony_ci			goto out_wait;
46162306a36Sopenharmony_ci		}
46262306a36Sopenharmony_ci		ret = ttm_bo_kmap(bo, i, 1, &new_map);
46362306a36Sopenharmony_ci		if (unlikely(ret != 0)) {
46462306a36Sopenharmony_ci			DRM_ERROR("Failed mapping new COTable on resize.\n");
46562306a36Sopenharmony_ci			goto out_map_new;
46662306a36Sopenharmony_ci		}
46762306a36Sopenharmony_ci		memcpy(ttm_kmap_obj_virtual(&new_map, &dummy),
46862306a36Sopenharmony_ci		       ttm_kmap_obj_virtual(&old_map, &dummy),
46962306a36Sopenharmony_ci		       PAGE_SIZE);
47062306a36Sopenharmony_ci		ttm_bo_kunmap(&new_map);
47162306a36Sopenharmony_ci		ttm_bo_kunmap(&old_map);
47262306a36Sopenharmony_ci	}
47362306a36Sopenharmony_ci
47462306a36Sopenharmony_ci	/* Unpin new buffer, and switch backup buffers. */
47562306a36Sopenharmony_ci	vmw_bo_placement_set(buf,
47662306a36Sopenharmony_ci			     VMW_BO_DOMAIN_MOB,
47762306a36Sopenharmony_ci			     VMW_BO_DOMAIN_MOB);
47862306a36Sopenharmony_ci	ret = ttm_bo_validate(bo, &buf->placement, &ctx);
47962306a36Sopenharmony_ci	if (unlikely(ret != 0)) {
48062306a36Sopenharmony_ci		DRM_ERROR("Failed validating new COTable backup buffer.\n");
48162306a36Sopenharmony_ci		goto out_wait;
48262306a36Sopenharmony_ci	}
48362306a36Sopenharmony_ci
48462306a36Sopenharmony_ci	vmw_resource_mob_detach(res);
48562306a36Sopenharmony_ci	res->guest_memory_bo = buf;
48662306a36Sopenharmony_ci	res->guest_memory_size = new_size;
48762306a36Sopenharmony_ci	vcotbl->size_read_back = cur_size_read_back;
48862306a36Sopenharmony_ci
48962306a36Sopenharmony_ci	/*
49062306a36Sopenharmony_ci	 * Now tell the device to switch. If this fails, then we need to
49162306a36Sopenharmony_ci	 * revert the full resize.
49262306a36Sopenharmony_ci	 */
49362306a36Sopenharmony_ci	ret = vmw_cotable_unscrub(res);
49462306a36Sopenharmony_ci	if (ret) {
49562306a36Sopenharmony_ci		DRM_ERROR("Failed switching COTable backup buffer.\n");
49662306a36Sopenharmony_ci		res->guest_memory_bo = old_buf;
49762306a36Sopenharmony_ci		res->guest_memory_size = old_size;
49862306a36Sopenharmony_ci		vcotbl->size_read_back = old_size_read_back;
49962306a36Sopenharmony_ci		vmw_resource_mob_attach(res);
50062306a36Sopenharmony_ci		goto out_wait;
50162306a36Sopenharmony_ci	}
50262306a36Sopenharmony_ci
50362306a36Sopenharmony_ci	vmw_resource_mob_attach(res);
50462306a36Sopenharmony_ci	/* Let go of the old mob. */
50562306a36Sopenharmony_ci	vmw_user_bo_unref(&old_buf);
50662306a36Sopenharmony_ci	res->id = vcotbl->type;
50762306a36Sopenharmony_ci
50862306a36Sopenharmony_ci	ret = dma_resv_reserve_fences(bo->base.resv, 1);
50962306a36Sopenharmony_ci	if (unlikely(ret))
51062306a36Sopenharmony_ci		goto out_wait;
51162306a36Sopenharmony_ci
51262306a36Sopenharmony_ci	/* Release the pin acquired in vmw_bo_create */
51362306a36Sopenharmony_ci	ttm_bo_unpin(bo);
51462306a36Sopenharmony_ci
51562306a36Sopenharmony_ci	MKS_STAT_TIME_POP(MKSSTAT_KERN_COTABLE_RESIZE);
51662306a36Sopenharmony_ci
51762306a36Sopenharmony_ci	return 0;
51862306a36Sopenharmony_ci
51962306a36Sopenharmony_ciout_map_new:
52062306a36Sopenharmony_ci	ttm_bo_kunmap(&old_map);
52162306a36Sopenharmony_ciout_wait:
52262306a36Sopenharmony_ci	ttm_bo_unpin(bo);
52362306a36Sopenharmony_ci	ttm_bo_unreserve(bo);
52462306a36Sopenharmony_ci	vmw_user_bo_unref(&buf);
52562306a36Sopenharmony_ci
52662306a36Sopenharmony_ciout_done:
52762306a36Sopenharmony_ci	MKS_STAT_TIME_POP(MKSSTAT_KERN_COTABLE_RESIZE);
52862306a36Sopenharmony_ci
52962306a36Sopenharmony_ci	return ret;
53062306a36Sopenharmony_ci}
53162306a36Sopenharmony_ci
53262306a36Sopenharmony_ci/**
53362306a36Sopenharmony_ci * vmw_cotable_create - Cotable resource create callback
53462306a36Sopenharmony_ci *
53562306a36Sopenharmony_ci * @res: Pointer to a cotable resource.
53662306a36Sopenharmony_ci *
53762306a36Sopenharmony_ci * There is no separate create command for cotables, so this callback, which
53862306a36Sopenharmony_ci * is called before bind() in the validation sequence is instead used for two
53962306a36Sopenharmony_ci * things.
54062306a36Sopenharmony_ci * 1) Unscrub the cotable if it is scrubbed and still attached to a backup
54162306a36Sopenharmony_ci *    buffer.
54262306a36Sopenharmony_ci * 2) Resize the cotable if needed.
54362306a36Sopenharmony_ci */
54462306a36Sopenharmony_cistatic int vmw_cotable_create(struct vmw_resource *res)
54562306a36Sopenharmony_ci{
54662306a36Sopenharmony_ci	struct vmw_cotable *vcotbl = vmw_cotable(res);
54762306a36Sopenharmony_ci	size_t new_size = res->guest_memory_size;
54862306a36Sopenharmony_ci	size_t needed_size;
54962306a36Sopenharmony_ci	int ret;
55062306a36Sopenharmony_ci
55162306a36Sopenharmony_ci	/* Check whether we need to resize the cotable */
55262306a36Sopenharmony_ci	needed_size = (vcotbl->seen_entries + 1) * co_info[vcotbl->type].size;
55362306a36Sopenharmony_ci	while (needed_size > new_size)
55462306a36Sopenharmony_ci		new_size *= 2;
55562306a36Sopenharmony_ci
55662306a36Sopenharmony_ci	if (likely(new_size <= res->guest_memory_size)) {
55762306a36Sopenharmony_ci		if (vcotbl->scrubbed && vmw_resource_mob_attached(res)) {
55862306a36Sopenharmony_ci			ret = vmw_cotable_unscrub(res);
55962306a36Sopenharmony_ci			if (ret)
56062306a36Sopenharmony_ci				return ret;
56162306a36Sopenharmony_ci		}
56262306a36Sopenharmony_ci		res->id = vcotbl->type;
56362306a36Sopenharmony_ci		return 0;
56462306a36Sopenharmony_ci	}
56562306a36Sopenharmony_ci
56662306a36Sopenharmony_ci	return vmw_cotable_resize(res, new_size);
56762306a36Sopenharmony_ci}
56862306a36Sopenharmony_ci
56962306a36Sopenharmony_ci/**
57062306a36Sopenharmony_ci * vmw_hw_cotable_destroy - Cotable hw_destroy callback
57162306a36Sopenharmony_ci *
57262306a36Sopenharmony_ci * @res: Pointer to a cotable resource.
57362306a36Sopenharmony_ci *
57462306a36Sopenharmony_ci * The final (part of resource destruction) destroy callback.
57562306a36Sopenharmony_ci */
57662306a36Sopenharmony_cistatic void vmw_hw_cotable_destroy(struct vmw_resource *res)
57762306a36Sopenharmony_ci{
57862306a36Sopenharmony_ci	(void) vmw_cotable_destroy(res);
57962306a36Sopenharmony_ci}
58062306a36Sopenharmony_ci
58162306a36Sopenharmony_ci/**
58262306a36Sopenharmony_ci * vmw_cotable_free - Cotable resource destructor
58362306a36Sopenharmony_ci *
58462306a36Sopenharmony_ci * @res: Pointer to a cotable resource.
58562306a36Sopenharmony_ci */
58662306a36Sopenharmony_cistatic void vmw_cotable_free(struct vmw_resource *res)
58762306a36Sopenharmony_ci{
58862306a36Sopenharmony_ci	kfree(res);
58962306a36Sopenharmony_ci}
59062306a36Sopenharmony_ci
59162306a36Sopenharmony_ci/**
59262306a36Sopenharmony_ci * vmw_cotable_alloc - Create a cotable resource
59362306a36Sopenharmony_ci *
59462306a36Sopenharmony_ci * @dev_priv: Pointer to a device private struct.
59562306a36Sopenharmony_ci * @ctx: Pointer to the context resource.
59662306a36Sopenharmony_ci * The cotable resource will not add a refcount.
59762306a36Sopenharmony_ci * @type: The cotable type.
59862306a36Sopenharmony_ci */
59962306a36Sopenharmony_cistruct vmw_resource *vmw_cotable_alloc(struct vmw_private *dev_priv,
60062306a36Sopenharmony_ci				       struct vmw_resource *ctx,
60162306a36Sopenharmony_ci				       u32 type)
60262306a36Sopenharmony_ci{
60362306a36Sopenharmony_ci	struct vmw_cotable *vcotbl;
60462306a36Sopenharmony_ci	int ret;
60562306a36Sopenharmony_ci	u32 num_entries;
60662306a36Sopenharmony_ci
60762306a36Sopenharmony_ci	vcotbl = kzalloc(sizeof(*vcotbl), GFP_KERNEL);
60862306a36Sopenharmony_ci	if (unlikely(!vcotbl)) {
60962306a36Sopenharmony_ci		ret = -ENOMEM;
61062306a36Sopenharmony_ci		goto out_no_alloc;
61162306a36Sopenharmony_ci	}
61262306a36Sopenharmony_ci
61362306a36Sopenharmony_ci	ret = vmw_resource_init(dev_priv, &vcotbl->res, true,
61462306a36Sopenharmony_ci				vmw_cotable_free, &vmw_cotable_func);
61562306a36Sopenharmony_ci	if (unlikely(ret != 0))
61662306a36Sopenharmony_ci		goto out_no_init;
61762306a36Sopenharmony_ci
61862306a36Sopenharmony_ci	INIT_LIST_HEAD(&vcotbl->resource_list);
61962306a36Sopenharmony_ci	vcotbl->res.id = type;
62062306a36Sopenharmony_ci	vcotbl->res.guest_memory_size = PAGE_SIZE;
62162306a36Sopenharmony_ci	num_entries = PAGE_SIZE / co_info[type].size;
62262306a36Sopenharmony_ci	if (num_entries < co_info[type].min_initial_entries) {
62362306a36Sopenharmony_ci		vcotbl->res.guest_memory_size = co_info[type].min_initial_entries *
62462306a36Sopenharmony_ci			co_info[type].size;
62562306a36Sopenharmony_ci		vcotbl->res.guest_memory_size = PFN_ALIGN(vcotbl->res.guest_memory_size);
62662306a36Sopenharmony_ci	}
62762306a36Sopenharmony_ci
62862306a36Sopenharmony_ci	vcotbl->scrubbed = true;
62962306a36Sopenharmony_ci	vcotbl->seen_entries = -1;
63062306a36Sopenharmony_ci	vcotbl->type = type;
63162306a36Sopenharmony_ci	vcotbl->ctx = ctx;
63262306a36Sopenharmony_ci
63362306a36Sopenharmony_ci	vcotbl->res.hw_destroy = vmw_hw_cotable_destroy;
63462306a36Sopenharmony_ci
63562306a36Sopenharmony_ci	return &vcotbl->res;
63662306a36Sopenharmony_ci
63762306a36Sopenharmony_ciout_no_init:
63862306a36Sopenharmony_ci	kfree(vcotbl);
63962306a36Sopenharmony_ciout_no_alloc:
64062306a36Sopenharmony_ci	return ERR_PTR(ret);
64162306a36Sopenharmony_ci}
64262306a36Sopenharmony_ci
64362306a36Sopenharmony_ci/**
64462306a36Sopenharmony_ci * vmw_cotable_notify - Notify the cotable about an item creation
64562306a36Sopenharmony_ci *
64662306a36Sopenharmony_ci * @res: Pointer to a cotable resource.
64762306a36Sopenharmony_ci * @id: Item id.
64862306a36Sopenharmony_ci */
64962306a36Sopenharmony_ciint vmw_cotable_notify(struct vmw_resource *res, int id)
65062306a36Sopenharmony_ci{
65162306a36Sopenharmony_ci	struct vmw_cotable *vcotbl = vmw_cotable(res);
65262306a36Sopenharmony_ci
65362306a36Sopenharmony_ci	if (id < 0 || id >= SVGA_COTABLE_MAX_IDS) {
65462306a36Sopenharmony_ci		DRM_ERROR("Illegal COTable id. Type is %u. Id is %d\n",
65562306a36Sopenharmony_ci			  (unsigned) vcotbl->type, id);
65662306a36Sopenharmony_ci		return -EINVAL;
65762306a36Sopenharmony_ci	}
65862306a36Sopenharmony_ci
65962306a36Sopenharmony_ci	if (vcotbl->seen_entries < id) {
66062306a36Sopenharmony_ci		/* Trigger a call to create() on next validate */
66162306a36Sopenharmony_ci		res->id = -1;
66262306a36Sopenharmony_ci		vcotbl->seen_entries = id;
66362306a36Sopenharmony_ci	}
66462306a36Sopenharmony_ci
66562306a36Sopenharmony_ci	return 0;
66662306a36Sopenharmony_ci}
66762306a36Sopenharmony_ci
66862306a36Sopenharmony_ci/**
66962306a36Sopenharmony_ci * vmw_cotable_add_resource - add a view to the cotable's list of active views.
67062306a36Sopenharmony_ci *
67162306a36Sopenharmony_ci * @res: pointer struct vmw_resource representing the cotable.
67262306a36Sopenharmony_ci * @head: pointer to the struct list_head member of the resource, dedicated
67362306a36Sopenharmony_ci * to the cotable active resource list.
67462306a36Sopenharmony_ci */
67562306a36Sopenharmony_civoid vmw_cotable_add_resource(struct vmw_resource *res, struct list_head *head)
67662306a36Sopenharmony_ci{
67762306a36Sopenharmony_ci	struct vmw_cotable *vcotbl =
67862306a36Sopenharmony_ci		container_of(res, struct vmw_cotable, res);
67962306a36Sopenharmony_ci
68062306a36Sopenharmony_ci	list_add_tail(head, &vcotbl->resource_list);
68162306a36Sopenharmony_ci}
682