1 /*
2 *
3 * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
4 *
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
8 * of such GNU licence.
9 *
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
13 *
14 */
15
16
17
18 #include <linux/anon_inodes.h>
19 #include <linux/atomic.h>
20 #include <linux/hrtimer.h>
21 #include <linux/jiffies.h>
22 #include <linux/kthread.h>
23 #include <linux/list.h>
24 #include <linux/mm.h>
25 #include <linux/poll.h>
26 #include <linux/preempt.h>
27 #include <linux/slab.h>
28 #include <linux/wait.h>
29
30 #include <mali_kbase.h>
31 #include <mali_kbase_hwaccess_instr.h>
32 #include <mali_kbase_hwaccess_jm.h>
33 #include <mali_kbase_hwcnt_reader.h>
34 #include <mali_kbase_mem_linux.h>
35 #include <mali_kbase_tlstream.h>
36
37 /*****************************************************************************/
38
39 /* Hwcnt reader API version */
40 #define HWCNT_READER_API 1
41
42 /* The number of nanoseconds in a second. */
43 #define NSECS_IN_SEC 1000000000ull /* ns */
44
45 /* The time resolution of dumping service. */
46 #define DUMPING_RESOLUTION 500000ull /* ns */
47
48 /* The maximal supported number of dumping buffers. */
49 #define MAX_BUFFER_COUNT 32
50
51 /* Size and number of hw counters blocks. */
52 #define NR_CNT_BLOCKS_PER_GROUP 8
53 #define NR_CNT_PER_BLOCK 64
54 #define NR_BYTES_PER_CNT 4
55 #define NR_BYTES_PER_HDR 16
56 #define PRFCNT_EN_MASK_OFFSET 0x8
57
58 /*****************************************************************************/
59
60 enum {
61 SHADER_HWCNT_BM,
62 TILER_HWCNT_BM,
63 MMU_L2_HWCNT_BM,
64 JM_HWCNT_BM
65 };
66
67 enum vinstr_state {
68 VINSTR_IDLE,
69 VINSTR_DUMPING,
70 VINSTR_SUSPENDING,
71 VINSTR_SUSPENDED,
72 VINSTR_RESUMING
73 };
74
75 /**
76 * struct kbase_vinstr_context - vinstr context per device
77 * @lock: protects the entire vinstr context
78 * @kbdev: pointer to kbase device
79 * @kctx: pointer to kbase context
80 * @vmap: vinstr vmap for mapping hwcnt dump buffer
81 * @gpu_va: GPU hwcnt dump buffer address
82 * @cpu_va: the CPU side mapping of the hwcnt dump buffer
83 * @dump_size: size of the dump buffer in bytes
84 * @bitmap: current set of counters monitored, not always in sync
85 * with hardware
86 * @reprogram: when true, reprogram hwcnt block with the new set of
87 * counters
88 * @state: vinstr state
89 * @state_lock: protects information about vinstr state
90 * @suspend_waitq: notification queue to trigger state re-validation
91 * @suspend_cnt: reference counter of vinstr's suspend state
92 * @suspend_work: worker to execute on entering suspended state
93 * @resume_work: worker to execute on leaving suspended state
94 * @nclients: number of attached clients, pending or otherwise
95 * @waiting_clients: head of list of clients being periodically sampled
96 * @idle_clients: head of list of clients being idle
97 * @suspended_clients: head of list of clients being suspended
98 * @thread: periodic sampling thread
99 * @waitq: notification queue of sampling thread
100 * @request_pending: request for action for sampling thread
101 */
102 struct kbase_vinstr_context {
103 struct mutex lock;
104 struct kbase_device *kbdev;
105 struct kbase_context *kctx;
106
107 struct kbase_vmap_struct vmap;
108 u64 gpu_va;
109 void *cpu_va;
110 size_t dump_size;
111 u32 bitmap[4];
112 bool reprogram;
113
114 enum vinstr_state state;
115 struct spinlock state_lock;
116 wait_queue_head_t suspend_waitq;
117 unsigned int suspend_cnt;
118 struct work_struct suspend_work;
119 struct work_struct resume_work;
120
121 u32 nclients;
122 struct list_head waiting_clients;
123 struct list_head idle_clients;
124 struct list_head suspended_clients;
125
126 struct task_struct *thread;
127 wait_queue_head_t waitq;
128 atomic_t request_pending;
129 };
130
131 /**
132 * struct kbase_vinstr_client - a vinstr client attached to a vinstr context
133 * @vinstr_ctx: vinstr context client is attached to
134 * @list: node used to attach this client to list in vinstr context
135 * @buffer_count: number of buffers this client is using
136 * @event_mask: events this client reacts to
137 * @dump_size: size of one dump buffer in bytes
138 * @bitmap: bitmap request for JM, TILER, SHADER and MMU counters
139 * @legacy_buffer: userspace hwcnt dump buffer (legacy interface)
140 * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface)
141 * @accum_buffer: temporary accumulation buffer for preserving counters
142 * @dump_time: next time this clients shall request hwcnt dump
143 * @dump_interval: interval between periodic hwcnt dumps
144 * @dump_buffers: kernel hwcnt dump buffers allocated by this client
145 * @dump_buffers_meta: metadata of dump buffers
146 * @meta_idx: index of metadata being accessed by userspace
147 * @read_idx: index of buffer read by userspace
148 * @write_idx: index of buffer being written by dumping service
149 * @waitq: client's notification queue
150 * @pending: when true, client has attached but hwcnt not yet updated
151 */
152 struct kbase_vinstr_client {
153 struct kbase_vinstr_context *vinstr_ctx;
154 struct list_head list;
155 unsigned int buffer_count;
156 u32 event_mask;
157 size_t dump_size;
158 u32 bitmap[4];
159 void __user *legacy_buffer;
160 void *kernel_buffer;
161 void *accum_buffer;
162 u64 dump_time;
163 u32 dump_interval;
164 char *dump_buffers;
165 struct kbase_hwcnt_reader_metadata *dump_buffers_meta;
166 atomic_t meta_idx;
167 atomic_t read_idx;
168 atomic_t write_idx;
169 wait_queue_head_t waitq;
170 bool pending;
171 };
172
173 /**
174 * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer
175 * @hrtimer: high resolution timer
176 * @vinstr_ctx: vinstr context
177 */
178 struct kbasep_vinstr_wake_up_timer {
179 struct hrtimer hrtimer;
180 struct kbase_vinstr_context *vinstr_ctx;
181 };
182
183 /*****************************************************************************/
184
185 static int kbasep_vinstr_service_task(void *data);
186
187 static unsigned int kbasep_vinstr_hwcnt_reader_poll(
188 struct file *filp,
189 poll_table *wait);
190 static long kbasep_vinstr_hwcnt_reader_ioctl(
191 struct file *filp,
192 unsigned int cmd,
193 unsigned long arg);
194 static int kbasep_vinstr_hwcnt_reader_mmap(
195 struct file *filp,
196 struct vm_area_struct *vma);
197 static int kbasep_vinstr_hwcnt_reader_release(
198 struct inode *inode,
199 struct file *filp);
200
201 /* The timeline stream file operations structure. */
202 static const struct file_operations vinstr_client_fops = {
203 .poll = kbasep_vinstr_hwcnt_reader_poll,
204 .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
205 .compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
206 .mmap = kbasep_vinstr_hwcnt_reader_mmap,
207 .release = kbasep_vinstr_hwcnt_reader_release,
208 };
209
210 /*****************************************************************************/
211
enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)212 static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
213 {
214 struct kbase_context *kctx = vinstr_ctx->kctx;
215 struct kbase_device *kbdev = kctx->kbdev;
216 struct kbase_uk_hwcnt_setup setup;
217 int err;
218
219 setup.dump_buffer = vinstr_ctx->gpu_va;
220 setup.jm_bm = vinstr_ctx->bitmap[JM_HWCNT_BM];
221 setup.tiler_bm = vinstr_ctx->bitmap[TILER_HWCNT_BM];
222 setup.shader_bm = vinstr_ctx->bitmap[SHADER_HWCNT_BM];
223 setup.mmu_l2_bm = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM];
224
225 /* Mark the context as active so the GPU is kept turned on */
226 /* A suspend won't happen here, because we're in a syscall from a
227 * userspace thread. */
228 kbase_pm_context_active(kbdev);
229
230 /* Schedule the context in */
231 kbasep_js_schedule_privileged_ctx(kbdev, kctx);
232 err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup);
233 if (err) {
234 /* Release the context. This had its own Power Manager Active
235 * reference */
236 kbasep_js_release_privileged_ctx(kbdev, kctx);
237
238 /* Also release our Power Manager Active reference */
239 kbase_pm_context_idle(kbdev);
240 }
241
242 return err;
243 }
244
disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)245 static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
246 {
247 struct kbase_context *kctx = vinstr_ctx->kctx;
248 struct kbase_device *kbdev = kctx->kbdev;
249 int err;
250
251 err = kbase_instr_hwcnt_disable_internal(kctx);
252 if (err) {
253 dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)",
254 kctx);
255 return;
256 }
257
258 /* Release the context. This had its own Power Manager Active reference. */
259 kbasep_js_release_privileged_ctx(kbdev, kctx);
260
261 /* Also release our Power Manager Active reference. */
262 kbase_pm_context_idle(kbdev);
263
264 dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx);
265 }
266
reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx)267 static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
268 {
269 disable_hwcnt(vinstr_ctx);
270 return enable_hwcnt(vinstr_ctx);
271 }
272
hwcnt_bitmap_set(u32 dst[4], u32 src[4])273 static void hwcnt_bitmap_set(u32 dst[4], u32 src[4])
274 {
275 dst[JM_HWCNT_BM] = src[JM_HWCNT_BM];
276 dst[TILER_HWCNT_BM] = src[TILER_HWCNT_BM];
277 dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM];
278 dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM];
279 }
280
hwcnt_bitmap_union(u32 dst[4], u32 src[4])281 static void hwcnt_bitmap_union(u32 dst[4], u32 src[4])
282 {
283 dst[JM_HWCNT_BM] |= src[JM_HWCNT_BM];
284 dst[TILER_HWCNT_BM] |= src[TILER_HWCNT_BM];
285 dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM];
286 dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM];
287 }
288
kbase_vinstr_dump_size(struct kbase_device *kbdev)289 size_t kbase_vinstr_dump_size(struct kbase_device *kbdev)
290 {
291 size_t dump_size;
292
293 #ifndef CONFIG_MALI_NO_MALI
294 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) {
295 u32 nr_cg;
296
297 nr_cg = kbdev->gpu_props.num_core_groups;
298 dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP *
299 NR_CNT_PER_BLOCK *
300 NR_BYTES_PER_CNT;
301 } else
302 #endif /* CONFIG_MALI_NO_MALI */
303 {
304 /* assume v5 for now */
305 base_gpu_props *props = &kbdev->gpu_props.props;
306 u32 nr_l2 = props->l2_props.num_l2_slices;
307 u64 core_mask = props->coherency_info.group[0].core_mask;
308 u32 nr_blocks = fls64(core_mask);
309
310 /* JM and tiler counter blocks are always present */
311 dump_size = (2 + nr_l2 + nr_blocks) *
312 NR_CNT_PER_BLOCK *
313 NR_BYTES_PER_CNT;
314 }
315 return dump_size;
316 }
317 KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size);
318
kbasep_vinstr_dump_size_ctx( struct kbase_vinstr_context *vinstr_ctx)319 static size_t kbasep_vinstr_dump_size_ctx(
320 struct kbase_vinstr_context *vinstr_ctx)
321 {
322 return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev);
323 }
324
kbasep_vinstr_map_kernel_dump_buffer( struct kbase_vinstr_context *vinstr_ctx)325 static int kbasep_vinstr_map_kernel_dump_buffer(
326 struct kbase_vinstr_context *vinstr_ctx)
327 {
328 struct kbase_va_region *reg;
329 struct kbase_context *kctx = vinstr_ctx->kctx;
330 u64 flags, nr_pages;
331
332 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR;
333 vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
334 nr_pages = PFN_UP(vinstr_ctx->dump_size);
335
336 reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags,
337 &vinstr_ctx->gpu_va);
338 if (!reg)
339 return -ENOMEM;
340
341 vinstr_ctx->cpu_va = kbase_vmap(
342 kctx,
343 vinstr_ctx->gpu_va,
344 vinstr_ctx->dump_size,
345 &vinstr_ctx->vmap);
346 if (!vinstr_ctx->cpu_va) {
347 kbase_mem_free(kctx, vinstr_ctx->gpu_va);
348 return -ENOMEM;
349 }
350
351 return 0;
352 }
353
kbasep_vinstr_unmap_kernel_dump_buffer( struct kbase_vinstr_context *vinstr_ctx)354 static void kbasep_vinstr_unmap_kernel_dump_buffer(
355 struct kbase_vinstr_context *vinstr_ctx)
356 {
357 struct kbase_context *kctx = vinstr_ctx->kctx;
358
359 kbase_vunmap(kctx, &vinstr_ctx->vmap);
360 kbase_mem_free(kctx, vinstr_ctx->gpu_va);
361 }
362
363 /**
364 * kbasep_vinstr_create_kctx - create kernel context for vinstr
365 * @vinstr_ctx: vinstr context
366 * Return: zero on success
367 */
kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx)368 static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx)
369 {
370 struct kbase_device *kbdev = vinstr_ctx->kbdev;
371 struct kbasep_kctx_list_element *element;
372 unsigned long flags;
373 bool enable_backend = false;
374 int err;
375
376 vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true);
377 if (!vinstr_ctx->kctx)
378 return -ENOMEM;
379
380 /* Map the master kernel dump buffer. The HW dumps the counters
381 * into this memory region. */
382 err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx);
383 if (err) {
384 kbase_destroy_context(vinstr_ctx->kctx);
385 vinstr_ctx->kctx = NULL;
386 return err;
387 }
388
389 /* Add kernel context to list of contexts associated with device. */
390 element = kzalloc(sizeof(*element), GFP_KERNEL);
391 if (element) {
392 element->kctx = vinstr_ctx->kctx;
393 mutex_lock(&kbdev->kctx_list_lock);
394 list_add(&element->link, &kbdev->kctx_list);
395
396 /* Inform timeline client about new context.
397 * Do this while holding the lock to avoid tracepoint
398 * being created in both body and summary stream. */
399 KBASE_TLSTREAM_TL_NEW_CTX(
400 vinstr_ctx->kctx,
401 (u32)(vinstr_ctx->kctx->id),
402 (u32)(vinstr_ctx->kctx->tgid));
403
404 mutex_unlock(&kbdev->kctx_list_lock);
405 } else {
406 /* Don't treat this as a fail - just warn about it. */
407 dev_warn(kbdev->dev,
408 "couldn't add kctx to kctx_list\n");
409 }
410
411 /* Don't enable hardware counters if vinstr is suspended.
412 * Note that vinstr resume code is run under vinstr context lock,
413 * lower layer will be enabled as needed on resume. */
414 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
415 if (VINSTR_IDLE == vinstr_ctx->state)
416 enable_backend = true;
417 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
418 if (enable_backend)
419 err = enable_hwcnt(vinstr_ctx);
420
421 if (err) {
422 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
423 kbase_destroy_context(vinstr_ctx->kctx);
424 if (element) {
425 mutex_lock(&kbdev->kctx_list_lock);
426 list_del(&element->link);
427 kfree(element);
428 mutex_unlock(&kbdev->kctx_list_lock);
429 }
430 KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx);
431 vinstr_ctx->kctx = NULL;
432 return err;
433 }
434
435 vinstr_ctx->thread = kthread_run(
436 kbasep_vinstr_service_task,
437 vinstr_ctx,
438 "mali_vinstr_service");
439 if (!vinstr_ctx->thread) {
440 disable_hwcnt(vinstr_ctx);
441 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
442 kbase_destroy_context(vinstr_ctx->kctx);
443 if (element) {
444 mutex_lock(&kbdev->kctx_list_lock);
445 list_del(&element->link);
446 kfree(element);
447 mutex_unlock(&kbdev->kctx_list_lock);
448 }
449 KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx);
450 vinstr_ctx->kctx = NULL;
451 return -EFAULT;
452 }
453
454 return 0;
455 }
456
457 /**
458 * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context
459 * @vinstr_ctx: vinstr context
460 */
kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx)461 static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx)
462 {
463 struct kbase_device *kbdev = vinstr_ctx->kbdev;
464 struct kbasep_kctx_list_element *element;
465 struct kbasep_kctx_list_element *tmp;
466 bool found = false;
467
468 /* Release hw counters dumping resources. */
469 vinstr_ctx->thread = NULL;
470 disable_hwcnt(vinstr_ctx);
471 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
472 kbase_destroy_context(vinstr_ctx->kctx);
473
474 /* Remove kernel context from the device's contexts list. */
475 mutex_lock(&kbdev->kctx_list_lock);
476 list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) {
477 if (element->kctx == vinstr_ctx->kctx) {
478 list_del(&element->link);
479 kfree(element);
480 found = true;
481 }
482 }
483 mutex_unlock(&kbdev->kctx_list_lock);
484
485 if (!found)
486 dev_warn(kbdev->dev, "kctx not in kctx_list\n");
487
488 /* Inform timeline client about context destruction. */
489 KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx);
490
491 vinstr_ctx->kctx = NULL;
492 }
493
494 /**
495 * kbasep_vinstr_attach_client - Attach a client to the vinstr core
496 * @vinstr_ctx: vinstr context
497 * @buffer_count: requested number of dump buffers
498 * @bitmap: bitmaps describing which counters should be enabled
499 * @argp: pointer where notification descriptor shall be stored
500 * @kernel_buffer: pointer to kernel side buffer
501 *
502 * Return: vinstr opaque client handle or NULL on failure
503 */
kbasep_vinstr_attach_client( struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count, u32 bitmap[4], void *argp, void *kernel_buffer)504 static struct kbase_vinstr_client *kbasep_vinstr_attach_client(
505 struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count,
506 u32 bitmap[4], void *argp, void *kernel_buffer)
507 {
508 struct task_struct *thread = NULL;
509 struct kbase_vinstr_client *cli;
510
511 KBASE_DEBUG_ASSERT(vinstr_ctx);
512
513 if (buffer_count > MAX_BUFFER_COUNT
514 || (buffer_count & (buffer_count - 1)))
515 return NULL;
516
517 cli = kzalloc(sizeof(*cli), GFP_KERNEL);
518 if (!cli)
519 return NULL;
520
521 cli->vinstr_ctx = vinstr_ctx;
522 cli->buffer_count = buffer_count;
523 cli->event_mask =
524 (1 << BASE_HWCNT_READER_EVENT_MANUAL) |
525 (1 << BASE_HWCNT_READER_EVENT_PERIODIC);
526 cli->pending = true;
527
528 hwcnt_bitmap_set(cli->bitmap, bitmap);
529
530 mutex_lock(&vinstr_ctx->lock);
531
532 hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap);
533 vinstr_ctx->reprogram = true;
534
535 /* If this is the first client, create the vinstr kbase
536 * context. This context is permanently resident until the
537 * last client exits. */
538 if (!vinstr_ctx->nclients) {
539 hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap);
540 if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0)
541 goto error;
542
543 vinstr_ctx->reprogram = false;
544 cli->pending = false;
545 }
546
547 /* The GPU resets the counter block every time there is a request
548 * to dump it. We need a per client kernel buffer for accumulating
549 * the counters. */
550 cli->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
551 cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL);
552 if (!cli->accum_buffer)
553 goto error;
554
555 /* Prepare buffers. */
556 if (cli->buffer_count) {
557 int *fd = (int *)argp;
558 size_t tmp;
559
560 /* Allocate area for buffers metadata storage. */
561 tmp = sizeof(struct kbase_hwcnt_reader_metadata) *
562 cli->buffer_count;
563 cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL);
564 if (!cli->dump_buffers_meta)
565 goto error;
566
567 /* Allocate required number of dumping buffers. */
568 cli->dump_buffers = (char *)__get_free_pages(
569 GFP_KERNEL | __GFP_ZERO,
570 get_order(cli->dump_size * cli->buffer_count));
571 if (!cli->dump_buffers)
572 goto error;
573
574 /* Create descriptor for user-kernel data exchange. */
575 *fd = anon_inode_getfd(
576 "[mali_vinstr_desc]",
577 &vinstr_client_fops,
578 cli,
579 O_RDONLY | O_CLOEXEC);
580 if (0 > *fd)
581 goto error;
582 } else if (kernel_buffer) {
583 cli->kernel_buffer = kernel_buffer;
584 } else {
585 cli->legacy_buffer = (void __user *)argp;
586 }
587
588 atomic_set(&cli->read_idx, 0);
589 atomic_set(&cli->meta_idx, 0);
590 atomic_set(&cli->write_idx, 0);
591 init_waitqueue_head(&cli->waitq);
592
593 vinstr_ctx->nclients++;
594 list_add(&cli->list, &vinstr_ctx->idle_clients);
595
596 mutex_unlock(&vinstr_ctx->lock);
597
598 return cli;
599
600 error:
601 kfree(cli->dump_buffers_meta);
602 if (cli->dump_buffers)
603 free_pages(
604 (unsigned long)cli->dump_buffers,
605 get_order(cli->dump_size * cli->buffer_count));
606 kfree(cli->accum_buffer);
607 if (!vinstr_ctx->nclients && vinstr_ctx->kctx) {
608 thread = vinstr_ctx->thread;
609 kbasep_vinstr_destroy_kctx(vinstr_ctx);
610 }
611 kfree(cli);
612
613 mutex_unlock(&vinstr_ctx->lock);
614
615 /* Thread must be stopped after lock is released. */
616 if (thread)
617 kthread_stop(thread);
618
619 return NULL;
620 }
621
kbase_vinstr_detach_client(struct kbase_vinstr_client *cli)622 void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli)
623 {
624 struct kbase_vinstr_context *vinstr_ctx;
625 struct kbase_vinstr_client *iter, *tmp;
626 struct task_struct *thread = NULL;
627 u32 zerobitmap[4] = { 0 };
628 int cli_found = 0;
629
630 KBASE_DEBUG_ASSERT(cli);
631 vinstr_ctx = cli->vinstr_ctx;
632 KBASE_DEBUG_ASSERT(vinstr_ctx);
633
634 mutex_lock(&vinstr_ctx->lock);
635
636 list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) {
637 if (iter == cli) {
638 vinstr_ctx->reprogram = true;
639 cli_found = 1;
640 list_del(&iter->list);
641 break;
642 }
643 }
644 if (!cli_found) {
645 list_for_each_entry_safe(
646 iter, tmp, &vinstr_ctx->waiting_clients, list) {
647 if (iter == cli) {
648 vinstr_ctx->reprogram = true;
649 cli_found = 1;
650 list_del(&iter->list);
651 break;
652 }
653 }
654 }
655 KBASE_DEBUG_ASSERT(cli_found);
656
657 kfree(cli->dump_buffers_meta);
658 free_pages(
659 (unsigned long)cli->dump_buffers,
660 get_order(cli->dump_size * cli->buffer_count));
661 kfree(cli->accum_buffer);
662 kfree(cli);
663
664 vinstr_ctx->nclients--;
665 if (!vinstr_ctx->nclients) {
666 thread = vinstr_ctx->thread;
667 kbasep_vinstr_destroy_kctx(vinstr_ctx);
668 }
669
670 /* Rebuild context bitmap now that the client has detached */
671 hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap);
672 list_for_each_entry(iter, &vinstr_ctx->idle_clients, list)
673 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
674 list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list)
675 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
676
677 mutex_unlock(&vinstr_ctx->lock);
678
679 /* Thread must be stopped after lock is released. */
680 if (thread)
681 kthread_stop(thread);
682 }
683 KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client);
684
685 /* Accumulate counters in the dump buffer */
accum_dump_buffer(void *dst, void *src, size_t dump_size)686 static void accum_dump_buffer(void *dst, void *src, size_t dump_size)
687 {
688 size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
689 u32 *d = dst;
690 u32 *s = src;
691 size_t i, j;
692
693 for (i = 0; i < dump_size; i += block_size) {
694 /* skip over the header block */
695 d += NR_BYTES_PER_HDR / sizeof(u32);
696 s += NR_BYTES_PER_HDR / sizeof(u32);
697 for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) {
698 /* saturate result if addition would result in wraparound */
699 if (U32_MAX - *d < *s)
700 *d = U32_MAX;
701 else
702 *d += *s;
703 d++;
704 s++;
705 }
706 }
707 }
708
709 /* This is the Midgard v4 patch function. It copies the headers for each
710 * of the defined blocks from the master kernel buffer and then patches up
711 * the performance counter enable mask for each of the blocks to exclude
712 * counters that were not requested by the client. */
patch_dump_buffer_hdr_v4( struct kbase_vinstr_context *vinstr_ctx, struct kbase_vinstr_client *cli)713 static void patch_dump_buffer_hdr_v4(
714 struct kbase_vinstr_context *vinstr_ctx,
715 struct kbase_vinstr_client *cli)
716 {
717 u32 *mask;
718 u8 *dst = cli->accum_buffer;
719 u8 *src = vinstr_ctx->cpu_va;
720 u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups;
721 size_t i, group_size, group;
722 enum {
723 SC0_BASE = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
724 SC1_BASE = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
725 SC2_BASE = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
726 SC3_BASE = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
727 TILER_BASE = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
728 MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
729 JM_BASE = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT
730 };
731
732 group_size = NR_CNT_BLOCKS_PER_GROUP *
733 NR_CNT_PER_BLOCK *
734 NR_BYTES_PER_CNT;
735 for (i = 0; i < nr_cg; i++) {
736 group = i * group_size;
737 /* copy shader core headers */
738 memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE],
739 NR_BYTES_PER_HDR);
740 memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE],
741 NR_BYTES_PER_HDR);
742 memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE],
743 NR_BYTES_PER_HDR);
744 memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE],
745 NR_BYTES_PER_HDR);
746
747 /* copy tiler header */
748 memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE],
749 NR_BYTES_PER_HDR);
750
751 /* copy mmu header */
752 memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE],
753 NR_BYTES_PER_HDR);
754
755 /* copy job manager header */
756 memcpy(&dst[group + JM_BASE], &src[group + JM_BASE],
757 NR_BYTES_PER_HDR);
758
759 /* patch the shader core enable mask */
760 mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET];
761 *mask &= cli->bitmap[SHADER_HWCNT_BM];
762 mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET];
763 *mask &= cli->bitmap[SHADER_HWCNT_BM];
764 mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET];
765 *mask &= cli->bitmap[SHADER_HWCNT_BM];
766 mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET];
767 *mask &= cli->bitmap[SHADER_HWCNT_BM];
768
769 /* patch the tiler core enable mask */
770 mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET];
771 *mask &= cli->bitmap[TILER_HWCNT_BM];
772
773 /* patch the mmu core enable mask */
774 mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET];
775 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
776
777 /* patch the job manager enable mask */
778 mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET];
779 *mask &= cli->bitmap[JM_HWCNT_BM];
780 }
781 }
782
783 /* This is the Midgard v5 patch function. It copies the headers for each
784 * of the defined blocks from the master kernel buffer and then patches up
785 * the performance counter enable mask for each of the blocks to exclude
786 * counters that were not requested by the client. */
patch_dump_buffer_hdr_v5( struct kbase_vinstr_context *vinstr_ctx, struct kbase_vinstr_client *cli)787 static void patch_dump_buffer_hdr_v5(
788 struct kbase_vinstr_context *vinstr_ctx,
789 struct kbase_vinstr_client *cli)
790 {
791 struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev;
792 u32 i, nr_l2;
793 u64 core_mask;
794 u32 *mask;
795 u8 *dst = cli->accum_buffer;
796 u8 *src = vinstr_ctx->cpu_va;
797 size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
798
799 /* copy and patch job manager header */
800 memcpy(dst, src, NR_BYTES_PER_HDR);
801 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
802 *mask &= cli->bitmap[JM_HWCNT_BM];
803 dst += block_size;
804 src += block_size;
805
806 /* copy and patch tiler header */
807 memcpy(dst, src, NR_BYTES_PER_HDR);
808 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
809 *mask &= cli->bitmap[TILER_HWCNT_BM];
810 dst += block_size;
811 src += block_size;
812
813 /* copy and patch MMU/L2C headers */
814 nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices;
815 for (i = 0; i < nr_l2; i++) {
816 memcpy(dst, src, NR_BYTES_PER_HDR);
817 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
818 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
819 dst += block_size;
820 src += block_size;
821 }
822
823 /* copy and patch shader core headers */
824 core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask;
825 while (0ull != core_mask) {
826 memcpy(dst, src, NR_BYTES_PER_HDR);
827 if (0ull != (core_mask & 1ull)) {
828 /* if block is not reserved update header */
829 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
830 *mask &= cli->bitmap[SHADER_HWCNT_BM];
831 }
832 dst += block_size;
833 src += block_size;
834
835 core_mask >>= 1;
836 }
837 }
838
839 /**
840 * accum_clients - accumulate dumped hw counters for all known clients
841 * @vinstr_ctx: vinstr context
842 */
accum_clients(struct kbase_vinstr_context *vinstr_ctx)843 static void accum_clients(struct kbase_vinstr_context *vinstr_ctx)
844 {
845 struct kbase_vinstr_client *iter;
846 int v4 = 0;
847
848 #ifndef CONFIG_MALI_NO_MALI
849 v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4);
850 #endif
851
852 list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) {
853 /* Don't bother accumulating clients whose hwcnt requests
854 * have not yet been honoured. */
855 if (iter->pending)
856 continue;
857 if (v4)
858 patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
859 else
860 patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
861 accum_dump_buffer(
862 iter->accum_buffer,
863 vinstr_ctx->cpu_va,
864 iter->dump_size);
865 }
866 list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) {
867 /* Don't bother accumulating clients whose hwcnt requests
868 * have not yet been honoured. */
869 if (iter->pending)
870 continue;
871 if (v4)
872 patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
873 else
874 patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
875 accum_dump_buffer(
876 iter->accum_buffer,
877 vinstr_ctx->cpu_va,
878 iter->dump_size);
879 }
880 }
881
882 /*****************************************************************************/
883
884 /**
885 * kbasep_vinstr_get_timestamp - return timestamp
886 *
887 * Function returns timestamp value based on raw monotonic timer. Value will
888 * wrap around zero in case of overflow.
889 *
890 * Return: timestamp value
891 */
kbasep_vinstr_get_timestamp(void)892 static u64 kbasep_vinstr_get_timestamp(void)
893 {
894 struct timespec64 ts;
895
896 ktime_get_raw_ts64(&ts);
897 return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
898 }
899
900 /**
901 * kbasep_vinstr_add_dump_request - register client's dumping request
902 * @cli: requesting client
903 * @waiting_clients: list of pending dumping requests
904 */
kbasep_vinstr_add_dump_request( struct kbase_vinstr_client *cli, struct list_head *waiting_clients)905 static void kbasep_vinstr_add_dump_request(
906 struct kbase_vinstr_client *cli,
907 struct list_head *waiting_clients)
908 {
909 struct kbase_vinstr_client *tmp;
910
911 if (list_empty(waiting_clients)) {
912 list_add(&cli->list, waiting_clients);
913 return;
914 }
915 list_for_each_entry(tmp, waiting_clients, list) {
916 if (tmp->dump_time > cli->dump_time) {
917 list_add_tail(&cli->list, &tmp->list);
918 return;
919 }
920 }
921 list_add_tail(&cli->list, waiting_clients);
922 }
923
924 /**
925 * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level
926 * dump and accumulate them for known
927 * clients
928 * @vinstr_ctx: vinstr context
929 * @timestamp: pointer where collection timestamp will be recorded
930 *
931 * Return: zero on success
932 */
kbasep_vinstr_collect_and_accumulate( struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp)933 static int kbasep_vinstr_collect_and_accumulate(
934 struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp)
935 {
936 unsigned long flags;
937 int rcode;
938
939 #ifdef CONFIG_MALI_NO_MALI
940 /* The dummy model needs the CPU mapping. */
941 gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va);
942 #endif
943
944 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
945 if (VINSTR_IDLE != vinstr_ctx->state) {
946 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
947 return -EAGAIN;
948 } else {
949 vinstr_ctx->state = VINSTR_DUMPING;
950 }
951 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
952
953 /* Request HW counters dump.
954 * Disable preemption to make dump timestamp more accurate. */
955 preempt_disable();
956 *timestamp = kbasep_vinstr_get_timestamp();
957 rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx);
958 preempt_enable();
959
960 if (!rcode)
961 rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx);
962 WARN_ON(rcode);
963
964 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
965 switch (vinstr_ctx->state)
966 {
967 case VINSTR_SUSPENDING:
968 schedule_work(&vinstr_ctx->suspend_work);
969 break;
970 case VINSTR_DUMPING:
971 vinstr_ctx->state = VINSTR_IDLE;
972 wake_up_all(&vinstr_ctx->suspend_waitq);
973 break;
974 default:
975 break;
976 }
977 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
978
979 /* Accumulate values of collected counters. */
980 if (!rcode)
981 accum_clients(vinstr_ctx);
982
983 return rcode;
984 }
985
986 /**
987 * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel
988 * buffer
989 * @cli: requesting client
990 * @timestamp: timestamp when counters were collected
991 * @event_id: id of event that caused triggered counters collection
992 *
993 * Return: zero on success
994 */
kbasep_vinstr_fill_dump_buffer( struct kbase_vinstr_client *cli, u64 timestamp, enum base_hwcnt_reader_event event_id)995 static int kbasep_vinstr_fill_dump_buffer(
996 struct kbase_vinstr_client *cli, u64 timestamp,
997 enum base_hwcnt_reader_event event_id)
998 {
999 unsigned int write_idx = atomic_read(&cli->write_idx);
1000 unsigned int read_idx = atomic_read(&cli->read_idx);
1001
1002 struct kbase_hwcnt_reader_metadata *meta;
1003 void *buffer;
1004
1005 /* Check if there is a place to copy HWC block into. */
1006 if (write_idx - read_idx == cli->buffer_count)
1007 return -1;
1008 write_idx %= cli->buffer_count;
1009
1010 /* Fill in dump buffer and its metadata. */
1011 buffer = &cli->dump_buffers[write_idx * cli->dump_size];
1012 meta = &cli->dump_buffers_meta[write_idx];
1013 meta->timestamp = timestamp;
1014 meta->event_id = event_id;
1015 meta->buffer_idx = write_idx;
1016 memcpy(buffer, cli->accum_buffer, cli->dump_size);
1017 return 0;
1018 }
1019
1020 /**
1021 * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer
1022 * allocated in userspace
1023 * @cli: requesting client
1024 *
1025 * Return: zero on success
1026 *
1027 * This is part of legacy ioctl interface.
1028 */
kbasep_vinstr_fill_dump_buffer_legacy( struct kbase_vinstr_client *cli)1029 static int kbasep_vinstr_fill_dump_buffer_legacy(
1030 struct kbase_vinstr_client *cli)
1031 {
1032 void __user *buffer = cli->legacy_buffer;
1033 int rcode;
1034
1035 /* Copy data to user buffer. */
1036 rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size);
1037 if (rcode)
1038 pr_warn("error while copying buffer to user\n");
1039 return rcode;
1040 }
1041
1042 /**
1043 * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer
1044 * allocated in kernel space
1045 * @cli: requesting client
1046 *
1047 * Return: zero on success
1048 *
1049 * This is part of the kernel client interface.
1050 */
kbasep_vinstr_fill_dump_buffer_kernel( struct kbase_vinstr_client *cli)1051 static int kbasep_vinstr_fill_dump_buffer_kernel(
1052 struct kbase_vinstr_client *cli)
1053 {
1054 memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size);
1055
1056 return 0;
1057 }
1058
1059 /**
1060 * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst
1061 * @vinstr_ctx: vinstr context
1062 */
kbasep_vinstr_reprogram( struct kbase_vinstr_context *vinstr_ctx)1063 static void kbasep_vinstr_reprogram(
1064 struct kbase_vinstr_context *vinstr_ctx)
1065 {
1066 unsigned long flags;
1067 bool suspended = false;
1068
1069 /* Don't enable hardware counters if vinstr is suspended. */
1070 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1071 if (VINSTR_IDLE != vinstr_ctx->state)
1072 suspended = true;
1073 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1074 if (suspended)
1075 return;
1076
1077 /* Change to suspended state is done while holding vinstr context
1078 * lock. Below code will then no re-enable the instrumentation. */
1079
1080 if (vinstr_ctx->reprogram) {
1081 struct kbase_vinstr_client *iter;
1082
1083 if (!reprogram_hwcnt(vinstr_ctx)) {
1084 vinstr_ctx->reprogram = false;
1085 list_for_each_entry(
1086 iter,
1087 &vinstr_ctx->idle_clients,
1088 list)
1089 iter->pending = false;
1090 list_for_each_entry(
1091 iter,
1092 &vinstr_ctx->waiting_clients,
1093 list)
1094 iter->pending = false;
1095 }
1096 }
1097 }
1098
1099 /**
1100 * kbasep_vinstr_update_client - copy accumulated counters to user readable
1101 * buffer and notify the user
1102 * @cli: requesting client
1103 * @timestamp: timestamp when counters were collected
1104 * @event_id: id of event that caused triggered counters collection
1105 *
1106 * Return: zero on success
1107 */
kbasep_vinstr_update_client( struct kbase_vinstr_client *cli, u64 timestamp, enum base_hwcnt_reader_event event_id)1108 static int kbasep_vinstr_update_client(
1109 struct kbase_vinstr_client *cli, u64 timestamp,
1110 enum base_hwcnt_reader_event event_id)
1111 {
1112 int rcode = 0;
1113
1114 /* Copy collected counters to user readable buffer. */
1115 if (cli->buffer_count)
1116 rcode = kbasep_vinstr_fill_dump_buffer(
1117 cli, timestamp, event_id);
1118 else if (cli->kernel_buffer)
1119 rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli);
1120 else
1121 rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli);
1122
1123 if (rcode)
1124 goto exit;
1125
1126
1127 /* Notify client. Make sure all changes to memory are visible. */
1128 wmb();
1129 atomic_inc(&cli->write_idx);
1130 wake_up_interruptible(&cli->waitq);
1131
1132 /* Prepare for next request. */
1133 memset(cli->accum_buffer, 0, cli->dump_size);
1134
1135 exit:
1136 return rcode;
1137 }
1138
1139 /**
1140 * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function
1141 *
1142 * @hrtimer: high resolution timer
1143 *
1144 * Return: High resolution timer restart enum.
1145 */
kbasep_vinstr_wake_up_callback( struct hrtimer *hrtimer)1146 static enum hrtimer_restart kbasep_vinstr_wake_up_callback(
1147 struct hrtimer *hrtimer)
1148 {
1149 struct kbasep_vinstr_wake_up_timer *timer =
1150 container_of(
1151 hrtimer,
1152 struct kbasep_vinstr_wake_up_timer,
1153 hrtimer);
1154
1155 KBASE_DEBUG_ASSERT(timer);
1156
1157 atomic_set(&timer->vinstr_ctx->request_pending, 1);
1158 wake_up_all(&timer->vinstr_ctx->waitq);
1159
1160 return HRTIMER_NORESTART;
1161 }
1162
1163 #ifdef CONFIG_DEBUG_OBJECT_TIMERS
1164 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0))
1165 /**
1166 * kbase_destroy_hrtimer_on_stack - kernel's destroy_hrtimer_on_stack(),
1167 * rewritten
1168 *
1169 * @timer: high resolution timer
1170 *
1171 * destroy_hrtimer_on_stack() was exported only for 4.7.0 kernel so for
1172 * earlier kernel versions it is not possible to call it explicitly.
1173 * Since this function must accompany hrtimer_init_on_stack(), which
1174 * has to be used for hrtimer initialization if CONFIG_DEBUG_OBJECT_TIMERS
1175 * is defined in order to avoid the warning about object on stack not being
1176 * annotated, we rewrite it here to be used for earlier kernel versions.
1177 */
kbase_destroy_hrtimer_on_stack(struct hrtimer *timer)1178 static void kbase_destroy_hrtimer_on_stack(struct hrtimer *timer)
1179 {
1180 debug_object_free(timer, &hrtimer_debug_descr);
1181 }
1182 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) */
1183 #endif /* CONFIG_DEBUG_OBJECT_TIMERS */
1184
1185 /**
1186 * kbasep_vinstr_service_task - HWC dumping service thread
1187 *
1188 * @data: Pointer to vinstr context structure.
1189 *
1190 * Return: Always returns zero.
1191 */
kbasep_vinstr_service_task(void *data)1192 static int kbasep_vinstr_service_task(void *data)
1193 {
1194 struct kbase_vinstr_context *vinstr_ctx = data;
1195 struct kbasep_vinstr_wake_up_timer timer;
1196
1197 KBASE_DEBUG_ASSERT(vinstr_ctx);
1198
1199 hrtimer_init_on_stack(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1200
1201 timer.hrtimer.function = kbasep_vinstr_wake_up_callback;
1202 timer.vinstr_ctx = vinstr_ctx;
1203
1204 while (!kthread_should_stop()) {
1205 struct kbase_vinstr_client *cli = NULL;
1206 struct kbase_vinstr_client *tmp;
1207 int rcode;
1208
1209 u64 timestamp = kbasep_vinstr_get_timestamp();
1210 u64 dump_time = 0;
1211 struct list_head expired_requests;
1212
1213 /* Hold lock while performing operations on lists of clients. */
1214 mutex_lock(&vinstr_ctx->lock);
1215
1216 /* Closing thread must not interact with client requests. */
1217 if (current == vinstr_ctx->thread) {
1218 atomic_set(&vinstr_ctx->request_pending, 0);
1219
1220 if (!list_empty(&vinstr_ctx->waiting_clients)) {
1221 cli = list_first_entry(
1222 &vinstr_ctx->waiting_clients,
1223 struct kbase_vinstr_client,
1224 list);
1225 dump_time = cli->dump_time;
1226 }
1227 }
1228
1229 if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) {
1230 mutex_unlock(&vinstr_ctx->lock);
1231
1232 /* Sleep until next dumping event or service request. */
1233 if (cli) {
1234 u64 diff = dump_time - timestamp;
1235
1236 hrtimer_start(
1237 &timer.hrtimer,
1238 ns_to_ktime(diff),
1239 HRTIMER_MODE_REL);
1240 }
1241 wait_event(
1242 vinstr_ctx->waitq,
1243 atomic_read(
1244 &vinstr_ctx->request_pending) ||
1245 kthread_should_stop());
1246 hrtimer_cancel(&timer.hrtimer);
1247 continue;
1248 }
1249
1250 rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx,
1251 ×tamp);
1252
1253 INIT_LIST_HEAD(&expired_requests);
1254
1255 /* Find all expired requests. */
1256 list_for_each_entry_safe(
1257 cli,
1258 tmp,
1259 &vinstr_ctx->waiting_clients,
1260 list) {
1261 s64 tdiff =
1262 (s64)(timestamp + DUMPING_RESOLUTION) -
1263 (s64)cli->dump_time;
1264 if (tdiff >= 0ll) {
1265 list_del(&cli->list);
1266 list_add(&cli->list, &expired_requests);
1267 } else {
1268 break;
1269 }
1270 }
1271
1272 /* Fill data for each request found. */
1273 list_for_each_entry_safe(cli, tmp, &expired_requests, list) {
1274 /* Ensure that legacy buffer will not be used from
1275 * this kthread context. */
1276 BUG_ON(0 == cli->buffer_count);
1277 /* Expect only periodically sampled clients. */
1278 BUG_ON(0 == cli->dump_interval);
1279
1280 if (!rcode)
1281 kbasep_vinstr_update_client(
1282 cli,
1283 timestamp,
1284 BASE_HWCNT_READER_EVENT_PERIODIC);
1285
1286 /* Set new dumping time. Drop missed probing times. */
1287 do {
1288 cli->dump_time += cli->dump_interval;
1289 } while (cli->dump_time < timestamp);
1290
1291 list_del(&cli->list);
1292 kbasep_vinstr_add_dump_request(
1293 cli,
1294 &vinstr_ctx->waiting_clients);
1295 }
1296
1297 /* Reprogram counters set if required. */
1298 kbasep_vinstr_reprogram(vinstr_ctx);
1299
1300 mutex_unlock(&vinstr_ctx->lock);
1301 }
1302
1303 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
1304 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0))
1305 kbase_destroy_hrtimer_on_stack(&timer.hrtimer);
1306 #else
1307 destroy_hrtimer_on_stack(&timer.hrtimer);
1308 #endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) */
1309 #endif /* CONFIG_DEBUG_OBJECTS_TIMERS */
1310
1311 return 0;
1312 }
1313
1314 /*****************************************************************************/
1315
1316 /**
1317 * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers
1318 * @cli: pointer to vinstr client structure
1319 *
1320 * Return: non-zero if client has at least one dumping buffer filled that was
1321 * not notified to user yet
1322 */
kbasep_vinstr_hwcnt_reader_buffer_ready( struct kbase_vinstr_client *cli)1323 static int kbasep_vinstr_hwcnt_reader_buffer_ready(
1324 struct kbase_vinstr_client *cli)
1325 {
1326 KBASE_DEBUG_ASSERT(cli);
1327 return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx);
1328 }
1329
1330 /**
1331 * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command
1332 * @cli: pointer to vinstr client structure
1333 * @buffer: pointer to userspace buffer
1334 * @size: size of buffer
1335 *
1336 * Return: zero on success
1337 */
kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( struct kbase_vinstr_client *cli, void __user *buffer, size_t size)1338 static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1339 struct kbase_vinstr_client *cli, void __user *buffer,
1340 size_t size)
1341 {
1342 unsigned int meta_idx = atomic_read(&cli->meta_idx);
1343 unsigned int idx = meta_idx % cli->buffer_count;
1344
1345 struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx];
1346
1347 /* Metadata sanity check. */
1348 KBASE_DEBUG_ASSERT(idx == meta->buffer_idx);
1349
1350 if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1351 return -EINVAL;
1352
1353 /* Check if there is any buffer available. */
1354 if (atomic_read(&cli->write_idx) == meta_idx)
1355 return -EAGAIN;
1356
1357 /* Check if previously taken buffer was put back. */
1358 if (atomic_read(&cli->read_idx) != meta_idx)
1359 return -EBUSY;
1360
1361 /* Copy next available buffer's metadata to user. */
1362 if (copy_to_user(buffer, meta, size))
1363 return -EFAULT;
1364
1365 atomic_inc(&cli->meta_idx);
1366
1367 return 0;
1368 }
1369
1370 /**
1371 * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command
1372 * @cli: pointer to vinstr client structure
1373 * @buffer: pointer to userspace buffer
1374 * @size: size of buffer
1375 *
1376 * Return: zero on success
1377 */
kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( struct kbase_vinstr_client *cli, void __user *buffer, size_t size)1378 static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1379 struct kbase_vinstr_client *cli, void __user *buffer,
1380 size_t size)
1381 {
1382 unsigned int read_idx = atomic_read(&cli->read_idx);
1383 unsigned int idx = read_idx % cli->buffer_count;
1384
1385 struct kbase_hwcnt_reader_metadata meta;
1386
1387 if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1388 return -EINVAL;
1389
1390 /* Check if any buffer was taken. */
1391 if (atomic_read(&cli->meta_idx) == read_idx)
1392 return -EPERM;
1393
1394 /* Check if correct buffer is put back. */
1395 if (copy_from_user(&meta, buffer, size))
1396 return -EFAULT;
1397 if (idx != meta.buffer_idx)
1398 return -EINVAL;
1399
1400 atomic_inc(&cli->read_idx);
1401
1402 return 0;
1403 }
1404
1405 /**
1406 * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command
1407 * @cli: pointer to vinstr client structure
1408 * @interval: periodic dumping interval (disable periodic dumping if zero)
1409 *
1410 * Return: zero on success
1411 */
kbasep_vinstr_hwcnt_reader_ioctl_set_interval( struct kbase_vinstr_client *cli, u32 interval)1412 static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1413 struct kbase_vinstr_client *cli, u32 interval)
1414 {
1415 struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1416
1417 KBASE_DEBUG_ASSERT(vinstr_ctx);
1418
1419 mutex_lock(&vinstr_ctx->lock);
1420
1421 list_del(&cli->list);
1422
1423 cli->dump_interval = interval;
1424
1425 /* If interval is non-zero, enable periodic dumping for this client. */
1426 if (cli->dump_interval) {
1427 if (DUMPING_RESOLUTION > cli->dump_interval)
1428 cli->dump_interval = DUMPING_RESOLUTION;
1429 cli->dump_time =
1430 kbasep_vinstr_get_timestamp() + cli->dump_interval;
1431
1432 kbasep_vinstr_add_dump_request(
1433 cli, &vinstr_ctx->waiting_clients);
1434
1435 atomic_set(&vinstr_ctx->request_pending, 1);
1436 wake_up_all(&vinstr_ctx->waitq);
1437 } else {
1438 list_add(&cli->list, &vinstr_ctx->idle_clients);
1439 }
1440
1441 mutex_unlock(&vinstr_ctx->lock);
1442
1443 return 0;
1444 }
1445
1446 /**
1447 * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id
1448 * @event_id: id of event
1449 * Return: event_mask or zero if event is not supported or maskable
1450 */
kbasep_vinstr_hwcnt_reader_event_mask( enum base_hwcnt_reader_event event_id)1451 static u32 kbasep_vinstr_hwcnt_reader_event_mask(
1452 enum base_hwcnt_reader_event event_id)
1453 {
1454 u32 event_mask = 0;
1455
1456 switch (event_id) {
1457 case BASE_HWCNT_READER_EVENT_PREJOB:
1458 case BASE_HWCNT_READER_EVENT_POSTJOB:
1459 /* These event are maskable. */
1460 event_mask = (1 << event_id);
1461 break;
1462
1463 case BASE_HWCNT_READER_EVENT_MANUAL:
1464 case BASE_HWCNT_READER_EVENT_PERIODIC:
1465 /* These event are non-maskable. */
1466 default:
1467 /* These event are not supported. */
1468 break;
1469 }
1470
1471 return event_mask;
1472 }
1473
1474 /**
1475 * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command
1476 * @cli: pointer to vinstr client structure
1477 * @event_id: id of event to enable
1478 *
1479 * Return: zero on success
1480 */
kbasep_vinstr_hwcnt_reader_ioctl_enable_event( struct kbase_vinstr_client *cli, enum base_hwcnt_reader_event event_id)1481 static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1482 struct kbase_vinstr_client *cli,
1483 enum base_hwcnt_reader_event event_id)
1484 {
1485 struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1486 u32 event_mask;
1487
1488 KBASE_DEBUG_ASSERT(vinstr_ctx);
1489
1490 event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1491 if (!event_mask)
1492 return -EINVAL;
1493
1494 mutex_lock(&vinstr_ctx->lock);
1495 cli->event_mask |= event_mask;
1496 mutex_unlock(&vinstr_ctx->lock);
1497
1498 return 0;
1499 }
1500
1501 /**
1502 * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command
1503 * @cli: pointer to vinstr client structure
1504 * @event_id: id of event to disable
1505 *
1506 * Return: zero on success
1507 */
kbasep_vinstr_hwcnt_reader_ioctl_disable_event( struct kbase_vinstr_client *cli, enum base_hwcnt_reader_event event_id)1508 static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1509 struct kbase_vinstr_client *cli,
1510 enum base_hwcnt_reader_event event_id)
1511 {
1512 struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1513 u32 event_mask;
1514
1515 KBASE_DEBUG_ASSERT(vinstr_ctx);
1516
1517 event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1518 if (!event_mask)
1519 return -EINVAL;
1520
1521 mutex_lock(&vinstr_ctx->lock);
1522 cli->event_mask &= ~event_mask;
1523 mutex_unlock(&vinstr_ctx->lock);
1524
1525 return 0;
1526 }
1527
1528 /**
1529 * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command
1530 * @cli: pointer to vinstr client structure
1531 * @hwver: pointer to user buffer where hw version will be stored
1532 *
1533 * Return: zero on success
1534 */
kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( struct kbase_vinstr_client *cli, u32 __user *hwver)1535 static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1536 struct kbase_vinstr_client *cli, u32 __user *hwver)
1537 {
1538 #ifndef CONFIG_MALI_NO_MALI
1539 struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1540 #endif
1541
1542 u32 ver = 5;
1543
1544 #ifndef CONFIG_MALI_NO_MALI
1545 KBASE_DEBUG_ASSERT(vinstr_ctx);
1546 if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4))
1547 ver = 4;
1548 #endif
1549
1550 return put_user(ver, hwver);
1551 }
1552
1553 /**
1554 * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl
1555 * @filp: pointer to file structure
1556 * @cmd: user command
1557 * @arg: command's argument
1558 *
1559 * Return: zero on success
1560 */
kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)1561 static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp,
1562 unsigned int cmd, unsigned long arg)
1563 {
1564 long rcode = 0;
1565 struct kbase_vinstr_client *cli;
1566
1567 KBASE_DEBUG_ASSERT(filp);
1568
1569 cli = filp->private_data;
1570 KBASE_DEBUG_ASSERT(cli);
1571
1572 if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd)))
1573 return -EINVAL;
1574
1575 switch (cmd) {
1576 case KBASE_HWCNT_READER_GET_API_VERSION:
1577 rcode = put_user(HWCNT_READER_API, (u32 __user *)arg);
1578 break;
1579 case KBASE_HWCNT_READER_GET_HWVER:
1580 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1581 cli, (u32 __user *)arg);
1582 break;
1583 case KBASE_HWCNT_READER_GET_BUFFER_SIZE:
1584 KBASE_DEBUG_ASSERT(cli->vinstr_ctx);
1585 rcode = put_user(
1586 (u32)cli->vinstr_ctx->dump_size,
1587 (u32 __user *)arg);
1588 break;
1589 case KBASE_HWCNT_READER_DUMP:
1590 rcode = kbase_vinstr_hwc_dump(
1591 cli, BASE_HWCNT_READER_EVENT_MANUAL);
1592 break;
1593 case KBASE_HWCNT_READER_CLEAR:
1594 rcode = kbase_vinstr_hwc_clear(cli);
1595 break;
1596 case KBASE_HWCNT_READER_GET_BUFFER:
1597 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1598 cli, (void __user *)arg, _IOC_SIZE(cmd));
1599 break;
1600 case KBASE_HWCNT_READER_PUT_BUFFER:
1601 rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1602 cli, (void __user *)arg, _IOC_SIZE(cmd));
1603 break;
1604 case KBASE_HWCNT_READER_SET_INTERVAL:
1605 rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1606 cli, (u32)arg);
1607 break;
1608 case KBASE_HWCNT_READER_ENABLE_EVENT:
1609 rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1610 cli, (enum base_hwcnt_reader_event)arg);
1611 break;
1612 case KBASE_HWCNT_READER_DISABLE_EVENT:
1613 rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1614 cli, (enum base_hwcnt_reader_event)arg);
1615 break;
1616 default:
1617 rcode = -EINVAL;
1618 break;
1619 }
1620
1621 return rcode;
1622 }
1623
1624 /**
1625 * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll
1626 * @filp: pointer to file structure
1627 * @wait: pointer to poll table
1628 * Return: POLLIN if data can be read without blocking, otherwise zero
1629 */
kbasep_vinstr_hwcnt_reader_poll(struct file *filp, poll_table *wait)1630 static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp,
1631 poll_table *wait)
1632 {
1633 struct kbase_vinstr_client *cli;
1634
1635 KBASE_DEBUG_ASSERT(filp);
1636 KBASE_DEBUG_ASSERT(wait);
1637
1638 cli = filp->private_data;
1639 KBASE_DEBUG_ASSERT(cli);
1640
1641 poll_wait(filp, &cli->waitq, wait);
1642 if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli))
1643 return POLLIN;
1644 return 0;
1645 }
1646
1647 /**
1648 * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap
1649 * @filp: pointer to file structure
1650 * @vma: pointer to vma structure
1651 * Return: zero on success
1652 */
kbasep_vinstr_hwcnt_reader_mmap(struct file *filp, struct vm_area_struct *vma)1653 static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp,
1654 struct vm_area_struct *vma)
1655 {
1656 struct kbase_vinstr_client *cli;
1657 unsigned long size, addr, pfn, offset;
1658 unsigned long vm_size = vma->vm_end - vma->vm_start;
1659
1660 KBASE_DEBUG_ASSERT(filp);
1661 KBASE_DEBUG_ASSERT(vma);
1662
1663 cli = filp->private_data;
1664 KBASE_DEBUG_ASSERT(cli);
1665
1666 size = cli->buffer_count * cli->dump_size;
1667
1668 if (vma->vm_pgoff > (size >> PAGE_SHIFT))
1669 return -EINVAL;
1670
1671 offset = vma->vm_pgoff << PAGE_SHIFT;
1672 if (vm_size > size - offset)
1673 return -EINVAL;
1674
1675 addr = __pa((unsigned long)cli->dump_buffers + offset);
1676 pfn = addr >> PAGE_SHIFT;
1677
1678 return remap_pfn_range(
1679 vma,
1680 vma->vm_start,
1681 pfn,
1682 vm_size,
1683 vma->vm_page_prot);
1684 }
1685
1686 /**
1687 * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release
1688 * @inode: pointer to inode structure
1689 * @filp: pointer to file structure
1690 * Return always return zero
1691 */
kbasep_vinstr_hwcnt_reader_release(struct inode *inode, struct file *filp)1692 static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode,
1693 struct file *filp)
1694 {
1695 struct kbase_vinstr_client *cli;
1696
1697 KBASE_DEBUG_ASSERT(inode);
1698 KBASE_DEBUG_ASSERT(filp);
1699
1700 cli = filp->private_data;
1701 KBASE_DEBUG_ASSERT(cli);
1702
1703 kbase_vinstr_detach_client(cli);
1704 return 0;
1705 }
1706
1707 /*****************************************************************************/
1708
1709 /**
1710 * kbasep_vinstr_kick_scheduler - trigger scheduler cycle
1711 * @kbdev: pointer to kbase device structure
1712 */
kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev)1713 static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev)
1714 {
1715 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1716 unsigned long flags;
1717
1718 down(&js_devdata->schedule_sem);
1719 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1720 kbase_backend_slot_update(kbdev);
1721 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1722 up(&js_devdata->schedule_sem);
1723 }
1724
1725 /**
1726 * kbasep_vinstr_suspend_worker - worker suspending vinstr module
1727 * @data: pointer to work structure
1728 */
kbasep_vinstr_suspend_worker(struct work_struct *data)1729 static void kbasep_vinstr_suspend_worker(struct work_struct *data)
1730 {
1731 struct kbase_vinstr_context *vinstr_ctx;
1732 unsigned long flags;
1733
1734 vinstr_ctx = container_of(data, struct kbase_vinstr_context,
1735 suspend_work);
1736
1737 mutex_lock(&vinstr_ctx->lock);
1738
1739 if (vinstr_ctx->kctx)
1740 disable_hwcnt(vinstr_ctx);
1741
1742 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1743 vinstr_ctx->state = VINSTR_SUSPENDED;
1744 wake_up_all(&vinstr_ctx->suspend_waitq);
1745 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1746
1747 mutex_unlock(&vinstr_ctx->lock);
1748
1749 /* Kick GPU scheduler to allow entering protected mode.
1750 * This must happen after vinstr was suspended. */
1751 kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
1752 }
1753
1754 /**
1755 * kbasep_vinstr_suspend_worker - worker resuming vinstr module
1756 * @data: pointer to work structure
1757 */
kbasep_vinstr_resume_worker(struct work_struct *data)1758 static void kbasep_vinstr_resume_worker(struct work_struct *data)
1759 {
1760 struct kbase_vinstr_context *vinstr_ctx;
1761 unsigned long flags;
1762
1763 vinstr_ctx = container_of(data, struct kbase_vinstr_context,
1764 resume_work);
1765
1766 mutex_lock(&vinstr_ctx->lock);
1767
1768 if (vinstr_ctx->kctx)
1769 enable_hwcnt(vinstr_ctx);
1770
1771 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1772 vinstr_ctx->state = VINSTR_IDLE;
1773 wake_up_all(&vinstr_ctx->suspend_waitq);
1774 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1775
1776 mutex_unlock(&vinstr_ctx->lock);
1777
1778 /* Kick GPU scheduler to allow entering protected mode.
1779 * Note that scheduler state machine might requested re-entry to
1780 * protected mode before vinstr was resumed.
1781 * This must happen after vinstr was release. */
1782 kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
1783 }
1784
1785 /*****************************************************************************/
1786
kbase_vinstr_init(struct kbase_device *kbdev)1787 struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev)
1788 {
1789 struct kbase_vinstr_context *vinstr_ctx;
1790
1791 vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL);
1792 if (!vinstr_ctx)
1793 return NULL;
1794
1795 INIT_LIST_HEAD(&vinstr_ctx->idle_clients);
1796 INIT_LIST_HEAD(&vinstr_ctx->waiting_clients);
1797 mutex_init(&vinstr_ctx->lock);
1798 spin_lock_init(&vinstr_ctx->state_lock);
1799 vinstr_ctx->kbdev = kbdev;
1800 vinstr_ctx->thread = NULL;
1801 vinstr_ctx->state = VINSTR_IDLE;
1802 vinstr_ctx->suspend_cnt = 0;
1803 INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker);
1804 INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker);
1805 init_waitqueue_head(&vinstr_ctx->suspend_waitq);
1806
1807 atomic_set(&vinstr_ctx->request_pending, 0);
1808 init_waitqueue_head(&vinstr_ctx->waitq);
1809
1810 return vinstr_ctx;
1811 }
1812
kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx)1813 void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx)
1814 {
1815 struct kbase_vinstr_client *cli;
1816
1817 /* Stop service thread first. */
1818 if (vinstr_ctx->thread)
1819 kthread_stop(vinstr_ctx->thread);
1820
1821 /* Wait for workers. */
1822 flush_work(&vinstr_ctx->suspend_work);
1823 flush_work(&vinstr_ctx->resume_work);
1824
1825 while (1) {
1826 struct list_head *list = &vinstr_ctx->idle_clients;
1827
1828 if (list_empty(list)) {
1829 list = &vinstr_ctx->waiting_clients;
1830 if (list_empty(list))
1831 break;
1832 }
1833
1834 cli = list_first_entry(list, struct kbase_vinstr_client, list);
1835 list_del(&cli->list);
1836 kfree(cli->accum_buffer);
1837 kfree(cli);
1838 vinstr_ctx->nclients--;
1839 }
1840 KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients);
1841 if (vinstr_ctx->kctx)
1842 kbasep_vinstr_destroy_kctx(vinstr_ctx);
1843 kfree(vinstr_ctx);
1844 }
1845
kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx, struct kbase_uk_hwcnt_reader_setup *setup)1846 int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx,
1847 struct kbase_uk_hwcnt_reader_setup *setup)
1848 {
1849 struct kbase_vinstr_client *cli;
1850 u32 bitmap[4];
1851
1852 KBASE_DEBUG_ASSERT(vinstr_ctx);
1853 KBASE_DEBUG_ASSERT(setup);
1854 KBASE_DEBUG_ASSERT(setup->buffer_count);
1855
1856 bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1857 bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
1858 bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1859 bitmap[JM_HWCNT_BM] = setup->jm_bm;
1860
1861 cli = kbasep_vinstr_attach_client(
1862 vinstr_ctx,
1863 setup->buffer_count,
1864 bitmap,
1865 &setup->fd,
1866 NULL);
1867
1868 if (!cli)
1869 return -ENOMEM;
1870
1871 return 0;
1872 }
1873
kbase_vinstr_legacy_hwc_setup( struct kbase_vinstr_context *vinstr_ctx, struct kbase_vinstr_client **cli, struct kbase_uk_hwcnt_setup *setup)1874 int kbase_vinstr_legacy_hwc_setup(
1875 struct kbase_vinstr_context *vinstr_ctx,
1876 struct kbase_vinstr_client **cli,
1877 struct kbase_uk_hwcnt_setup *setup)
1878 {
1879 KBASE_DEBUG_ASSERT(vinstr_ctx);
1880 KBASE_DEBUG_ASSERT(setup);
1881 KBASE_DEBUG_ASSERT(cli);
1882
1883 if (setup->dump_buffer) {
1884 u32 bitmap[4];
1885
1886 bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1887 bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
1888 bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1889 bitmap[JM_HWCNT_BM] = setup->jm_bm;
1890
1891 if (*cli)
1892 return -EBUSY;
1893
1894 *cli = kbasep_vinstr_attach_client(
1895 vinstr_ctx,
1896 0,
1897 bitmap,
1898 (void *)(long)setup->dump_buffer,
1899 NULL);
1900
1901 if (!(*cli))
1902 return -ENOMEM;
1903 } else {
1904 if (!*cli)
1905 return -EINVAL;
1906
1907 kbase_vinstr_detach_client(*cli);
1908 *cli = NULL;
1909 }
1910
1911 return 0;
1912 }
1913
kbase_vinstr_hwcnt_kernel_setup( struct kbase_vinstr_context *vinstr_ctx, struct kbase_uk_hwcnt_reader_setup *setup, void *kernel_buffer)1914 struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup(
1915 struct kbase_vinstr_context *vinstr_ctx,
1916 struct kbase_uk_hwcnt_reader_setup *setup,
1917 void *kernel_buffer)
1918 {
1919 u32 bitmap[4];
1920
1921 if (!vinstr_ctx || !setup || !kernel_buffer)
1922 return NULL;
1923
1924 bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1925 bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
1926 bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1927 bitmap[JM_HWCNT_BM] = setup->jm_bm;
1928
1929 return kbasep_vinstr_attach_client(
1930 vinstr_ctx,
1931 0,
1932 bitmap,
1933 NULL,
1934 kernel_buffer);
1935 }
1936 KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup);
1937
kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli, enum base_hwcnt_reader_event event_id)1938 int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli,
1939 enum base_hwcnt_reader_event event_id)
1940 {
1941 int rcode = 0;
1942 struct kbase_vinstr_context *vinstr_ctx;
1943 u64 timestamp;
1944 u32 event_mask;
1945
1946 if (!cli)
1947 return -EINVAL;
1948
1949 vinstr_ctx = cli->vinstr_ctx;
1950 KBASE_DEBUG_ASSERT(vinstr_ctx);
1951
1952 KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT);
1953 event_mask = 1 << event_id;
1954
1955 mutex_lock(&vinstr_ctx->lock);
1956
1957 if (event_mask & cli->event_mask) {
1958 rcode = kbasep_vinstr_collect_and_accumulate(
1959 vinstr_ctx,
1960 ×tamp);
1961 if (rcode)
1962 goto exit;
1963
1964 rcode = kbasep_vinstr_update_client(cli, timestamp, event_id);
1965 if (rcode)
1966 goto exit;
1967
1968 kbasep_vinstr_reprogram(vinstr_ctx);
1969 }
1970
1971 exit:
1972 mutex_unlock(&vinstr_ctx->lock);
1973
1974 return rcode;
1975 }
1976 KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump);
1977
kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli)1978 int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli)
1979 {
1980 struct kbase_vinstr_context *vinstr_ctx;
1981 int rcode;
1982 u64 unused;
1983
1984 if (!cli)
1985 return -EINVAL;
1986
1987 vinstr_ctx = cli->vinstr_ctx;
1988 KBASE_DEBUG_ASSERT(vinstr_ctx);
1989
1990 mutex_lock(&vinstr_ctx->lock);
1991
1992 rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused);
1993 if (rcode)
1994 goto exit;
1995 rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx);
1996 if (rcode)
1997 goto exit;
1998 memset(cli->accum_buffer, 0, cli->dump_size);
1999
2000 kbasep_vinstr_reprogram(vinstr_ctx);
2001
2002 exit:
2003 mutex_unlock(&vinstr_ctx->lock);
2004
2005 return rcode;
2006 }
2007
kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx)2008 int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx)
2009 {
2010 unsigned long flags;
2011 int ret = -EAGAIN;
2012
2013 KBASE_DEBUG_ASSERT(vinstr_ctx);
2014
2015 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
2016 switch (vinstr_ctx->state) {
2017 case VINSTR_SUSPENDED:
2018 vinstr_ctx->suspend_cnt++;
2019 /* overflow shall not happen */
2020 BUG_ON(0 == vinstr_ctx->suspend_cnt);
2021 ret = 0;
2022 break;
2023
2024 case VINSTR_IDLE:
2025 vinstr_ctx->state = VINSTR_SUSPENDING;
2026 schedule_work(&vinstr_ctx->suspend_work);
2027 break;
2028
2029 case VINSTR_DUMPING:
2030 vinstr_ctx->state = VINSTR_SUSPENDING;
2031 break;
2032
2033 case VINSTR_SUSPENDING:
2034 /* fall through */
2035 case VINSTR_RESUMING:
2036 break;
2037
2038 default:
2039 BUG();
2040 break;
2041 }
2042 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
2043
2044 return ret;
2045 }
2046
kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx)2047 void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx)
2048 {
2049 wait_event(vinstr_ctx->suspend_waitq,
2050 (0 == kbase_vinstr_try_suspend(vinstr_ctx)));
2051 }
2052
kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx)2053 void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx)
2054 {
2055 unsigned long flags;
2056
2057 KBASE_DEBUG_ASSERT(vinstr_ctx);
2058
2059 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
2060 BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state);
2061 if (VINSTR_SUSPENDED == vinstr_ctx->state) {
2062 BUG_ON(0 == vinstr_ctx->suspend_cnt);
2063 vinstr_ctx->suspend_cnt--;
2064 if (0 == vinstr_ctx->suspend_cnt) {
2065 vinstr_ctx->state = VINSTR_RESUMING;
2066 schedule_work(&vinstr_ctx->resume_work);
2067 }
2068 }
2069 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
2070 }
2071