18c2ecf20Sopenharmony_ci// SPDX-License-Identifier: GPL-2.0
28c2ecf20Sopenharmony_ci/*
38c2ecf20Sopenharmony_ci * Arm Statistical Profiling Extensions (SPE) support
48c2ecf20Sopenharmony_ci * Copyright (c) 2017-2018, Arm Ltd.
58c2ecf20Sopenharmony_ci */
68c2ecf20Sopenharmony_ci
78c2ecf20Sopenharmony_ci#include <byteswap.h>
88c2ecf20Sopenharmony_ci#include <endian.h>
98c2ecf20Sopenharmony_ci#include <errno.h>
108c2ecf20Sopenharmony_ci#include <inttypes.h>
118c2ecf20Sopenharmony_ci#include <linux/bitops.h>
128c2ecf20Sopenharmony_ci#include <linux/kernel.h>
138c2ecf20Sopenharmony_ci#include <linux/log2.h>
148c2ecf20Sopenharmony_ci#include <linux/types.h>
158c2ecf20Sopenharmony_ci#include <linux/zalloc.h>
168c2ecf20Sopenharmony_ci#include <stdlib.h>
178c2ecf20Sopenharmony_ci#include <unistd.h>
188c2ecf20Sopenharmony_ci
198c2ecf20Sopenharmony_ci#include "auxtrace.h"
208c2ecf20Sopenharmony_ci#include "color.h"
218c2ecf20Sopenharmony_ci#include "debug.h"
228c2ecf20Sopenharmony_ci#include "evlist.h"
238c2ecf20Sopenharmony_ci#include "evsel.h"
248c2ecf20Sopenharmony_ci#include "machine.h"
258c2ecf20Sopenharmony_ci#include "session.h"
268c2ecf20Sopenharmony_ci#include "symbol.h"
278c2ecf20Sopenharmony_ci#include "thread.h"
288c2ecf20Sopenharmony_ci#include "thread-stack.h"
298c2ecf20Sopenharmony_ci#include "tool.h"
308c2ecf20Sopenharmony_ci#include "util/synthetic-events.h"
318c2ecf20Sopenharmony_ci
328c2ecf20Sopenharmony_ci#include "arm-spe.h"
338c2ecf20Sopenharmony_ci#include "arm-spe-decoder/arm-spe-decoder.h"
348c2ecf20Sopenharmony_ci#include "arm-spe-decoder/arm-spe-pkt-decoder.h"
358c2ecf20Sopenharmony_ci
368c2ecf20Sopenharmony_ci#define MAX_TIMESTAMP (~0ULL)
378c2ecf20Sopenharmony_ci
388c2ecf20Sopenharmony_cistruct arm_spe {
398c2ecf20Sopenharmony_ci	struct auxtrace			auxtrace;
408c2ecf20Sopenharmony_ci	struct auxtrace_queues		queues;
418c2ecf20Sopenharmony_ci	struct auxtrace_heap		heap;
428c2ecf20Sopenharmony_ci	struct itrace_synth_opts        synth_opts;
438c2ecf20Sopenharmony_ci	u32				auxtrace_type;
448c2ecf20Sopenharmony_ci	struct perf_session		*session;
458c2ecf20Sopenharmony_ci	struct machine			*machine;
468c2ecf20Sopenharmony_ci	u32				pmu_type;
478c2ecf20Sopenharmony_ci
488c2ecf20Sopenharmony_ci	u8				timeless_decoding;
498c2ecf20Sopenharmony_ci	u8				data_queued;
508c2ecf20Sopenharmony_ci
518c2ecf20Sopenharmony_ci	u64				sample_type;
528c2ecf20Sopenharmony_ci	u8				sample_flc;
538c2ecf20Sopenharmony_ci	u8				sample_llc;
548c2ecf20Sopenharmony_ci	u8				sample_tlb;
558c2ecf20Sopenharmony_ci	u8				sample_branch;
568c2ecf20Sopenharmony_ci	u8				sample_remote_access;
578c2ecf20Sopenharmony_ci
588c2ecf20Sopenharmony_ci	u64				l1d_miss_id;
598c2ecf20Sopenharmony_ci	u64				l1d_access_id;
608c2ecf20Sopenharmony_ci	u64				llc_miss_id;
618c2ecf20Sopenharmony_ci	u64				llc_access_id;
628c2ecf20Sopenharmony_ci	u64				tlb_miss_id;
638c2ecf20Sopenharmony_ci	u64				tlb_access_id;
648c2ecf20Sopenharmony_ci	u64				branch_miss_id;
658c2ecf20Sopenharmony_ci	u64				remote_access_id;
668c2ecf20Sopenharmony_ci
678c2ecf20Sopenharmony_ci	u64				kernel_start;
688c2ecf20Sopenharmony_ci
698c2ecf20Sopenharmony_ci	unsigned long			num_events;
708c2ecf20Sopenharmony_ci};
718c2ecf20Sopenharmony_ci
728c2ecf20Sopenharmony_cistruct arm_spe_queue {
738c2ecf20Sopenharmony_ci	struct arm_spe			*spe;
748c2ecf20Sopenharmony_ci	unsigned int			queue_nr;
758c2ecf20Sopenharmony_ci	struct auxtrace_buffer		*buffer;
768c2ecf20Sopenharmony_ci	struct auxtrace_buffer		*old_buffer;
778c2ecf20Sopenharmony_ci	union perf_event		*event_buf;
788c2ecf20Sopenharmony_ci	bool				on_heap;
798c2ecf20Sopenharmony_ci	bool				done;
808c2ecf20Sopenharmony_ci	pid_t				pid;
818c2ecf20Sopenharmony_ci	pid_t				tid;
828c2ecf20Sopenharmony_ci	int				cpu;
838c2ecf20Sopenharmony_ci	struct arm_spe_decoder		*decoder;
848c2ecf20Sopenharmony_ci	u64				time;
858c2ecf20Sopenharmony_ci	u64				timestamp;
868c2ecf20Sopenharmony_ci	struct thread			*thread;
878c2ecf20Sopenharmony_ci};
888c2ecf20Sopenharmony_ci
898c2ecf20Sopenharmony_cistatic void arm_spe_dump(struct arm_spe *spe __maybe_unused,
908c2ecf20Sopenharmony_ci			 unsigned char *buf, size_t len)
918c2ecf20Sopenharmony_ci{
928c2ecf20Sopenharmony_ci	struct arm_spe_pkt packet;
938c2ecf20Sopenharmony_ci	size_t pos = 0;
948c2ecf20Sopenharmony_ci	int ret, pkt_len, i;
958c2ecf20Sopenharmony_ci	char desc[ARM_SPE_PKT_DESC_MAX];
968c2ecf20Sopenharmony_ci	const char *color = PERF_COLOR_BLUE;
978c2ecf20Sopenharmony_ci
988c2ecf20Sopenharmony_ci	color_fprintf(stdout, color,
998c2ecf20Sopenharmony_ci		      ". ... ARM SPE data: size %zu bytes\n",
1008c2ecf20Sopenharmony_ci		      len);
1018c2ecf20Sopenharmony_ci
1028c2ecf20Sopenharmony_ci	while (len) {
1038c2ecf20Sopenharmony_ci		ret = arm_spe_get_packet(buf, len, &packet);
1048c2ecf20Sopenharmony_ci		if (ret > 0)
1058c2ecf20Sopenharmony_ci			pkt_len = ret;
1068c2ecf20Sopenharmony_ci		else
1078c2ecf20Sopenharmony_ci			pkt_len = 1;
1088c2ecf20Sopenharmony_ci		printf(".");
1098c2ecf20Sopenharmony_ci		color_fprintf(stdout, color, "  %08x: ", pos);
1108c2ecf20Sopenharmony_ci		for (i = 0; i < pkt_len; i++)
1118c2ecf20Sopenharmony_ci			color_fprintf(stdout, color, " %02x", buf[i]);
1128c2ecf20Sopenharmony_ci		for (; i < 16; i++)
1138c2ecf20Sopenharmony_ci			color_fprintf(stdout, color, "   ");
1148c2ecf20Sopenharmony_ci		if (ret > 0) {
1158c2ecf20Sopenharmony_ci			ret = arm_spe_pkt_desc(&packet, desc,
1168c2ecf20Sopenharmony_ci					       ARM_SPE_PKT_DESC_MAX);
1178c2ecf20Sopenharmony_ci			if (ret > 0)
1188c2ecf20Sopenharmony_ci				color_fprintf(stdout, color, " %s\n", desc);
1198c2ecf20Sopenharmony_ci		} else {
1208c2ecf20Sopenharmony_ci			color_fprintf(stdout, color, " Bad packet!\n");
1218c2ecf20Sopenharmony_ci		}
1228c2ecf20Sopenharmony_ci		pos += pkt_len;
1238c2ecf20Sopenharmony_ci		buf += pkt_len;
1248c2ecf20Sopenharmony_ci		len -= pkt_len;
1258c2ecf20Sopenharmony_ci	}
1268c2ecf20Sopenharmony_ci}
1278c2ecf20Sopenharmony_ci
1288c2ecf20Sopenharmony_cistatic void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf,
1298c2ecf20Sopenharmony_ci			       size_t len)
1308c2ecf20Sopenharmony_ci{
1318c2ecf20Sopenharmony_ci	printf(".\n");
1328c2ecf20Sopenharmony_ci	arm_spe_dump(spe, buf, len);
1338c2ecf20Sopenharmony_ci}
1348c2ecf20Sopenharmony_ci
1358c2ecf20Sopenharmony_cistatic int arm_spe_get_trace(struct arm_spe_buffer *b, void *data)
1368c2ecf20Sopenharmony_ci{
1378c2ecf20Sopenharmony_ci	struct arm_spe_queue *speq = data;
1388c2ecf20Sopenharmony_ci	struct auxtrace_buffer *buffer = speq->buffer;
1398c2ecf20Sopenharmony_ci	struct auxtrace_buffer *old_buffer = speq->old_buffer;
1408c2ecf20Sopenharmony_ci	struct auxtrace_queue *queue;
1418c2ecf20Sopenharmony_ci
1428c2ecf20Sopenharmony_ci	queue = &speq->spe->queues.queue_array[speq->queue_nr];
1438c2ecf20Sopenharmony_ci
1448c2ecf20Sopenharmony_ci	buffer = auxtrace_buffer__next(queue, buffer);
1458c2ecf20Sopenharmony_ci	/* If no more data, drop the previous auxtrace_buffer and return */
1468c2ecf20Sopenharmony_ci	if (!buffer) {
1478c2ecf20Sopenharmony_ci		if (old_buffer)
1488c2ecf20Sopenharmony_ci			auxtrace_buffer__drop_data(old_buffer);
1498c2ecf20Sopenharmony_ci		b->len = 0;
1508c2ecf20Sopenharmony_ci		return 0;
1518c2ecf20Sopenharmony_ci	}
1528c2ecf20Sopenharmony_ci
1538c2ecf20Sopenharmony_ci	speq->buffer = buffer;
1548c2ecf20Sopenharmony_ci
1558c2ecf20Sopenharmony_ci	/* If the aux_buffer doesn't have data associated, try to load it */
1568c2ecf20Sopenharmony_ci	if (!buffer->data) {
1578c2ecf20Sopenharmony_ci		/* get the file desc associated with the perf data file */
1588c2ecf20Sopenharmony_ci		int fd = perf_data__fd(speq->spe->session->data);
1598c2ecf20Sopenharmony_ci
1608c2ecf20Sopenharmony_ci		buffer->data = auxtrace_buffer__get_data(buffer, fd);
1618c2ecf20Sopenharmony_ci		if (!buffer->data)
1628c2ecf20Sopenharmony_ci			return -ENOMEM;
1638c2ecf20Sopenharmony_ci	}
1648c2ecf20Sopenharmony_ci
1658c2ecf20Sopenharmony_ci	b->len = buffer->size;
1668c2ecf20Sopenharmony_ci	b->buf = buffer->data;
1678c2ecf20Sopenharmony_ci
1688c2ecf20Sopenharmony_ci	if (b->len) {
1698c2ecf20Sopenharmony_ci		if (old_buffer)
1708c2ecf20Sopenharmony_ci			auxtrace_buffer__drop_data(old_buffer);
1718c2ecf20Sopenharmony_ci		speq->old_buffer = buffer;
1728c2ecf20Sopenharmony_ci	} else {
1738c2ecf20Sopenharmony_ci		auxtrace_buffer__drop_data(buffer);
1748c2ecf20Sopenharmony_ci		return arm_spe_get_trace(b, data);
1758c2ecf20Sopenharmony_ci	}
1768c2ecf20Sopenharmony_ci
1778c2ecf20Sopenharmony_ci	return 0;
1788c2ecf20Sopenharmony_ci}
1798c2ecf20Sopenharmony_ci
1808c2ecf20Sopenharmony_cistatic struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe,
1818c2ecf20Sopenharmony_ci		unsigned int queue_nr)
1828c2ecf20Sopenharmony_ci{
1838c2ecf20Sopenharmony_ci	struct arm_spe_params params = { .get_trace = 0, };
1848c2ecf20Sopenharmony_ci	struct arm_spe_queue *speq;
1858c2ecf20Sopenharmony_ci
1868c2ecf20Sopenharmony_ci	speq = zalloc(sizeof(*speq));
1878c2ecf20Sopenharmony_ci	if (!speq)
1888c2ecf20Sopenharmony_ci		return NULL;
1898c2ecf20Sopenharmony_ci
1908c2ecf20Sopenharmony_ci	speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
1918c2ecf20Sopenharmony_ci	if (!speq->event_buf)
1928c2ecf20Sopenharmony_ci		goto out_free;
1938c2ecf20Sopenharmony_ci
1948c2ecf20Sopenharmony_ci	speq->spe = spe;
1958c2ecf20Sopenharmony_ci	speq->queue_nr = queue_nr;
1968c2ecf20Sopenharmony_ci	speq->pid = -1;
1978c2ecf20Sopenharmony_ci	speq->tid = -1;
1988c2ecf20Sopenharmony_ci	speq->cpu = -1;
1998c2ecf20Sopenharmony_ci
2008c2ecf20Sopenharmony_ci	/* params set */
2018c2ecf20Sopenharmony_ci	params.get_trace = arm_spe_get_trace;
2028c2ecf20Sopenharmony_ci	params.data = speq;
2038c2ecf20Sopenharmony_ci
2048c2ecf20Sopenharmony_ci	/* create new decoder */
2058c2ecf20Sopenharmony_ci	speq->decoder = arm_spe_decoder_new(&params);
2068c2ecf20Sopenharmony_ci	if (!speq->decoder)
2078c2ecf20Sopenharmony_ci		goto out_free;
2088c2ecf20Sopenharmony_ci
2098c2ecf20Sopenharmony_ci	return speq;
2108c2ecf20Sopenharmony_ci
2118c2ecf20Sopenharmony_ciout_free:
2128c2ecf20Sopenharmony_ci	zfree(&speq->event_buf);
2138c2ecf20Sopenharmony_ci	free(speq);
2148c2ecf20Sopenharmony_ci
2158c2ecf20Sopenharmony_ci	return NULL;
2168c2ecf20Sopenharmony_ci}
2178c2ecf20Sopenharmony_ci
2188c2ecf20Sopenharmony_cistatic inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip)
2198c2ecf20Sopenharmony_ci{
2208c2ecf20Sopenharmony_ci	return ip >= spe->kernel_start ?
2218c2ecf20Sopenharmony_ci		PERF_RECORD_MISC_KERNEL :
2228c2ecf20Sopenharmony_ci		PERF_RECORD_MISC_USER;
2238c2ecf20Sopenharmony_ci}
2248c2ecf20Sopenharmony_ci
2258c2ecf20Sopenharmony_cistatic void arm_spe_prep_sample(struct arm_spe *spe,
2268c2ecf20Sopenharmony_ci				struct arm_spe_queue *speq,
2278c2ecf20Sopenharmony_ci				union perf_event *event,
2288c2ecf20Sopenharmony_ci				struct perf_sample *sample)
2298c2ecf20Sopenharmony_ci{
2308c2ecf20Sopenharmony_ci	struct arm_spe_record *record = &speq->decoder->record;
2318c2ecf20Sopenharmony_ci
2328c2ecf20Sopenharmony_ci	if (!spe->timeless_decoding)
2338c2ecf20Sopenharmony_ci		sample->time = speq->timestamp;
2348c2ecf20Sopenharmony_ci
2358c2ecf20Sopenharmony_ci	sample->ip = record->from_ip;
2368c2ecf20Sopenharmony_ci	sample->cpumode = arm_spe_cpumode(spe, sample->ip);
2378c2ecf20Sopenharmony_ci	sample->pid = speq->pid;
2388c2ecf20Sopenharmony_ci	sample->tid = speq->tid;
2398c2ecf20Sopenharmony_ci	sample->addr = record->to_ip;
2408c2ecf20Sopenharmony_ci	sample->period = 1;
2418c2ecf20Sopenharmony_ci	sample->cpu = speq->cpu;
2428c2ecf20Sopenharmony_ci
2438c2ecf20Sopenharmony_ci	event->sample.header.type = PERF_RECORD_SAMPLE;
2448c2ecf20Sopenharmony_ci	event->sample.header.misc = sample->cpumode;
2458c2ecf20Sopenharmony_ci	event->sample.header.size = sizeof(struct perf_event_header);
2468c2ecf20Sopenharmony_ci}
2478c2ecf20Sopenharmony_ci
2488c2ecf20Sopenharmony_cistatic int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type)
2498c2ecf20Sopenharmony_ci{
2508c2ecf20Sopenharmony_ci	event->header.size = perf_event__sample_event_size(sample, type, 0);
2518c2ecf20Sopenharmony_ci	return perf_event__synthesize_sample(event, type, 0, sample);
2528c2ecf20Sopenharmony_ci}
2538c2ecf20Sopenharmony_ci
2548c2ecf20Sopenharmony_cistatic inline int
2558c2ecf20Sopenharmony_ciarm_spe_deliver_synth_event(struct arm_spe *spe,
2568c2ecf20Sopenharmony_ci			    struct arm_spe_queue *speq __maybe_unused,
2578c2ecf20Sopenharmony_ci			    union perf_event *event,
2588c2ecf20Sopenharmony_ci			    struct perf_sample *sample)
2598c2ecf20Sopenharmony_ci{
2608c2ecf20Sopenharmony_ci	int ret;
2618c2ecf20Sopenharmony_ci
2628c2ecf20Sopenharmony_ci	if (spe->synth_opts.inject) {
2638c2ecf20Sopenharmony_ci		ret = arm_spe__inject_event(event, sample, spe->sample_type);
2648c2ecf20Sopenharmony_ci		if (ret)
2658c2ecf20Sopenharmony_ci			return ret;
2668c2ecf20Sopenharmony_ci	}
2678c2ecf20Sopenharmony_ci
2688c2ecf20Sopenharmony_ci	ret = perf_session__deliver_synth_event(spe->session, event, sample);
2698c2ecf20Sopenharmony_ci	if (ret)
2708c2ecf20Sopenharmony_ci		pr_err("ARM SPE: failed to deliver event, error %d\n", ret);
2718c2ecf20Sopenharmony_ci
2728c2ecf20Sopenharmony_ci	return ret;
2738c2ecf20Sopenharmony_ci}
2748c2ecf20Sopenharmony_ci
2758c2ecf20Sopenharmony_cistatic int
2768c2ecf20Sopenharmony_ciarm_spe_synth_spe_events_sample(struct arm_spe_queue *speq,
2778c2ecf20Sopenharmony_ci				u64 spe_events_id)
2788c2ecf20Sopenharmony_ci{
2798c2ecf20Sopenharmony_ci	struct arm_spe *spe = speq->spe;
2808c2ecf20Sopenharmony_ci	union perf_event *event = speq->event_buf;
2818c2ecf20Sopenharmony_ci	struct perf_sample sample = { .ip = 0, };
2828c2ecf20Sopenharmony_ci
2838c2ecf20Sopenharmony_ci	arm_spe_prep_sample(spe, speq, event, &sample);
2848c2ecf20Sopenharmony_ci
2858c2ecf20Sopenharmony_ci	sample.id = spe_events_id;
2868c2ecf20Sopenharmony_ci	sample.stream_id = spe_events_id;
2878c2ecf20Sopenharmony_ci
2888c2ecf20Sopenharmony_ci	return arm_spe_deliver_synth_event(spe, speq, event, &sample);
2898c2ecf20Sopenharmony_ci}
2908c2ecf20Sopenharmony_ci
2918c2ecf20Sopenharmony_cistatic int arm_spe_sample(struct arm_spe_queue *speq)
2928c2ecf20Sopenharmony_ci{
2938c2ecf20Sopenharmony_ci	const struct arm_spe_record *record = &speq->decoder->record;
2948c2ecf20Sopenharmony_ci	struct arm_spe *spe = speq->spe;
2958c2ecf20Sopenharmony_ci	int err;
2968c2ecf20Sopenharmony_ci
2978c2ecf20Sopenharmony_ci	if (spe->sample_flc) {
2988c2ecf20Sopenharmony_ci		if (record->type & ARM_SPE_L1D_MISS) {
2998c2ecf20Sopenharmony_ci			err = arm_spe_synth_spe_events_sample(
3008c2ecf20Sopenharmony_ci					speq, spe->l1d_miss_id);
3018c2ecf20Sopenharmony_ci			if (err)
3028c2ecf20Sopenharmony_ci				return err;
3038c2ecf20Sopenharmony_ci		}
3048c2ecf20Sopenharmony_ci
3058c2ecf20Sopenharmony_ci		if (record->type & ARM_SPE_L1D_ACCESS) {
3068c2ecf20Sopenharmony_ci			err = arm_spe_synth_spe_events_sample(
3078c2ecf20Sopenharmony_ci					speq, spe->l1d_access_id);
3088c2ecf20Sopenharmony_ci			if (err)
3098c2ecf20Sopenharmony_ci				return err;
3108c2ecf20Sopenharmony_ci		}
3118c2ecf20Sopenharmony_ci	}
3128c2ecf20Sopenharmony_ci
3138c2ecf20Sopenharmony_ci	if (spe->sample_llc) {
3148c2ecf20Sopenharmony_ci		if (record->type & ARM_SPE_LLC_MISS) {
3158c2ecf20Sopenharmony_ci			err = arm_spe_synth_spe_events_sample(
3168c2ecf20Sopenharmony_ci					speq, spe->llc_miss_id);
3178c2ecf20Sopenharmony_ci			if (err)
3188c2ecf20Sopenharmony_ci				return err;
3198c2ecf20Sopenharmony_ci		}
3208c2ecf20Sopenharmony_ci
3218c2ecf20Sopenharmony_ci		if (record->type & ARM_SPE_LLC_ACCESS) {
3228c2ecf20Sopenharmony_ci			err = arm_spe_synth_spe_events_sample(
3238c2ecf20Sopenharmony_ci					speq, spe->llc_access_id);
3248c2ecf20Sopenharmony_ci			if (err)
3258c2ecf20Sopenharmony_ci				return err;
3268c2ecf20Sopenharmony_ci		}
3278c2ecf20Sopenharmony_ci	}
3288c2ecf20Sopenharmony_ci
3298c2ecf20Sopenharmony_ci	if (spe->sample_tlb) {
3308c2ecf20Sopenharmony_ci		if (record->type & ARM_SPE_TLB_MISS) {
3318c2ecf20Sopenharmony_ci			err = arm_spe_synth_spe_events_sample(
3328c2ecf20Sopenharmony_ci					speq, spe->tlb_miss_id);
3338c2ecf20Sopenharmony_ci			if (err)
3348c2ecf20Sopenharmony_ci				return err;
3358c2ecf20Sopenharmony_ci		}
3368c2ecf20Sopenharmony_ci
3378c2ecf20Sopenharmony_ci		if (record->type & ARM_SPE_TLB_ACCESS) {
3388c2ecf20Sopenharmony_ci			err = arm_spe_synth_spe_events_sample(
3398c2ecf20Sopenharmony_ci					speq, spe->tlb_access_id);
3408c2ecf20Sopenharmony_ci			if (err)
3418c2ecf20Sopenharmony_ci				return err;
3428c2ecf20Sopenharmony_ci		}
3438c2ecf20Sopenharmony_ci	}
3448c2ecf20Sopenharmony_ci
3458c2ecf20Sopenharmony_ci	if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) {
3468c2ecf20Sopenharmony_ci		err = arm_spe_synth_spe_events_sample(speq,
3478c2ecf20Sopenharmony_ci						      spe->branch_miss_id);
3488c2ecf20Sopenharmony_ci		if (err)
3498c2ecf20Sopenharmony_ci			return err;
3508c2ecf20Sopenharmony_ci	}
3518c2ecf20Sopenharmony_ci
3528c2ecf20Sopenharmony_ci	if (spe->sample_remote_access &&
3538c2ecf20Sopenharmony_ci	    (record->type & ARM_SPE_REMOTE_ACCESS)) {
3548c2ecf20Sopenharmony_ci		err = arm_spe_synth_spe_events_sample(speq,
3558c2ecf20Sopenharmony_ci						      spe->remote_access_id);
3568c2ecf20Sopenharmony_ci		if (err)
3578c2ecf20Sopenharmony_ci			return err;
3588c2ecf20Sopenharmony_ci	}
3598c2ecf20Sopenharmony_ci
3608c2ecf20Sopenharmony_ci	return 0;
3618c2ecf20Sopenharmony_ci}
3628c2ecf20Sopenharmony_ci
3638c2ecf20Sopenharmony_cistatic int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp)
3648c2ecf20Sopenharmony_ci{
3658c2ecf20Sopenharmony_ci	struct arm_spe *spe = speq->spe;
3668c2ecf20Sopenharmony_ci	int ret;
3678c2ecf20Sopenharmony_ci
3688c2ecf20Sopenharmony_ci	if (!spe->kernel_start)
3698c2ecf20Sopenharmony_ci		spe->kernel_start = machine__kernel_start(spe->machine);
3708c2ecf20Sopenharmony_ci
3718c2ecf20Sopenharmony_ci	while (1) {
3728c2ecf20Sopenharmony_ci		ret = arm_spe_decode(speq->decoder);
3738c2ecf20Sopenharmony_ci		if (!ret) {
3748c2ecf20Sopenharmony_ci			pr_debug("No data or all data has been processed.\n");
3758c2ecf20Sopenharmony_ci			return 1;
3768c2ecf20Sopenharmony_ci		}
3778c2ecf20Sopenharmony_ci
3788c2ecf20Sopenharmony_ci		/*
3798c2ecf20Sopenharmony_ci		 * Error is detected when decode SPE trace data, continue to
3808c2ecf20Sopenharmony_ci		 * the next trace data and find out more records.
3818c2ecf20Sopenharmony_ci		 */
3828c2ecf20Sopenharmony_ci		if (ret < 0)
3838c2ecf20Sopenharmony_ci			continue;
3848c2ecf20Sopenharmony_ci
3858c2ecf20Sopenharmony_ci		ret = arm_spe_sample(speq);
3868c2ecf20Sopenharmony_ci		if (ret)
3878c2ecf20Sopenharmony_ci			return ret;
3888c2ecf20Sopenharmony_ci
3898c2ecf20Sopenharmony_ci		if (!spe->timeless_decoding && speq->timestamp >= *timestamp) {
3908c2ecf20Sopenharmony_ci			*timestamp = speq->timestamp;
3918c2ecf20Sopenharmony_ci			return 0;
3928c2ecf20Sopenharmony_ci		}
3938c2ecf20Sopenharmony_ci	}
3948c2ecf20Sopenharmony_ci
3958c2ecf20Sopenharmony_ci	return 0;
3968c2ecf20Sopenharmony_ci}
3978c2ecf20Sopenharmony_ci
3988c2ecf20Sopenharmony_cistatic int arm_spe__setup_queue(struct arm_spe *spe,
3998c2ecf20Sopenharmony_ci			       struct auxtrace_queue *queue,
4008c2ecf20Sopenharmony_ci			       unsigned int queue_nr)
4018c2ecf20Sopenharmony_ci{
4028c2ecf20Sopenharmony_ci	struct arm_spe_queue *speq = queue->priv;
4038c2ecf20Sopenharmony_ci	struct arm_spe_record *record;
4048c2ecf20Sopenharmony_ci
4058c2ecf20Sopenharmony_ci	if (list_empty(&queue->head) || speq)
4068c2ecf20Sopenharmony_ci		return 0;
4078c2ecf20Sopenharmony_ci
4088c2ecf20Sopenharmony_ci	speq = arm_spe__alloc_queue(spe, queue_nr);
4098c2ecf20Sopenharmony_ci
4108c2ecf20Sopenharmony_ci	if (!speq)
4118c2ecf20Sopenharmony_ci		return -ENOMEM;
4128c2ecf20Sopenharmony_ci
4138c2ecf20Sopenharmony_ci	queue->priv = speq;
4148c2ecf20Sopenharmony_ci
4158c2ecf20Sopenharmony_ci	if (queue->cpu != -1)
4168c2ecf20Sopenharmony_ci		speq->cpu = queue->cpu;
4178c2ecf20Sopenharmony_ci
4188c2ecf20Sopenharmony_ci	if (!speq->on_heap) {
4198c2ecf20Sopenharmony_ci		int ret;
4208c2ecf20Sopenharmony_ci
4218c2ecf20Sopenharmony_ci		if (spe->timeless_decoding)
4228c2ecf20Sopenharmony_ci			return 0;
4238c2ecf20Sopenharmony_ci
4248c2ecf20Sopenharmony_ciretry:
4258c2ecf20Sopenharmony_ci		ret = arm_spe_decode(speq->decoder);
4268c2ecf20Sopenharmony_ci
4278c2ecf20Sopenharmony_ci		if (!ret)
4288c2ecf20Sopenharmony_ci			return 0;
4298c2ecf20Sopenharmony_ci
4308c2ecf20Sopenharmony_ci		if (ret < 0)
4318c2ecf20Sopenharmony_ci			goto retry;
4328c2ecf20Sopenharmony_ci
4338c2ecf20Sopenharmony_ci		record = &speq->decoder->record;
4348c2ecf20Sopenharmony_ci
4358c2ecf20Sopenharmony_ci		speq->timestamp = record->timestamp;
4368c2ecf20Sopenharmony_ci		ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp);
4378c2ecf20Sopenharmony_ci		if (ret)
4388c2ecf20Sopenharmony_ci			return ret;
4398c2ecf20Sopenharmony_ci		speq->on_heap = true;
4408c2ecf20Sopenharmony_ci	}
4418c2ecf20Sopenharmony_ci
4428c2ecf20Sopenharmony_ci	return 0;
4438c2ecf20Sopenharmony_ci}
4448c2ecf20Sopenharmony_ci
4458c2ecf20Sopenharmony_cistatic int arm_spe__setup_queues(struct arm_spe *spe)
4468c2ecf20Sopenharmony_ci{
4478c2ecf20Sopenharmony_ci	unsigned int i;
4488c2ecf20Sopenharmony_ci	int ret;
4498c2ecf20Sopenharmony_ci
4508c2ecf20Sopenharmony_ci	for (i = 0; i < spe->queues.nr_queues; i++) {
4518c2ecf20Sopenharmony_ci		ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i);
4528c2ecf20Sopenharmony_ci		if (ret)
4538c2ecf20Sopenharmony_ci			return ret;
4548c2ecf20Sopenharmony_ci	}
4558c2ecf20Sopenharmony_ci
4568c2ecf20Sopenharmony_ci	return 0;
4578c2ecf20Sopenharmony_ci}
4588c2ecf20Sopenharmony_ci
4598c2ecf20Sopenharmony_cistatic int arm_spe__update_queues(struct arm_spe *spe)
4608c2ecf20Sopenharmony_ci{
4618c2ecf20Sopenharmony_ci	if (spe->queues.new_data) {
4628c2ecf20Sopenharmony_ci		spe->queues.new_data = false;
4638c2ecf20Sopenharmony_ci		return arm_spe__setup_queues(spe);
4648c2ecf20Sopenharmony_ci	}
4658c2ecf20Sopenharmony_ci
4668c2ecf20Sopenharmony_ci	return 0;
4678c2ecf20Sopenharmony_ci}
4688c2ecf20Sopenharmony_ci
4698c2ecf20Sopenharmony_cistatic bool arm_spe__is_timeless_decoding(struct arm_spe *spe)
4708c2ecf20Sopenharmony_ci{
4718c2ecf20Sopenharmony_ci	struct evsel *evsel;
4728c2ecf20Sopenharmony_ci	struct evlist *evlist = spe->session->evlist;
4738c2ecf20Sopenharmony_ci	bool timeless_decoding = true;
4748c2ecf20Sopenharmony_ci
4758c2ecf20Sopenharmony_ci	/*
4768c2ecf20Sopenharmony_ci	 * Circle through the list of event and complain if we find one
4778c2ecf20Sopenharmony_ci	 * with the time bit set.
4788c2ecf20Sopenharmony_ci	 */
4798c2ecf20Sopenharmony_ci	evlist__for_each_entry(evlist, evsel) {
4808c2ecf20Sopenharmony_ci		if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
4818c2ecf20Sopenharmony_ci			timeless_decoding = false;
4828c2ecf20Sopenharmony_ci	}
4838c2ecf20Sopenharmony_ci
4848c2ecf20Sopenharmony_ci	return timeless_decoding;
4858c2ecf20Sopenharmony_ci}
4868c2ecf20Sopenharmony_ci
4878c2ecf20Sopenharmony_cistatic void arm_spe_set_pid_tid_cpu(struct arm_spe *spe,
4888c2ecf20Sopenharmony_ci				    struct auxtrace_queue *queue)
4898c2ecf20Sopenharmony_ci{
4908c2ecf20Sopenharmony_ci	struct arm_spe_queue *speq = queue->priv;
4918c2ecf20Sopenharmony_ci	pid_t tid;
4928c2ecf20Sopenharmony_ci
4938c2ecf20Sopenharmony_ci	tid = machine__get_current_tid(spe->machine, speq->cpu);
4948c2ecf20Sopenharmony_ci	if (tid != -1) {
4958c2ecf20Sopenharmony_ci		speq->tid = tid;
4968c2ecf20Sopenharmony_ci		thread__zput(speq->thread);
4978c2ecf20Sopenharmony_ci	} else
4988c2ecf20Sopenharmony_ci		speq->tid = queue->tid;
4998c2ecf20Sopenharmony_ci
5008c2ecf20Sopenharmony_ci	if ((!speq->thread) && (speq->tid != -1)) {
5018c2ecf20Sopenharmony_ci		speq->thread = machine__find_thread(spe->machine, -1,
5028c2ecf20Sopenharmony_ci						    speq->tid);
5038c2ecf20Sopenharmony_ci	}
5048c2ecf20Sopenharmony_ci
5058c2ecf20Sopenharmony_ci	if (speq->thread) {
5068c2ecf20Sopenharmony_ci		speq->pid = speq->thread->pid_;
5078c2ecf20Sopenharmony_ci		if (queue->cpu == -1)
5088c2ecf20Sopenharmony_ci			speq->cpu = speq->thread->cpu;
5098c2ecf20Sopenharmony_ci	}
5108c2ecf20Sopenharmony_ci}
5118c2ecf20Sopenharmony_ci
5128c2ecf20Sopenharmony_cistatic int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp)
5138c2ecf20Sopenharmony_ci{
5148c2ecf20Sopenharmony_ci	unsigned int queue_nr;
5158c2ecf20Sopenharmony_ci	u64 ts;
5168c2ecf20Sopenharmony_ci	int ret;
5178c2ecf20Sopenharmony_ci
5188c2ecf20Sopenharmony_ci	while (1) {
5198c2ecf20Sopenharmony_ci		struct auxtrace_queue *queue;
5208c2ecf20Sopenharmony_ci		struct arm_spe_queue *speq;
5218c2ecf20Sopenharmony_ci
5228c2ecf20Sopenharmony_ci		if (!spe->heap.heap_cnt)
5238c2ecf20Sopenharmony_ci			return 0;
5248c2ecf20Sopenharmony_ci
5258c2ecf20Sopenharmony_ci		if (spe->heap.heap_array[0].ordinal >= timestamp)
5268c2ecf20Sopenharmony_ci			return 0;
5278c2ecf20Sopenharmony_ci
5288c2ecf20Sopenharmony_ci		queue_nr = spe->heap.heap_array[0].queue_nr;
5298c2ecf20Sopenharmony_ci		queue = &spe->queues.queue_array[queue_nr];
5308c2ecf20Sopenharmony_ci		speq = queue->priv;
5318c2ecf20Sopenharmony_ci
5328c2ecf20Sopenharmony_ci		auxtrace_heap__pop(&spe->heap);
5338c2ecf20Sopenharmony_ci
5348c2ecf20Sopenharmony_ci		if (spe->heap.heap_cnt) {
5358c2ecf20Sopenharmony_ci			ts = spe->heap.heap_array[0].ordinal + 1;
5368c2ecf20Sopenharmony_ci			if (ts > timestamp)
5378c2ecf20Sopenharmony_ci				ts = timestamp;
5388c2ecf20Sopenharmony_ci		} else {
5398c2ecf20Sopenharmony_ci			ts = timestamp;
5408c2ecf20Sopenharmony_ci		}
5418c2ecf20Sopenharmony_ci
5428c2ecf20Sopenharmony_ci		arm_spe_set_pid_tid_cpu(spe, queue);
5438c2ecf20Sopenharmony_ci
5448c2ecf20Sopenharmony_ci		ret = arm_spe_run_decoder(speq, &ts);
5458c2ecf20Sopenharmony_ci		if (ret < 0) {
5468c2ecf20Sopenharmony_ci			auxtrace_heap__add(&spe->heap, queue_nr, ts);
5478c2ecf20Sopenharmony_ci			return ret;
5488c2ecf20Sopenharmony_ci		}
5498c2ecf20Sopenharmony_ci
5508c2ecf20Sopenharmony_ci		if (!ret) {
5518c2ecf20Sopenharmony_ci			ret = auxtrace_heap__add(&spe->heap, queue_nr, ts);
5528c2ecf20Sopenharmony_ci			if (ret < 0)
5538c2ecf20Sopenharmony_ci				return ret;
5548c2ecf20Sopenharmony_ci		} else {
5558c2ecf20Sopenharmony_ci			speq->on_heap = false;
5568c2ecf20Sopenharmony_ci		}
5578c2ecf20Sopenharmony_ci	}
5588c2ecf20Sopenharmony_ci
5598c2ecf20Sopenharmony_ci	return 0;
5608c2ecf20Sopenharmony_ci}
5618c2ecf20Sopenharmony_ci
5628c2ecf20Sopenharmony_cistatic int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid,
5638c2ecf20Sopenharmony_ci					    u64 time_)
5648c2ecf20Sopenharmony_ci{
5658c2ecf20Sopenharmony_ci	struct auxtrace_queues *queues = &spe->queues;
5668c2ecf20Sopenharmony_ci	unsigned int i;
5678c2ecf20Sopenharmony_ci	u64 ts = 0;
5688c2ecf20Sopenharmony_ci
5698c2ecf20Sopenharmony_ci	for (i = 0; i < queues->nr_queues; i++) {
5708c2ecf20Sopenharmony_ci		struct auxtrace_queue *queue = &spe->queues.queue_array[i];
5718c2ecf20Sopenharmony_ci		struct arm_spe_queue *speq = queue->priv;
5728c2ecf20Sopenharmony_ci
5738c2ecf20Sopenharmony_ci		if (speq && (tid == -1 || speq->tid == tid)) {
5748c2ecf20Sopenharmony_ci			speq->time = time_;
5758c2ecf20Sopenharmony_ci			arm_spe_set_pid_tid_cpu(spe, queue);
5768c2ecf20Sopenharmony_ci			arm_spe_run_decoder(speq, &ts);
5778c2ecf20Sopenharmony_ci		}
5788c2ecf20Sopenharmony_ci	}
5798c2ecf20Sopenharmony_ci	return 0;
5808c2ecf20Sopenharmony_ci}
5818c2ecf20Sopenharmony_ci
5828c2ecf20Sopenharmony_cistatic int arm_spe_process_event(struct perf_session *session,
5838c2ecf20Sopenharmony_ci				 union perf_event *event,
5848c2ecf20Sopenharmony_ci				 struct perf_sample *sample,
5858c2ecf20Sopenharmony_ci				 struct perf_tool *tool)
5868c2ecf20Sopenharmony_ci{
5878c2ecf20Sopenharmony_ci	int err = 0;
5888c2ecf20Sopenharmony_ci	u64 timestamp;
5898c2ecf20Sopenharmony_ci	struct arm_spe *spe = container_of(session->auxtrace,
5908c2ecf20Sopenharmony_ci			struct arm_spe, auxtrace);
5918c2ecf20Sopenharmony_ci
5928c2ecf20Sopenharmony_ci	if (dump_trace)
5938c2ecf20Sopenharmony_ci		return 0;
5948c2ecf20Sopenharmony_ci
5958c2ecf20Sopenharmony_ci	if (!tool->ordered_events) {
5968c2ecf20Sopenharmony_ci		pr_err("SPE trace requires ordered events\n");
5978c2ecf20Sopenharmony_ci		return -EINVAL;
5988c2ecf20Sopenharmony_ci	}
5998c2ecf20Sopenharmony_ci
6008c2ecf20Sopenharmony_ci	if (sample->time && (sample->time != (u64) -1))
6018c2ecf20Sopenharmony_ci		timestamp = sample->time;
6028c2ecf20Sopenharmony_ci	else
6038c2ecf20Sopenharmony_ci		timestamp = 0;
6048c2ecf20Sopenharmony_ci
6058c2ecf20Sopenharmony_ci	if (timestamp || spe->timeless_decoding) {
6068c2ecf20Sopenharmony_ci		err = arm_spe__update_queues(spe);
6078c2ecf20Sopenharmony_ci		if (err)
6088c2ecf20Sopenharmony_ci			return err;
6098c2ecf20Sopenharmony_ci	}
6108c2ecf20Sopenharmony_ci
6118c2ecf20Sopenharmony_ci	if (spe->timeless_decoding) {
6128c2ecf20Sopenharmony_ci		if (event->header.type == PERF_RECORD_EXIT) {
6138c2ecf20Sopenharmony_ci			err = arm_spe_process_timeless_queues(spe,
6148c2ecf20Sopenharmony_ci					event->fork.tid,
6158c2ecf20Sopenharmony_ci					sample->time);
6168c2ecf20Sopenharmony_ci		}
6178c2ecf20Sopenharmony_ci	} else if (timestamp) {
6188c2ecf20Sopenharmony_ci		if (event->header.type == PERF_RECORD_EXIT) {
6198c2ecf20Sopenharmony_ci			err = arm_spe_process_queues(spe, timestamp);
6208c2ecf20Sopenharmony_ci			if (err)
6218c2ecf20Sopenharmony_ci				return err;
6228c2ecf20Sopenharmony_ci		}
6238c2ecf20Sopenharmony_ci	}
6248c2ecf20Sopenharmony_ci
6258c2ecf20Sopenharmony_ci	return err;
6268c2ecf20Sopenharmony_ci}
6278c2ecf20Sopenharmony_ci
6288c2ecf20Sopenharmony_cistatic int arm_spe_process_auxtrace_event(struct perf_session *session,
6298c2ecf20Sopenharmony_ci					  union perf_event *event,
6308c2ecf20Sopenharmony_ci					  struct perf_tool *tool __maybe_unused)
6318c2ecf20Sopenharmony_ci{
6328c2ecf20Sopenharmony_ci	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
6338c2ecf20Sopenharmony_ci					     auxtrace);
6348c2ecf20Sopenharmony_ci
6358c2ecf20Sopenharmony_ci	if (!spe->data_queued) {
6368c2ecf20Sopenharmony_ci		struct auxtrace_buffer *buffer;
6378c2ecf20Sopenharmony_ci		off_t data_offset;
6388c2ecf20Sopenharmony_ci		int fd = perf_data__fd(session->data);
6398c2ecf20Sopenharmony_ci		int err;
6408c2ecf20Sopenharmony_ci
6418c2ecf20Sopenharmony_ci		if (perf_data__is_pipe(session->data)) {
6428c2ecf20Sopenharmony_ci			data_offset = 0;
6438c2ecf20Sopenharmony_ci		} else {
6448c2ecf20Sopenharmony_ci			data_offset = lseek(fd, 0, SEEK_CUR);
6458c2ecf20Sopenharmony_ci			if (data_offset == -1)
6468c2ecf20Sopenharmony_ci				return -errno;
6478c2ecf20Sopenharmony_ci		}
6488c2ecf20Sopenharmony_ci
6498c2ecf20Sopenharmony_ci		err = auxtrace_queues__add_event(&spe->queues, session, event,
6508c2ecf20Sopenharmony_ci				data_offset, &buffer);
6518c2ecf20Sopenharmony_ci		if (err)
6528c2ecf20Sopenharmony_ci			return err;
6538c2ecf20Sopenharmony_ci
6548c2ecf20Sopenharmony_ci		/* Dump here now we have copied a piped trace out of the pipe */
6558c2ecf20Sopenharmony_ci		if (dump_trace) {
6568c2ecf20Sopenharmony_ci			if (auxtrace_buffer__get_data(buffer, fd)) {
6578c2ecf20Sopenharmony_ci				arm_spe_dump_event(spe, buffer->data,
6588c2ecf20Sopenharmony_ci						buffer->size);
6598c2ecf20Sopenharmony_ci				auxtrace_buffer__put_data(buffer);
6608c2ecf20Sopenharmony_ci			}
6618c2ecf20Sopenharmony_ci		}
6628c2ecf20Sopenharmony_ci	}
6638c2ecf20Sopenharmony_ci
6648c2ecf20Sopenharmony_ci	return 0;
6658c2ecf20Sopenharmony_ci}
6668c2ecf20Sopenharmony_ci
6678c2ecf20Sopenharmony_cistatic int arm_spe_flush(struct perf_session *session __maybe_unused,
6688c2ecf20Sopenharmony_ci			 struct perf_tool *tool __maybe_unused)
6698c2ecf20Sopenharmony_ci{
6708c2ecf20Sopenharmony_ci	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
6718c2ecf20Sopenharmony_ci			auxtrace);
6728c2ecf20Sopenharmony_ci	int ret;
6738c2ecf20Sopenharmony_ci
6748c2ecf20Sopenharmony_ci	if (dump_trace)
6758c2ecf20Sopenharmony_ci		return 0;
6768c2ecf20Sopenharmony_ci
6778c2ecf20Sopenharmony_ci	if (!tool->ordered_events)
6788c2ecf20Sopenharmony_ci		return -EINVAL;
6798c2ecf20Sopenharmony_ci
6808c2ecf20Sopenharmony_ci	ret = arm_spe__update_queues(spe);
6818c2ecf20Sopenharmony_ci	if (ret < 0)
6828c2ecf20Sopenharmony_ci		return ret;
6838c2ecf20Sopenharmony_ci
6848c2ecf20Sopenharmony_ci	if (spe->timeless_decoding)
6858c2ecf20Sopenharmony_ci		return arm_spe_process_timeless_queues(spe, -1,
6868c2ecf20Sopenharmony_ci				MAX_TIMESTAMP - 1);
6878c2ecf20Sopenharmony_ci
6888c2ecf20Sopenharmony_ci	return arm_spe_process_queues(spe, MAX_TIMESTAMP);
6898c2ecf20Sopenharmony_ci}
6908c2ecf20Sopenharmony_ci
6918c2ecf20Sopenharmony_cistatic void arm_spe_free_queue(void *priv)
6928c2ecf20Sopenharmony_ci{
6938c2ecf20Sopenharmony_ci	struct arm_spe_queue *speq = priv;
6948c2ecf20Sopenharmony_ci
6958c2ecf20Sopenharmony_ci	if (!speq)
6968c2ecf20Sopenharmony_ci		return;
6978c2ecf20Sopenharmony_ci	thread__zput(speq->thread);
6988c2ecf20Sopenharmony_ci	arm_spe_decoder_free(speq->decoder);
6998c2ecf20Sopenharmony_ci	zfree(&speq->event_buf);
7008c2ecf20Sopenharmony_ci	free(speq);
7018c2ecf20Sopenharmony_ci}
7028c2ecf20Sopenharmony_ci
7038c2ecf20Sopenharmony_cistatic void arm_spe_free_events(struct perf_session *session)
7048c2ecf20Sopenharmony_ci{
7058c2ecf20Sopenharmony_ci	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
7068c2ecf20Sopenharmony_ci					     auxtrace);
7078c2ecf20Sopenharmony_ci	struct auxtrace_queues *queues = &spe->queues;
7088c2ecf20Sopenharmony_ci	unsigned int i;
7098c2ecf20Sopenharmony_ci
7108c2ecf20Sopenharmony_ci	for (i = 0; i < queues->nr_queues; i++) {
7118c2ecf20Sopenharmony_ci		arm_spe_free_queue(queues->queue_array[i].priv);
7128c2ecf20Sopenharmony_ci		queues->queue_array[i].priv = NULL;
7138c2ecf20Sopenharmony_ci	}
7148c2ecf20Sopenharmony_ci	auxtrace_queues__free(queues);
7158c2ecf20Sopenharmony_ci}
7168c2ecf20Sopenharmony_ci
7178c2ecf20Sopenharmony_cistatic void arm_spe_free(struct perf_session *session)
7188c2ecf20Sopenharmony_ci{
7198c2ecf20Sopenharmony_ci	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
7208c2ecf20Sopenharmony_ci					     auxtrace);
7218c2ecf20Sopenharmony_ci
7228c2ecf20Sopenharmony_ci	auxtrace_heap__free(&spe->heap);
7238c2ecf20Sopenharmony_ci	arm_spe_free_events(session);
7248c2ecf20Sopenharmony_ci	session->auxtrace = NULL;
7258c2ecf20Sopenharmony_ci	free(spe);
7268c2ecf20Sopenharmony_ci}
7278c2ecf20Sopenharmony_ci
7288c2ecf20Sopenharmony_cistatic bool arm_spe_evsel_is_auxtrace(struct perf_session *session,
7298c2ecf20Sopenharmony_ci				      struct evsel *evsel)
7308c2ecf20Sopenharmony_ci{
7318c2ecf20Sopenharmony_ci	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace);
7328c2ecf20Sopenharmony_ci
7338c2ecf20Sopenharmony_ci	return evsel->core.attr.type == spe->pmu_type;
7348c2ecf20Sopenharmony_ci}
7358c2ecf20Sopenharmony_ci
7368c2ecf20Sopenharmony_cistatic const char * const arm_spe_info_fmts[] = {
7378c2ecf20Sopenharmony_ci	[ARM_SPE_PMU_TYPE]		= "  PMU Type           %"PRId64"\n",
7388c2ecf20Sopenharmony_ci};
7398c2ecf20Sopenharmony_ci
7408c2ecf20Sopenharmony_cistatic void arm_spe_print_info(__u64 *arr)
7418c2ecf20Sopenharmony_ci{
7428c2ecf20Sopenharmony_ci	if (!dump_trace)
7438c2ecf20Sopenharmony_ci		return;
7448c2ecf20Sopenharmony_ci
7458c2ecf20Sopenharmony_ci	fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]);
7468c2ecf20Sopenharmony_ci}
7478c2ecf20Sopenharmony_ci
7488c2ecf20Sopenharmony_cistruct arm_spe_synth {
7498c2ecf20Sopenharmony_ci	struct perf_tool dummy_tool;
7508c2ecf20Sopenharmony_ci	struct perf_session *session;
7518c2ecf20Sopenharmony_ci};
7528c2ecf20Sopenharmony_ci
7538c2ecf20Sopenharmony_cistatic int arm_spe_event_synth(struct perf_tool *tool,
7548c2ecf20Sopenharmony_ci			       union perf_event *event,
7558c2ecf20Sopenharmony_ci			       struct perf_sample *sample __maybe_unused,
7568c2ecf20Sopenharmony_ci			       struct machine *machine __maybe_unused)
7578c2ecf20Sopenharmony_ci{
7588c2ecf20Sopenharmony_ci	struct arm_spe_synth *arm_spe_synth =
7598c2ecf20Sopenharmony_ci		      container_of(tool, struct arm_spe_synth, dummy_tool);
7608c2ecf20Sopenharmony_ci
7618c2ecf20Sopenharmony_ci	return perf_session__deliver_synth_event(arm_spe_synth->session,
7628c2ecf20Sopenharmony_ci						 event, NULL);
7638c2ecf20Sopenharmony_ci}
7648c2ecf20Sopenharmony_ci
7658c2ecf20Sopenharmony_cistatic int arm_spe_synth_event(struct perf_session *session,
7668c2ecf20Sopenharmony_ci			       struct perf_event_attr *attr, u64 id)
7678c2ecf20Sopenharmony_ci{
7688c2ecf20Sopenharmony_ci	struct arm_spe_synth arm_spe_synth;
7698c2ecf20Sopenharmony_ci
7708c2ecf20Sopenharmony_ci	memset(&arm_spe_synth, 0, sizeof(struct arm_spe_synth));
7718c2ecf20Sopenharmony_ci	arm_spe_synth.session = session;
7728c2ecf20Sopenharmony_ci
7738c2ecf20Sopenharmony_ci	return perf_event__synthesize_attr(&arm_spe_synth.dummy_tool, attr, 1,
7748c2ecf20Sopenharmony_ci					   &id, arm_spe_event_synth);
7758c2ecf20Sopenharmony_ci}
7768c2ecf20Sopenharmony_ci
7778c2ecf20Sopenharmony_cistatic void arm_spe_set_event_name(struct evlist *evlist, u64 id,
7788c2ecf20Sopenharmony_ci				    const char *name)
7798c2ecf20Sopenharmony_ci{
7808c2ecf20Sopenharmony_ci	struct evsel *evsel;
7818c2ecf20Sopenharmony_ci
7828c2ecf20Sopenharmony_ci	evlist__for_each_entry(evlist, evsel) {
7838c2ecf20Sopenharmony_ci		if (evsel->core.id && evsel->core.id[0] == id) {
7848c2ecf20Sopenharmony_ci			if (evsel->name)
7858c2ecf20Sopenharmony_ci				zfree(&evsel->name);
7868c2ecf20Sopenharmony_ci			evsel->name = strdup(name);
7878c2ecf20Sopenharmony_ci			break;
7888c2ecf20Sopenharmony_ci		}
7898c2ecf20Sopenharmony_ci	}
7908c2ecf20Sopenharmony_ci}
7918c2ecf20Sopenharmony_ci
7928c2ecf20Sopenharmony_cistatic int
7938c2ecf20Sopenharmony_ciarm_spe_synth_events(struct arm_spe *spe, struct perf_session *session)
7948c2ecf20Sopenharmony_ci{
7958c2ecf20Sopenharmony_ci	struct evlist *evlist = session->evlist;
7968c2ecf20Sopenharmony_ci	struct evsel *evsel;
7978c2ecf20Sopenharmony_ci	struct perf_event_attr attr;
7988c2ecf20Sopenharmony_ci	bool found = false;
7998c2ecf20Sopenharmony_ci	u64 id;
8008c2ecf20Sopenharmony_ci	int err;
8018c2ecf20Sopenharmony_ci
8028c2ecf20Sopenharmony_ci	evlist__for_each_entry(evlist, evsel) {
8038c2ecf20Sopenharmony_ci		if (evsel->core.attr.type == spe->pmu_type) {
8048c2ecf20Sopenharmony_ci			found = true;
8058c2ecf20Sopenharmony_ci			break;
8068c2ecf20Sopenharmony_ci		}
8078c2ecf20Sopenharmony_ci	}
8088c2ecf20Sopenharmony_ci
8098c2ecf20Sopenharmony_ci	if (!found) {
8108c2ecf20Sopenharmony_ci		pr_debug("No selected events with SPE trace data\n");
8118c2ecf20Sopenharmony_ci		return 0;
8128c2ecf20Sopenharmony_ci	}
8138c2ecf20Sopenharmony_ci
8148c2ecf20Sopenharmony_ci	memset(&attr, 0, sizeof(struct perf_event_attr));
8158c2ecf20Sopenharmony_ci	attr.size = sizeof(struct perf_event_attr);
8168c2ecf20Sopenharmony_ci	attr.type = PERF_TYPE_HARDWARE;
8178c2ecf20Sopenharmony_ci	attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
8188c2ecf20Sopenharmony_ci	attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
8198c2ecf20Sopenharmony_ci		PERF_SAMPLE_PERIOD;
8208c2ecf20Sopenharmony_ci	if (spe->timeless_decoding)
8218c2ecf20Sopenharmony_ci		attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
8228c2ecf20Sopenharmony_ci	else
8238c2ecf20Sopenharmony_ci		attr.sample_type |= PERF_SAMPLE_TIME;
8248c2ecf20Sopenharmony_ci
8258c2ecf20Sopenharmony_ci	spe->sample_type = attr.sample_type;
8268c2ecf20Sopenharmony_ci
8278c2ecf20Sopenharmony_ci	attr.exclude_user = evsel->core.attr.exclude_user;
8288c2ecf20Sopenharmony_ci	attr.exclude_kernel = evsel->core.attr.exclude_kernel;
8298c2ecf20Sopenharmony_ci	attr.exclude_hv = evsel->core.attr.exclude_hv;
8308c2ecf20Sopenharmony_ci	attr.exclude_host = evsel->core.attr.exclude_host;
8318c2ecf20Sopenharmony_ci	attr.exclude_guest = evsel->core.attr.exclude_guest;
8328c2ecf20Sopenharmony_ci	attr.sample_id_all = evsel->core.attr.sample_id_all;
8338c2ecf20Sopenharmony_ci	attr.read_format = evsel->core.attr.read_format;
8348c2ecf20Sopenharmony_ci
8358c2ecf20Sopenharmony_ci	/* create new id val to be a fixed offset from evsel id */
8368c2ecf20Sopenharmony_ci	id = evsel->core.id[0] + 1000000000;
8378c2ecf20Sopenharmony_ci
8388c2ecf20Sopenharmony_ci	if (!id)
8398c2ecf20Sopenharmony_ci		id = 1;
8408c2ecf20Sopenharmony_ci
8418c2ecf20Sopenharmony_ci	if (spe->synth_opts.flc) {
8428c2ecf20Sopenharmony_ci		spe->sample_flc = true;
8438c2ecf20Sopenharmony_ci
8448c2ecf20Sopenharmony_ci		/* Level 1 data cache miss */
8458c2ecf20Sopenharmony_ci		err = arm_spe_synth_event(session, &attr, id);
8468c2ecf20Sopenharmony_ci		if (err)
8478c2ecf20Sopenharmony_ci			return err;
8488c2ecf20Sopenharmony_ci		spe->l1d_miss_id = id;
8498c2ecf20Sopenharmony_ci		arm_spe_set_event_name(evlist, id, "l1d-miss");
8508c2ecf20Sopenharmony_ci		id += 1;
8518c2ecf20Sopenharmony_ci
8528c2ecf20Sopenharmony_ci		/* Level 1 data cache access */
8538c2ecf20Sopenharmony_ci		err = arm_spe_synth_event(session, &attr, id);
8548c2ecf20Sopenharmony_ci		if (err)
8558c2ecf20Sopenharmony_ci			return err;
8568c2ecf20Sopenharmony_ci		spe->l1d_access_id = id;
8578c2ecf20Sopenharmony_ci		arm_spe_set_event_name(evlist, id, "l1d-access");
8588c2ecf20Sopenharmony_ci		id += 1;
8598c2ecf20Sopenharmony_ci	}
8608c2ecf20Sopenharmony_ci
8618c2ecf20Sopenharmony_ci	if (spe->synth_opts.llc) {
8628c2ecf20Sopenharmony_ci		spe->sample_llc = true;
8638c2ecf20Sopenharmony_ci
8648c2ecf20Sopenharmony_ci		/* Last level cache miss */
8658c2ecf20Sopenharmony_ci		err = arm_spe_synth_event(session, &attr, id);
8668c2ecf20Sopenharmony_ci		if (err)
8678c2ecf20Sopenharmony_ci			return err;
8688c2ecf20Sopenharmony_ci		spe->llc_miss_id = id;
8698c2ecf20Sopenharmony_ci		arm_spe_set_event_name(evlist, id, "llc-miss");
8708c2ecf20Sopenharmony_ci		id += 1;
8718c2ecf20Sopenharmony_ci
8728c2ecf20Sopenharmony_ci		/* Last level cache access */
8738c2ecf20Sopenharmony_ci		err = arm_spe_synth_event(session, &attr, id);
8748c2ecf20Sopenharmony_ci		if (err)
8758c2ecf20Sopenharmony_ci			return err;
8768c2ecf20Sopenharmony_ci		spe->llc_access_id = id;
8778c2ecf20Sopenharmony_ci		arm_spe_set_event_name(evlist, id, "llc-access");
8788c2ecf20Sopenharmony_ci		id += 1;
8798c2ecf20Sopenharmony_ci	}
8808c2ecf20Sopenharmony_ci
8818c2ecf20Sopenharmony_ci	if (spe->synth_opts.tlb) {
8828c2ecf20Sopenharmony_ci		spe->sample_tlb = true;
8838c2ecf20Sopenharmony_ci
8848c2ecf20Sopenharmony_ci		/* TLB miss */
8858c2ecf20Sopenharmony_ci		err = arm_spe_synth_event(session, &attr, id);
8868c2ecf20Sopenharmony_ci		if (err)
8878c2ecf20Sopenharmony_ci			return err;
8888c2ecf20Sopenharmony_ci		spe->tlb_miss_id = id;
8898c2ecf20Sopenharmony_ci		arm_spe_set_event_name(evlist, id, "tlb-miss");
8908c2ecf20Sopenharmony_ci		id += 1;
8918c2ecf20Sopenharmony_ci
8928c2ecf20Sopenharmony_ci		/* TLB access */
8938c2ecf20Sopenharmony_ci		err = arm_spe_synth_event(session, &attr, id);
8948c2ecf20Sopenharmony_ci		if (err)
8958c2ecf20Sopenharmony_ci			return err;
8968c2ecf20Sopenharmony_ci		spe->tlb_access_id = id;
8978c2ecf20Sopenharmony_ci		arm_spe_set_event_name(evlist, id, "tlb-access");
8988c2ecf20Sopenharmony_ci		id += 1;
8998c2ecf20Sopenharmony_ci	}
9008c2ecf20Sopenharmony_ci
9018c2ecf20Sopenharmony_ci	if (spe->synth_opts.branches) {
9028c2ecf20Sopenharmony_ci		spe->sample_branch = true;
9038c2ecf20Sopenharmony_ci
9048c2ecf20Sopenharmony_ci		/* Branch miss */
9058c2ecf20Sopenharmony_ci		err = arm_spe_synth_event(session, &attr, id);
9068c2ecf20Sopenharmony_ci		if (err)
9078c2ecf20Sopenharmony_ci			return err;
9088c2ecf20Sopenharmony_ci		spe->branch_miss_id = id;
9098c2ecf20Sopenharmony_ci		arm_spe_set_event_name(evlist, id, "branch-miss");
9108c2ecf20Sopenharmony_ci		id += 1;
9118c2ecf20Sopenharmony_ci	}
9128c2ecf20Sopenharmony_ci
9138c2ecf20Sopenharmony_ci	if (spe->synth_opts.remote_access) {
9148c2ecf20Sopenharmony_ci		spe->sample_remote_access = true;
9158c2ecf20Sopenharmony_ci
9168c2ecf20Sopenharmony_ci		/* Remote access */
9178c2ecf20Sopenharmony_ci		err = arm_spe_synth_event(session, &attr, id);
9188c2ecf20Sopenharmony_ci		if (err)
9198c2ecf20Sopenharmony_ci			return err;
9208c2ecf20Sopenharmony_ci		spe->remote_access_id = id;
9218c2ecf20Sopenharmony_ci		arm_spe_set_event_name(evlist, id, "remote-access");
9228c2ecf20Sopenharmony_ci		id += 1;
9238c2ecf20Sopenharmony_ci	}
9248c2ecf20Sopenharmony_ci
9258c2ecf20Sopenharmony_ci	return 0;
9268c2ecf20Sopenharmony_ci}
9278c2ecf20Sopenharmony_ci
9288c2ecf20Sopenharmony_ciint arm_spe_process_auxtrace_info(union perf_event *event,
9298c2ecf20Sopenharmony_ci				  struct perf_session *session)
9308c2ecf20Sopenharmony_ci{
9318c2ecf20Sopenharmony_ci	struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
9328c2ecf20Sopenharmony_ci	size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX;
9338c2ecf20Sopenharmony_ci	struct arm_spe *spe;
9348c2ecf20Sopenharmony_ci	int err;
9358c2ecf20Sopenharmony_ci
9368c2ecf20Sopenharmony_ci	if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +
9378c2ecf20Sopenharmony_ci					min_sz)
9388c2ecf20Sopenharmony_ci		return -EINVAL;
9398c2ecf20Sopenharmony_ci
9408c2ecf20Sopenharmony_ci	spe = zalloc(sizeof(struct arm_spe));
9418c2ecf20Sopenharmony_ci	if (!spe)
9428c2ecf20Sopenharmony_ci		return -ENOMEM;
9438c2ecf20Sopenharmony_ci
9448c2ecf20Sopenharmony_ci	err = auxtrace_queues__init(&spe->queues);
9458c2ecf20Sopenharmony_ci	if (err)
9468c2ecf20Sopenharmony_ci		goto err_free;
9478c2ecf20Sopenharmony_ci
9488c2ecf20Sopenharmony_ci	spe->session = session;
9498c2ecf20Sopenharmony_ci	spe->machine = &session->machines.host; /* No kvm support */
9508c2ecf20Sopenharmony_ci	spe->auxtrace_type = auxtrace_info->type;
9518c2ecf20Sopenharmony_ci	spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE];
9528c2ecf20Sopenharmony_ci
9538c2ecf20Sopenharmony_ci	spe->timeless_decoding = arm_spe__is_timeless_decoding(spe);
9548c2ecf20Sopenharmony_ci	spe->auxtrace.process_event = arm_spe_process_event;
9558c2ecf20Sopenharmony_ci	spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event;
9568c2ecf20Sopenharmony_ci	spe->auxtrace.flush_events = arm_spe_flush;
9578c2ecf20Sopenharmony_ci	spe->auxtrace.free_events = arm_spe_free_events;
9588c2ecf20Sopenharmony_ci	spe->auxtrace.free = arm_spe_free;
9598c2ecf20Sopenharmony_ci	spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace;
9608c2ecf20Sopenharmony_ci	session->auxtrace = &spe->auxtrace;
9618c2ecf20Sopenharmony_ci
9628c2ecf20Sopenharmony_ci	arm_spe_print_info(&auxtrace_info->priv[0]);
9638c2ecf20Sopenharmony_ci
9648c2ecf20Sopenharmony_ci	if (dump_trace)
9658c2ecf20Sopenharmony_ci		return 0;
9668c2ecf20Sopenharmony_ci
9678c2ecf20Sopenharmony_ci	if (session->itrace_synth_opts && session->itrace_synth_opts->set)
9688c2ecf20Sopenharmony_ci		spe->synth_opts = *session->itrace_synth_opts;
9698c2ecf20Sopenharmony_ci	else
9708c2ecf20Sopenharmony_ci		itrace_synth_opts__set_default(&spe->synth_opts, false);
9718c2ecf20Sopenharmony_ci
9728c2ecf20Sopenharmony_ci	err = arm_spe_synth_events(spe, session);
9738c2ecf20Sopenharmony_ci	if (err)
9748c2ecf20Sopenharmony_ci		goto err_free_queues;
9758c2ecf20Sopenharmony_ci
9768c2ecf20Sopenharmony_ci	err = auxtrace_queues__process_index(&spe->queues, session);
9778c2ecf20Sopenharmony_ci	if (err)
9788c2ecf20Sopenharmony_ci		goto err_free_queues;
9798c2ecf20Sopenharmony_ci
9808c2ecf20Sopenharmony_ci	if (spe->queues.populated)
9818c2ecf20Sopenharmony_ci		spe->data_queued = true;
9828c2ecf20Sopenharmony_ci
9838c2ecf20Sopenharmony_ci	return 0;
9848c2ecf20Sopenharmony_ci
9858c2ecf20Sopenharmony_cierr_free_queues:
9868c2ecf20Sopenharmony_ci	auxtrace_queues__free(&spe->queues);
9878c2ecf20Sopenharmony_ci	session->auxtrace = NULL;
9888c2ecf20Sopenharmony_cierr_free:
9898c2ecf20Sopenharmony_ci	free(spe);
9908c2ecf20Sopenharmony_ci	return err;
9918c2ecf20Sopenharmony_ci}
992