Lines Matching refs:perf
144 struct perf_ctx *perf;
169 struct perf_ctx *perf;
205 int (*cmd_recv)(struct perf_ctx *perf, int *pidx, enum perf_cmd *cmd,
247 static void perf_terminate_test(struct perf_ctx *perf);
253 link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);
260 struct perf_ctx *perf = peer->perf;
264 dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
277 sts = ntb_peer_spad_read(perf->ntb, peer->pidx,
278 PERF_SPAD_CMD(perf->gidx));
284 ntb_peer_spad_write(perf->ntb, peer->pidx,
285 PERF_SPAD_LDATA(perf->gidx),
287 ntb_peer_spad_write(perf->ntb, peer->pidx,
288 PERF_SPAD_HDATA(perf->gidx),
290 ntb_peer_spad_write(perf->ntb, peer->pidx,
291 PERF_SPAD_CMD(perf->gidx),
293 ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));
295 dev_dbg(&perf->ntb->dev, "DB ring peer %#llx\n",
304 static int perf_spad_cmd_recv(struct perf_ctx *perf, int *pidx,
310 ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
318 for (*pidx = 0; *pidx < perf->pcnt; (*pidx)++) {
319 peer = &perf->peers[*pidx];
324 val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));
330 val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));
333 val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));
337 ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),
340 dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
351 struct perf_ctx *perf = peer->perf;
355 dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
364 outbits = ntb_msg_outbits(perf->ntb);
369 ret = ntb_msg_clear_sts(perf->ntb, outbits);
373 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,
376 if (ntb_msg_read_sts(perf->ntb) & outbits) {
381 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,
385 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);
393 static int perf_msg_cmd_recv(struct perf_ctx *perf, int *pidx,
399 inbits = ntb_msg_inbits(perf->ntb);
401 if (hweight64(ntb_msg_read_sts(perf->ntb) & inbits) < 3)
404 val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_CMD);
407 val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_LDATA);
410 val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_HDATA);
414 ntb_msg_clear_sts(perf->ntb, inbits);
416 dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
423 struct perf_ctx *perf = peer->perf;
426 return perf->cmd_send(peer, cmd, data);
428 dev_err(&perf->ntb->dev, "Send invalid command\n");
442 dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");
449 dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);
456 static int perf_cmd_recv(struct perf_ctx *perf)
462 while (!(ret = perf->cmd_recv(perf, &pidx, &cmd, &data))) {
463 peer = &perf->peers[pidx];
473 dev_err(&perf->ntb->dev, "Recv invalid command\n");
484 struct perf_ctx *perf = ctx;
489 for (pidx = 0; pidx < perf->pcnt; pidx++) {
490 peer = &perf->peers[pidx];
506 struct perf_ctx *perf = ctx;
508 dev_dbg(&perf->ntb->dev, "DB vec %d mask %#llx bits %#llx\n", vec,
509 ntb_db_vector_mask(perf->ntb, vec), ntb_db_read(perf->ntb));
512 (void)perf_cmd_recv(perf);
517 struct perf_ctx *perf = ctx;
519 dev_dbg(&perf->ntb->dev, "Msg status bits %#llx\n",
520 ntb_msg_read_sts(perf->ntb));
523 (void)perf_cmd_recv(perf);
534 (void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
539 struct perf_ctx *perf = peer->perf;
543 ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
546 dev_err(&perf->ntb->dev, "Failed to set outbuf translation\n");
562 (void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
563 dma_free_coherent(&peer->perf->ntb->pdev->dev, peer->inbuf_size,
571 struct perf_ctx *perf = peer->perf;
575 ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,
578 dev_err(&perf->ntb->dev, "Couldn't get inbuf restrictions\n");
583 dev_err(&perf->ntb->dev, "Too big inbuf size %pa > %pa\n",
592 peer->inbuf = dma_alloc_coherent(&perf->ntb->pdev->dev,
596 dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",
602 dev_err(&perf->ntb->dev, "Unaligned inbuf allocated\n");
606 ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
609 dev_err(&perf->ntb->dev, "Failed to set inbuf translation\n");
647 if (test_bit(0, &peer->perf->busy_flag) &&
648 peer == peer->perf->test_peer) {
649 dev_warn(&peer->perf->ntb->dev,
651 perf_terminate_test(peer->perf);
658 static int perf_init_service(struct perf_ctx *perf)
662 if (ntb_peer_mw_count(perf->ntb) < perf->pcnt) {
663 dev_err(&perf->ntb->dev, "Not enough memory windows\n");
667 if (ntb_msg_count(perf->ntb) >= PERF_MSG_CNT) {
668 perf->cmd_send = perf_msg_cmd_send;
669 perf->cmd_recv = perf_msg_cmd_recv;
671 dev_dbg(&perf->ntb->dev, "Message service initialized\n");
676 dev_dbg(&perf->ntb->dev, "Message service unsupported\n");
678 mask = GENMASK_ULL(perf->pcnt, 0);
679 if (ntb_spad_count(perf->ntb) >= PERF_SPAD_CNT(perf->pcnt) &&
680 (ntb_db_valid_mask(perf->ntb) & mask) == mask) {
681 perf->cmd_send = perf_spad_cmd_send;
682 perf->cmd_recv = perf_spad_cmd_recv;
684 dev_dbg(&perf->ntb->dev, "Scratchpad service initialized\n");
689 dev_dbg(&perf->ntb->dev, "Scratchpad service unsupported\n");
691 dev_err(&perf->ntb->dev, "Command services unsupported\n");
696 static int perf_enable_service(struct perf_ctx *perf)
701 mask = ntb_db_valid_mask(perf->ntb);
702 (void)ntb_db_set_mask(perf->ntb, mask);
704 ret = ntb_set_ctx(perf->ntb, perf, &perf_ops);
708 if (perf->cmd_send == perf_msg_cmd_send) {
711 inbits = ntb_msg_inbits(perf->ntb);
712 outbits = ntb_msg_outbits(perf->ntb);
713 (void)ntb_msg_set_mask(perf->ntb, inbits | outbits);
716 ret = ntb_msg_clear_mask(perf->ntb, incmd_bit);
718 dev_dbg(&perf->ntb->dev, "MSG sts unmasked %#llx\n", incmd_bit);
720 scnt = ntb_spad_count(perf->ntb);
722 ntb_spad_write(perf->ntb, sidx, PERF_CMD_INVAL);
723 incmd_bit = PERF_SPAD_NOTIFY(perf->gidx);
724 ret = ntb_db_clear_mask(perf->ntb, incmd_bit);
726 dev_dbg(&perf->ntb->dev, "DB bits unmasked %#llx\n", incmd_bit);
729 ntb_clear_ctx(perf->ntb);
733 ntb_link_enable(perf->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
735 ntb_link_event(perf->ntb);
740 static void perf_disable_service(struct perf_ctx *perf)
744 if (perf->cmd_send == perf_msg_cmd_send) {
747 inbits = ntb_msg_inbits(perf->ntb);
748 (void)ntb_msg_set_mask(perf->ntb, inbits);
750 (void)ntb_db_set_mask(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
753 ntb_clear_ctx(perf->ntb);
755 for (pidx = 0; pidx < perf->pcnt; pidx++)
756 perf_cmd_exec(&perf->peers[pidx], PERF_CMD_CLEAR);
758 for (pidx = 0; pidx < perf->pcnt; pidx++)
759 flush_work(&perf->peers[pidx].service);
761 for (pidx = 0; pidx < perf->pcnt; pidx++) {
762 struct perf_peer *peer = &perf->peers[pidx];
764 ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx), 0);
767 ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
769 ntb_link_disable(perf->ntb);
792 struct perf_peer *peer = pthr->perf->test_peer;
853 return likely(atomic_read(&pthr->perf->tsync) > 0) ? 0 : -EINTR;
863 struct perf_ctx *perf = data;
866 node = dev_to_node(&perf->ntb->dev);
873 struct perf_ctx *perf = pthr->perf;
875 struct perf_peer *peer = pthr->perf->test_peer;
877 pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
878 dev_to_node(&perf->ntb->dev));
882 get_random_bytes(pthr->src, perf->test_peer->outbuf_size);
889 pthr->dma_chan = dma_request_channel(dma_mask, perf_dma_filter, perf);
891 dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
916 atomic_dec(&perf->tsync);
917 wake_up(&perf->twait);
924 struct perf_peer *peer = pthr->perf->test_peer;
925 struct perf_ctx *perf = pthr->perf;
945 dev_err(&perf->ntb->dev, "%d: Got error %d on test\n",
968 struct perf_ctx *perf = pthr->perf;
975 atomic_read(&perf->tsync) < 0));
977 if (atomic_read(&perf->tsync) < 0)
983 dev_dbg(&perf->ntb->dev, "%d: copied %llu bytes\n",
986 dev_dbg(&perf->ntb->dev, "%d: lasted %llu usecs\n",
989 dev_dbg(&perf->ntb->dev, "%d: %llu MBytes/s\n", pthr->tidx,
997 struct perf_ctx *perf = pthr->perf;
1007 if (pthr->perf->test_peer->dma_dst_addr)
1009 pthr->perf->test_peer->dma_dst_addr,
1010 pthr->perf->test_peer->outbuf_size,
1016 atomic_dec(&perf->tsync);
1017 wake_up(&perf->twait);
1051 static int perf_set_tcnt(struct perf_ctx *perf, u8 tcnt)
1056 if (test_and_set_bit_lock(0, &perf->busy_flag))
1059 perf->tcnt = tcnt;
1061 clear_bit_unlock(0, &perf->busy_flag);
1066 static void perf_terminate_test(struct perf_ctx *perf)
1070 atomic_set(&perf->tsync, -1);
1071 wake_up(&perf->twait);
1074 wake_up(&perf->threads[tidx].dma_wait);
1075 cancel_work_sync(&perf->threads[tidx].work);
1081 struct perf_ctx *perf = peer->perf;
1089 if (test_and_set_bit_lock(0, &perf->busy_flag))
1092 perf->test_peer = peer;
1093 atomic_set(&perf->tsync, perf->tcnt);
1096 pthr = &perf->threads[tidx];
1101 if (tidx < perf->tcnt)
1105 ret = wait_event_interruptible(perf->twait,
1106 atomic_read(&perf->tsync) <= 0);
1108 perf_terminate_test(perf);
1112 clear_bit_unlock(0, &perf->busy_flag);
1117 static int perf_read_stats(struct perf_ctx *perf, char *buf,
1123 if (test_and_set_bit_lock(0, &perf->busy_flag))
1127 " Peer %d test statistics:\n", perf->test_peer->pidx);
1130 pthr = &perf->threads[tidx];
1147 clear_bit_unlock(0, &perf->busy_flag);
1152 static void perf_init_threads(struct perf_ctx *perf)
1157 perf->tcnt = DEF_THREADS_CNT;
1158 perf->test_peer = &perf->peers[0];
1159 init_waitqueue_head(&perf->twait);
1162 pthr = &perf->threads[tidx];
1164 pthr->perf = perf;
1172 static void perf_clear_threads(struct perf_ctx *perf)
1174 perf_terminate_test(perf);
1185 struct perf_ctx *perf = filep->private_data;
1202 "Local port %d, Global index %d\n", ntb_port_number(perf->ntb),
1203 perf->gidx);
1205 if (test_bit(0, &perf->busy_flag)) {
1208 ntb_peer_port_number(perf->ntb, perf->test_peer->pidx),
1209 perf->test_peer->pidx);
1214 for (pidx = 0; pidx < perf->pcnt; pidx++) {
1215 peer = &perf->peers[pidx];
1219 ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,
1268 struct perf_ctx *perf = filep->private_data;
1276 ret = perf_read_stats(perf, buf, PERF_BUF_LEN, &pos);
1290 struct perf_ctx *perf = filep->private_data;
1298 if (pidx < 0 || pidx >= perf->pcnt)
1301 peer = &perf->peers[pidx];
1319 struct perf_ctx *perf = filep->private_data;
1323 pos = scnprintf(buf, sizeof(buf), "%hhu\n", perf->tcnt);
1332 struct perf_ctx *perf = filep->private_data;
1340 ret = perf_set_tcnt(perf, val);
1353 static void perf_setup_dbgfs(struct perf_ctx *perf)
1355 struct pci_dev *pdev = perf->ntb->pdev;
1357 perf->dbgfs_dir = debugfs_create_dir(pci_name(pdev), perf_dbgfs_topdir);
1358 if (IS_ERR(perf->dbgfs_dir)) {
1359 dev_warn(&perf->ntb->dev, "DebugFS unsupported\n");
1363 debugfs_create_file("info", 0600, perf->dbgfs_dir, perf,
1366 debugfs_create_file("run", 0600, perf->dbgfs_dir, perf,
1369 debugfs_create_file("threads_count", 0600, perf->dbgfs_dir, perf,
1373 debugfs_create_u8("chunk_order", 0500, perf->dbgfs_dir, &chunk_order);
1375 debugfs_create_u8("total_order", 0500, perf->dbgfs_dir, &total_order);
1377 debugfs_create_bool("use_dma", 0500, perf->dbgfs_dir, &use_dma);
1380 static void perf_clear_dbgfs(struct perf_ctx *perf)
1382 debugfs_remove_recursive(perf->dbgfs_dir);
1392 struct perf_ctx *perf;
1394 perf = devm_kzalloc(&ntb->dev, sizeof(*perf), GFP_KERNEL);
1395 if (!perf)
1398 perf->pcnt = ntb_peer_port_count(ntb);
1399 perf->peers = devm_kcalloc(&ntb->dev, perf->pcnt, sizeof(*perf->peers),
1401 if (!perf->peers)
1404 perf->ntb = ntb;
1406 return perf;
1411 struct perf_ctx *perf = peer->perf;
1416 ret = ntb_peer_mw_get_addr(perf->ntb, perf->gidx, &phys_addr,
1421 peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,
1430 dev_warn(&peer->perf->ntb->dev,
1438 static int perf_init_peers(struct perf_ctx *perf)
1443 lport = ntb_port_number(perf->ntb);
1444 perf->gidx = -1;
1445 for (pidx = 0; pidx < perf->pcnt; pidx++) {
1446 peer = &perf->peers[pidx];
1448 peer->perf = perf;
1450 if (lport < ntb_peer_port_number(perf->ntb, pidx)) {
1451 if (perf->gidx == -1)
1452 perf->gidx = pidx;
1460 if (perf->gidx == -1)
1461 perf->gidx = pidx;
1467 if (perf->pcnt == 1 && ntb_port_number(perf->ntb) == 0 &&
1468 ntb_peer_port_number(perf->ntb, 0) == 0) {
1469 perf->gidx = 0;
1470 perf->peers[0].gidx = 0;
1473 for (pidx = 0; pidx < perf->pcnt; pidx++) {
1474 ret = perf_setup_peer_mw(&perf->peers[pidx]);
1479 dev_dbg(&perf->ntb->dev, "Global port index %d\n", perf->gidx);
1486 struct perf_ctx *perf;
1489 perf = perf_create_data(ntb);
1490 if (IS_ERR(perf))
1491 return PTR_ERR(perf);
1493 ret = perf_init_peers(perf);
1497 perf_init_threads(perf);
1499 ret = perf_init_service(perf);
1503 ret = perf_enable_service(perf);
1507 perf_setup_dbgfs(perf);
1514 struct perf_ctx *perf = ntb->ctx;
1516 perf_clear_dbgfs(perf);
1518 perf_disable_service(perf);
1520 perf_clear_threads(perf);