Lines Matching refs:rcd
88 struct hfi1_ctxtdata *rcd;
94 ret = hfi1_create_ctxtdata(ppd, dd->node, &rcd);
105 rcd->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
111 if (rcd->ctxt == HFI1_CTRL_CTXT)
112 rcd->flags |= HFI1_CAP_DMA_RTAIL;
113 rcd->fast_handler = get_dma_rtail_setting(rcd) ?
117 hfi1_set_seq_cnt(rcd, 1);
119 rcd->sc = sc_alloc(dd, SC_ACK, rcd->rcvhdrqentsize, dd->node);
120 if (!rcd->sc) {
124 hfi1_init_ctxt(rcd->sc);
137 dd->rcd = kcalloc_node(dd->num_rcv_contexts, sizeof(*dd->rcd),
139 if (!dd->rcd)
150 for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i)
151 hfi1_free_ctxt(dd->rcd[i]);
154 kfree(dd->rcd);
155 dd->rcd = NULL;
160 * Helper routines for the receive context reference count (rcd and uctxt).
162 static void hfi1_rcd_init(struct hfi1_ctxtdata *rcd)
164 kref_init(&rcd->kref);
169 * @kref: pointer to an initialized rcd data structure
175 struct hfi1_ctxtdata *rcd =
178 spin_lock_irqsave(&rcd->dd->uctxt_lock, flags);
179 rcd->dd->rcd[rcd->ctxt] = NULL;
180 spin_unlock_irqrestore(&rcd->dd->uctxt_lock, flags);
182 hfi1_free_ctxtdata(rcd->dd, rcd);
184 kfree(rcd);
188 * hfi1_rcd_put - decrement reference for rcd
189 * @rcd: pointer to an initialized rcd data structure
193 int hfi1_rcd_put(struct hfi1_ctxtdata *rcd)
195 if (rcd)
196 return kref_put(&rcd->kref, hfi1_rcd_free);
202 * hfi1_rcd_get - increment reference for rcd
203 * @rcd: pointer to an initialized rcd data structure
210 int hfi1_rcd_get(struct hfi1_ctxtdata *rcd)
212 return kref_get_unless_zero(&rcd->kref);
216 * allocate_rcd_index - allocate an rcd index from the rcd array
218 * @rcd: rcd data structure to assign
221 * Find an empty index in the rcd array, and assign the given rcd to it.
226 struct hfi1_ctxtdata *rcd, u16 *index)
233 if (!dd->rcd[ctxt])
237 rcd->ctxt = ctxt;
238 dd->rcd[ctxt] = rcd;
239 hfi1_rcd_init(rcd);
255 * @ctxt: the index of an possilbe rcd
275 * @ctxt: the index of an possilbe rcd
277 * We need to protect access to the rcd array. If access is needed to
287 struct hfi1_ctxtdata *rcd = NULL;
290 if (dd->rcd[ctxt]) {
291 rcd = dd->rcd[ctxt];
292 if (!hfi1_rcd_get(rcd))
293 rcd = NULL;
297 return rcd;
308 struct hfi1_ctxtdata *rcd;
316 rcd = kzalloc_node(sizeof(*rcd), GFP_KERNEL, numa);
317 if (rcd) {
322 ret = allocate_rcd_index(dd, rcd, &ctxt);
325 kfree(rcd);
329 INIT_LIST_HEAD(&rcd->qp_wait_list);
330 hfi1_exp_tid_group_init(rcd);
331 rcd->ppd = ppd;
332 rcd->dd = dd;
333 rcd->numa_id = numa;
334 rcd->rcv_array_groups = dd->rcv_entries.ngroups;
335 rcd->rhf_rcv_function_map = normal_rhf_rcv_functions;
336 rcd->slow_handler = handle_receive_interrupt;
337 rcd->do_interrupt = rcd->slow_handler;
338 rcd->msix_intr = CCE_NUM_MSIX_VECTORS;
340 mutex_init(&rcd->exp_mutex);
341 spin_lock_init(&rcd->exp_lock);
342 INIT_LIST_HEAD(&rcd->flow_queue.queue_head);
343 INIT_LIST_HEAD(&rcd->rarr_queue.queue_head);
345 hfi1_cdbg(PROC, "setting up context %u", rcd->ctxt);
357 rcd->rcv_array_groups++;
369 rcd->rcv_array_groups++;
375 rcd->eager_base = base * dd->rcv_entries.group_size;
377 rcd->rcvhdrq_cnt = rcvhdrcnt;
378 rcd->rcvhdrqentsize = hfi1_hdrq_entsize;
379 rcd->rhf_offset =
380 rcd->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
392 max_entries = rcd->rcv_array_groups *
395 rcd->egrbufs.count = round_down(rcvtids,
397 if (rcd->egrbufs.count > MAX_EAGER_ENTRIES) {
399 rcd->ctxt);
400 rcd->egrbufs.count = MAX_EAGER_ENTRIES;
404 rcd->ctxt, rcd->egrbufs.count);
414 rcd->egrbufs.buffers =
415 kcalloc_node(rcd->egrbufs.count,
416 sizeof(*rcd->egrbufs.buffers),
418 if (!rcd->egrbufs.buffers)
420 rcd->egrbufs.rcvtids =
421 kcalloc_node(rcd->egrbufs.count,
422 sizeof(*rcd->egrbufs.rcvtids),
424 if (!rcd->egrbufs.rcvtids)
426 rcd->egrbufs.size = eager_buffer_size;
432 if (rcd->egrbufs.size < hfi1_max_mtu) {
433 rcd->egrbufs.size = __roundup_pow_of_two(hfi1_max_mtu);
436 rcd->ctxt, rcd->egrbufs.size);
438 rcd->egrbufs.rcvtid_size = HFI1_MAX_EAGER_BUFFER_SIZE;
442 rcd->opstats = kzalloc_node(sizeof(*rcd->opstats),
444 if (!rcd->opstats)
448 hfi1_kern_init_ctxt_generations(rcd);
451 *context = rcd;
457 hfi1_free_ctxt(rcd);
463 * @rcd: pointer to an initialized rcd data structure
471 void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd)
473 hfi1_rcd_put(rcd);
680 struct hfi1_ctxtdata *rcd;
687 rcd = hfi1_rcd_get_by_index(dd, i);
690 HFI1_RCVCTRL_TAILUPD_DIS, rcd);
691 hfi1_rcd_put(rcd);
702 struct hfi1_ctxtdata *rcd;
714 rcd = hfi1_rcd_get_by_index(dd, i);
715 if (!rcd)
718 rcvmask |= HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
720 if (!HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR))
722 if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_RHQ_FULL))
724 if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_EGR_FULL))
728 hfi1_rcvctrl(dd, rcvmask, rcd);
729 sc_enable(rcd->sc);
730 hfi1_rcd_put(rcd);
848 struct hfi1_ctxtdata *rcd;
878 /* dd->rcd can be NULL if early initialization failed */
879 for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i) {
886 rcd = hfi1_rcd_get_by_index(dd, i);
887 if (!rcd)
890 lastfail = hfi1_create_rcvhdrq(dd, rcd);
892 lastfail = hfi1_setup_eagerbufs(rcd);
894 lastfail = hfi1_kern_exp_rcv_init(rcd, reinit);
901 hfi1_rcd_put(rcd);
1007 struct hfi1_ctxtdata *rcd;
1032 rcd = hfi1_rcd_get_by_index(dd, i);
1037 HFI1_RCVCTRL_ONE_PKT_EGR_DIS, rcd);
1038 hfi1_rcd_put(rcd);
1081 * @rcd: the ctxtdata structure
1086 void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
1090 if (!rcd)
1093 if (rcd->rcvhdrq) {
1094 dma_free_coherent(&dd->pcidev->dev, rcvhdrq_size(rcd),
1095 rcd->rcvhdrq, rcd->rcvhdrq_dma);
1096 rcd->rcvhdrq = NULL;
1097 if (hfi1_rcvhdrtail_kvaddr(rcd)) {
1099 (void *)hfi1_rcvhdrtail_kvaddr(rcd),
1100 rcd->rcvhdrqtailaddr_dma);
1101 rcd->rcvhdrtail_kvaddr = NULL;
1106 kfree(rcd->egrbufs.rcvtids);
1107 rcd->egrbufs.rcvtids = NULL;
1109 for (e = 0; e < rcd->egrbufs.alloced; e++) {
1110 if (rcd->egrbufs.buffers[e].addr)
1112 rcd->egrbufs.buffers[e].len,
1113 rcd->egrbufs.buffers[e].addr,
1114 rcd->egrbufs.buffers[e].dma);
1116 kfree(rcd->egrbufs.buffers);
1117 rcd->egrbufs.alloced = 0;
1118 rcd->egrbufs.buffers = NULL;
1120 sc_free(rcd->sc);
1121 rcd->sc = NULL;
1123 vfree(rcd->subctxt_uregbase);
1124 vfree(rcd->subctxt_rcvegrbuf);
1125 vfree(rcd->subctxt_rcvhdr_base);
1126 kfree(rcd->opstats);
1128 rcd->subctxt_uregbase = NULL;
1129 rcd->subctxt_rcvegrbuf = NULL;
1130 rcd->subctxt_rcvhdr_base = NULL;
1131 rcd->opstats = NULL;
1513 for (ctxt = 0; dd->rcd && ctxt < dd->num_rcv_contexts; ctxt++) {
1514 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
1516 if (rcd) {
1517 hfi1_free_ctxt_rcv_groups(rcd);
1518 hfi1_free_ctxt(rcd);
1522 kfree(dd->rcd);
1523 dd->rcd = NULL;
1753 * @rcd: the context data
1759 int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
1763 if (!rcd->rcvhdrq) {
1764 amt = rcvhdrq_size(rcd);
1766 rcd->rcvhdrq = dma_alloc_coherent(&dd->pcidev->dev, amt,
1767 &rcd->rcvhdrq_dma,
1770 if (!rcd->rcvhdrq) {
1773 amt, rcd->ctxt);
1777 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ||
1778 HFI1_CAP_UGET_MASK(rcd->flags, DMA_RTAIL)) {
1779 rcd->rcvhdrtail_kvaddr = dma_alloc_coherent(&dd->pcidev->dev,
1781 &rcd->rcvhdrqtailaddr_dma,
1783 if (!rcd->rcvhdrtail_kvaddr)
1788 set_hdrq_regs(rcd->dd, rcd->ctxt, rcd->rcvhdrqentsize,
1789 rcd->rcvhdrq_cnt);
1796 rcd->ctxt);
1797 dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,
1798 rcd->rcvhdrq_dma);
1799 rcd->rcvhdrq = NULL;
1807 * @rcd: the context we are setting up.
1814 int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)
1816 struct hfi1_devdata *dd = rcd->dd;
1829 if (rcd->egrbufs.size < (round_mtu * dd->rcv_entries.group_size))
1830 rcd->egrbufs.size = round_mtu * dd->rcv_entries.group_size;
1835 if (!HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR))
1836 rcd->egrbufs.rcvtid_size = round_mtu;
1842 if (rcd->egrbufs.size <= (1 << 20))
1843 rcd->egrbufs.rcvtid_size = max((unsigned long)round_mtu,
1844 rounddown_pow_of_two(rcd->egrbufs.size / 8));
1846 while (alloced_bytes < rcd->egrbufs.size &&
1847 rcd->egrbufs.alloced < rcd->egrbufs.count) {
1848 rcd->egrbufs.buffers[idx].addr =
1850 rcd->egrbufs.rcvtid_size,
1851 &rcd->egrbufs.buffers[idx].dma,
1853 if (rcd->egrbufs.buffers[idx].addr) {
1854 rcd->egrbufs.buffers[idx].len =
1855 rcd->egrbufs.rcvtid_size;
1856 rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].addr =
1857 rcd->egrbufs.buffers[idx].addr;
1858 rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].dma =
1859 rcd->egrbufs.buffers[idx].dma;
1860 rcd->egrbufs.alloced++;
1861 alloced_bytes += rcd->egrbufs.rcvtid_size;
1873 if (rcd->egrbufs.rcvtid_size == round_mtu ||
1874 !HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR)) {
1876 rcd->ctxt);
1881 new_size = rcd->egrbufs.rcvtid_size / 2;
1889 rcd->egrbufs.rcvtid_size = new_size;
1897 rcd->egrbufs.alloced = 0;
1899 if (i >= rcd->egrbufs.count)
1901 rcd->egrbufs.rcvtids[i].dma =
1902 rcd->egrbufs.buffers[j].dma + offset;
1903 rcd->egrbufs.rcvtids[i].addr =
1904 rcd->egrbufs.buffers[j].addr + offset;
1905 rcd->egrbufs.alloced++;
1906 if ((rcd->egrbufs.buffers[j].dma + offset +
1908 (rcd->egrbufs.buffers[j].dma +
1909 rcd->egrbufs.buffers[j].len)) {
1916 rcd->egrbufs.rcvtid_size = new_size;
1919 rcd->egrbufs.numbufs = idx;
1920 rcd->egrbufs.size = alloced_bytes;
1924 rcd->ctxt, rcd->egrbufs.alloced,
1925 rcd->egrbufs.rcvtid_size / 1024, rcd->egrbufs.size / 1024);
1932 rcd->egrbufs.threshold =
1933 rounddown_pow_of_two(rcd->egrbufs.alloced / 2);
1939 max_entries = rcd->rcv_array_groups * dd->rcv_entries.group_size;
1940 egrtop = roundup(rcd->egrbufs.alloced, dd->rcv_entries.group_size);
1941 rcd->expected_count = max_entries - egrtop;
1942 if (rcd->expected_count > MAX_TID_PAIR_ENTRIES * 2)
1943 rcd->expected_count = MAX_TID_PAIR_ENTRIES * 2;
1945 rcd->expected_base = rcd->eager_base + egrtop;
1947 rcd->ctxt, rcd->egrbufs.alloced, rcd->expected_count,
1948 rcd->eager_base, rcd->expected_base);
1950 if (!hfi1_rcvbuf_validate(rcd->egrbufs.rcvtid_size, PT_EAGER, &order)) {
1953 rcd->ctxt, rcd->egrbufs.rcvtid_size);
1958 for (idx = 0; idx < rcd->egrbufs.alloced; idx++) {
1959 hfi1_put_tid(dd, rcd->eager_base + idx, PT_EAGER,
1960 rcd->egrbufs.rcvtids[idx].dma, order);
1967 for (idx = 0; idx < rcd->egrbufs.alloced &&
1968 rcd->egrbufs.buffers[idx].addr;
1971 rcd->egrbufs.buffers[idx].len,
1972 rcd->egrbufs.buffers[idx].addr,
1973 rcd->egrbufs.buffers[idx].dma);
1974 rcd->egrbufs.buffers[idx].addr = NULL;
1975 rcd->egrbufs.buffers[idx].dma = 0;
1976 rcd->egrbufs.buffers[idx].len = 0;