Lines Matching +full:cmd +full:- +full:db
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
100 * reuse of buffers (avoiding the need to re-fault in pages, hold
104 * is a trade-off for performance.
106 * If an application ping-pongs two buffers for a connection via
118 if (ps->prsv.prsv_nppods > 0)
119 t4_free_page_pods(&ps->prsv);
121 for (i = 0; i < ps->npages; i++) {
122 p = ps->pages[i];
141 if (ps->vm)
142 vmspace_free(ps->vm);
154 if (!(toep->ddp.flags & DDP_DEAD)) {
155 KASSERT(toep->ddp.cached_count + toep->ddp.active_count <
156 nitems(toep->ddp.db), ("too many wired pagesets"));
157 TAILQ_INSERT_HEAD(&toep->ddp.cached_pagesets, ps, link);
158 toep->ddp.cached_count++;
160 free_pageset(toep->td, ps);
173 copied = job->aio_received;
177 aio_complete(job, -1, error);
183 t4_free_page_pods(&drb->prsv);
184 free(drb->buf, M_CXGBE);
186 counter_u64_add(toep->ofld_rxq->ddp_buffer_free, 1);
194 if (!(toep->ddp.flags & DDP_DEAD) &&
195 toep->ddp.cached_count < t4_ddp_rcvbuf_cache) {
196 TAILQ_INSERT_HEAD(&toep->ddp.cached_buffers, drb, link);
197 toep->ddp.cached_count++;
211 if (!TAILQ_EMPTY(&toep->ddp.cached_buffers)) {
212 drb = TAILQ_FIRST(&toep->ddp.cached_buffers);
213 TAILQ_REMOVE(&toep->ddp.cached_buffers, drb, link);
214 toep->ddp.cached_count--;
215 counter_u64_add(toep->ofld_rxq->ddp_buffer_reuse, 1);
225 struct tom_data *td = toep->td;
234 drb->buf = contigmalloc(t4_ddp_rcvbuf_len, M_CXGBE, how, 0, ~0,
236 if (drb->buf == NULL) {
240 drb->len = t4_ddp_rcvbuf_len;
241 drb->refs = 1;
243 error = t4_alloc_page_pods_for_rcvbuf(&td->pr, drb);
245 free(drb->buf, M_CXGBE);
250 error = t4_write_page_pods_for_rcvbuf(sc, toep->ctrlq, toep->tid, drb);
252 t4_free_page_pods(&drb->prsv);
253 free(drb->buf, M_CXGBE);
259 counter_u64_add(toep->ofld_rxq->ddp_buffer_alloc, 1);
264 free_ddp_buffer(struct toepcb *toep, struct ddp_buffer *db)
266 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
267 if (db->drb != NULL)
268 free_ddp_rcv_buffer(toep, db->drb);
270 db->drb = NULL;
275 if (db->job) {
277 * XXX: If we are un-offloading the socket then we
282 if (!aio_clear_cancel_function(db->job))
283 ddp_complete_one(db->job, 0);
285 db->job = NULL;
289 if (db->ps) {
290 free_pageset(toep->td, db->ps);
292 db->ps = NULL;
301 toep->ddp.flags = DDP_OK;
302 toep->ddp.active_id = -1;
303 mtx_init(&toep->ddp.lock, "t4 ddp", NULL, MTX_DEF);
304 mtx_init(&toep->ddp.cache_lock, "t4 ddp cache", NULL, MTX_DEF);
311 mtx_destroy(&toep->ddp.lock);
312 mtx_destroy(&toep->ddp.cache_lock);
324 toep->ddp.flags |= DDP_DEAD;
326 for (i = 0; i < nitems(toep->ddp.db); i++) {
327 free_ddp_buffer(toep, &toep->ddp.db[i]);
329 if ((toep->ddp.flags & DDP_AIO) != 0) {
330 while ((ps = TAILQ_FIRST(&toep->ddp.cached_pagesets)) != NULL) {
331 TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link);
332 free_pageset(toep->td, ps);
336 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
338 while ((drb = TAILQ_FIRST(&toep->ddp.cached_buffers)) != NULL) {
339 TAILQ_REMOVE(&toep->ddp.cached_buffers, drb, link);
353 MPASS((toep->ddp.flags & (DDP_TASK_ACTIVE | DDP_DEAD)) != DDP_TASK_ACTIVE);
354 for (i = 0; i < nitems(toep->ddp.db); i++) {
355 if ((toep->ddp.flags & DDP_AIO) != 0) {
356 MPASS(toep->ddp.db[i].job == NULL);
357 MPASS(toep->ddp.db[i].ps == NULL);
359 MPASS(toep->ddp.db[i].drb == NULL);
361 if ((toep->ddp.flags & DDP_AIO) != 0) {
362 MPASS(TAILQ_EMPTY(&toep->ddp.cached_pagesets));
363 MPASS(TAILQ_EMPTY(&toep->ddp.aiojobq));
365 if ((toep->ddp.flags & DDP_RCVBUF) != 0)
366 MPASS(TAILQ_EMPTY(&toep->ddp.cached_buffers));
371 complete_ddp_buffer(struct toepcb *toep, struct ddp_buffer *db,
377 toep->ddp.active_count--;
378 if (toep->ddp.active_id == db_idx) {
379 if (toep->ddp.active_count == 0) {
380 if ((toep->ddp.flags & DDP_AIO) != 0)
381 KASSERT(toep->ddp.db[db_idx ^ 1].job == NULL,
384 KASSERT(toep->ddp.db[db_idx ^ 1].drb == NULL,
386 toep->ddp.active_id = -1;
388 toep->ddp.active_id ^= 1;
391 toep->tid, toep->ddp.active_id);
394 KASSERT(toep->ddp.active_count != 0 &&
395 toep->ddp.active_id != -1,
399 if ((toep->ddp.flags & DDP_AIO) != 0) {
400 db->cancel_pending = 0;
401 db->job = NULL;
402 recycle_pageset(toep, db->ps);
403 db->ps = NULL;
405 drb = db->drb;
406 if (atomic_fetchadd_int(&drb->refs, -1) == 1)
408 db->drb = NULL;
409 db->placed = 0;
413 KASSERT(toep->ddp.flags & db_flag,
415 __func__, toep, toep->ddp.flags));
416 toep->ddp.flags &= ~db_flag;
423 struct toepcb *toep = m->m_ext.ext_arg1;
424 struct ddp_rcv_buffer *drb = m->m_ext.ext_arg2;
432 struct inpcb *inp = toep->inp;
434 struct ddp_buffer *db;
443 m->m_pkthdr.rcvif = toep->vi->ifp;
445 db = &toep->ddp.db[db_idx];
446 drb = db->drb;
447 m_extaddref(m, (char *)drb->buf + db->placed, len, &drb->refs,
449 m->m_pkthdr.len = len;
450 m->m_len = len;
452 sb = &inp->inp_socket->so_rcv;
456 db->placed += len;
457 toep->ofld_rxq->rx_toe_ddp_octets += len;
464 struct inpcb *inp = toep->inp;
466 struct ddp_buffer *db;
479 ddp_rcvbuf = (toep->ddp.flags & DDP_RCVBUF) != 0;
480 tp->rcv_nxt += n;
482 KASSERT(tp->rcv_wnd >= n, ("%s: negative window size", __func__));
483 tp->rcv_wnd -= n;
487 while (toep->ddp.active_count > 0) {
488 MPASS(toep->ddp.active_id != -1);
489 db_idx = toep->ddp.active_id;
493 MPASS((toep->ddp.flags & db_flag) != 0);
494 db = &toep->ddp.db[db_idx];
497 if (placed > db->drb->len - db->placed)
498 placed = db->drb->len - db->placed;
501 complete_ddp_buffer(toep, db, db_idx);
502 n -= placed;
505 job = db->job;
506 copied = job->aio_received;
508 if (placed > job->uaiocb.aio_nbytes - copied)
509 placed = job->uaiocb.aio_nbytes - copied;
511 job->msgrcv = 1;
512 toep->ofld_rxq->rx_aio_ddp_jobs++;
514 toep->ofld_rxq->rx_aio_ddp_octets += placed;
521 job->aio_received += placed;
529 TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list);
530 toep->ddp.waiting_count++;
533 n -= placed;
534 complete_ddp_buffer(toep, db, db_idx);
554 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
555 ulpmc->len = htobe32(howmany(LEN__RX_DATA_ACK_ULP, 16));
558 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
559 ulpsc->len = htobe32(sizeof(*req));
562 OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tid));
563 req->credit_dack = htobe32(F_RX_MODULATE_RX);
567 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
568 ulpsc->len = htobe32(0);
598 wr = alloc_wrqe(wrlen, toep->ctrlq);
606 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid,
609 V_TCB_RX_DDP_BUF0_TAG(prsv->prsv_tag));
613 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid,
620 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid,
628 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_RX_DDP_FLAGS,
643 struct inpcb *inp = toep->inp;
644 struct ddp_buffer *db;
658 sb = &so->so_rcv;
661 KASSERT(toep->ddp.active_id == db_idx,
663 toep->ddp.active_id, toep->tid));
664 db = &toep->ddp.db[db_idx];
665 job = db->job;
667 if (__predict_false(inp->inp_flags & INP_DROPPED)) {
673 __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
687 * For RX_DATA_DDP, len might be non-zero, but it is only the
693 len += be32toh(rcv_nxt) - tp->rcv_nxt;
694 tp->rcv_nxt += len;
695 tp->t_rcvtime = ticks;
697 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
698 tp->rcv_wnd -= len;
702 toep->tid, db_idx, len, report);
706 MPASS(toep->vnet == so->so_vnet);
707 CURVNET_SET(toep->vnet);
709 if (sb->sb_flags & SB_AUTOSIZE &&
711 sb->sb_hiwat < V_tcp_autorcvbuf_max &&
713 struct adapter *sc = td_adapter(toep->td);
714 unsigned int hiwat = sb->sb_hiwat;
715 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
719 sb->sb_flags &= ~SB_AUTOSIZE;
724 job->msgrcv = 1;
725 toep->ofld_rxq->rx_aio_ddp_jobs++;
726 toep->ofld_rxq->rx_aio_ddp_octets += len;
727 if (db->cancel_pending) {
732 job->aio_received += len;
739 job->aio_received += len;
741 copied = job->aio_received;
745 __func__, toep->tid, job, copied, len);
748 t4_rcvd(&toep->td->tod, tp);
752 complete_ddp_buffer(toep, db, db_idx);
753 if (toep->ddp.waiting_count > 0)
765 struct adapter *sc = td_adapter(toep->td);
766 struct ddp_buffer *db;
773 KASSERT((toep->ddp.flags & DDP_DEAD) == 0, ("%s: DDP_DEAD", __func__));
774 KASSERT(toep->ddp.active_count < nitems(toep->ddp.db),
778 if (toep->ddp.db[0].drb == NULL) {
781 MPASS(toep->ddp.db[1].drb == NULL);
809 MPASS((toep->ddp.flags & buf_flag) == 0);
810 if ((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0) {
812 MPASS(toep->ddp.active_id == -1);
813 MPASS(toep->ddp.active_count == 0);
822 wr = mk_update_tcb_for_ddp(sc, toep, db_idx, &drb->prsv, 0, drb->len,
833 toep->tid, db_idx, ddp_flags, ddp_flags_mask);
839 drb->refs = 1;
841 /* Give the chip the go-ahead. */
843 db = &toep->ddp.db[db_idx];
844 db->drb = drb;
845 toep->ddp.flags |= buf_flag;
846 toep->ddp.active_count++;
847 if (toep->ddp.active_count == 1) {
848 MPASS(toep->ddp.active_id == -1);
849 toep->ddp.active_id = db_idx;
851 toep->ddp.active_id);
861 struct inpcb *inp = toep->inp;
865 struct ddp_buffer *db;
876 sb = &so->so_rcv;
879 KASSERT(toep->ddp.active_id == db_idx,
881 toep->ddp.active_id, toep->tid));
882 db = &toep->ddp.db[db_idx];
884 if (__predict_false(inp->inp_flags & INP_DROPPED)) {
890 __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
892 complete_ddp_buffer(toep, db, db_idx);
904 * For RX_DATA_DDP, len might be non-zero, but it is only the
910 len += be32toh(rcv_nxt) - tp->rcv_nxt;
911 tp->rcv_nxt += len;
912 tp->t_rcvtime = ticks;
914 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
915 tp->rcv_wnd -= len;
919 toep->tid, db_idx, len, report);
923 MPASS(toep->vnet == so->so_vnet);
924 CURVNET_SET(toep->vnet);
926 if (sb->sb_flags & SB_AUTOSIZE &&
928 sb->sb_hiwat < V_tcp_autorcvbuf_max &&
930 struct adapter *sc = td_adapter(toep->td);
931 unsigned int hiwat = sb->sb_hiwat;
932 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
936 sb->sb_flags &= ~SB_AUTOSIZE;
941 t4_rcvd_locked(&toep->td->tod, tp);
948 complete_ddp_buffer(toep, db, db_idx);
950 KASSERT(db->placed < db->drb->len,
953 if (toep->ddp.active_count != nitems(toep->ddp.db)) {
975 if ((toep->ddp.flags & DDP_RCVBUF) != 0)
986 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
995 MPASS(toep->ddp.active_count == 0);
996 MPASS((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0);
997 if (toep->ddp.waiting_count == 0) {
1007 toep->tid, toep->ddp.waiting_count);
1016 struct adapter *sc = iq->adapter;
1022 struct ddp_buffer *db;
1026 if (cpl->status != CPL_ERR_NONE)
1027 panic("XXX: tcp_rpl failed: %d", cpl->status);
1030 inp = toep->inp;
1031 switch (cpl->cookie) {
1037 KASSERT((toep->ddp.flags & DDP_AIO) != 0,
1039 db_idx = G_COOKIE(cpl->cookie) - CPL_COOKIE_DDP0;
1040 MPASS(db_idx < nitems(toep->ddp.db));
1043 db = &toep->ddp.db[db_idx];
1049 MPASS(db != NULL);
1050 MPASS(db->job != NULL);
1051 MPASS(db->cancel_pending);
1063 job = db->job;
1064 copied = job->aio_received;
1072 t4_rcvd(&toep->td->tod, intotcpcb(inp));
1075 complete_ddp_buffer(toep, db, db_idx);
1076 if (toep->ddp.waiting_count > 0)
1083 G_WORD(cpl->cookie), G_COOKIE(cpl->cookie));
1092 struct socket *so = toep->inp->inp_socket;
1093 struct sockbuf *sb = &so->so_rcv;
1094 struct ddp_buffer *db;
1104 INP_WLOCK_ASSERT(toep->inp);
1107 ddp_rcvbuf = (toep->ddp.flags & DDP_RCVBUF) != 0;
1109 /* - 1 is to ignore the byte for FIN */
1110 len = be32toh(rcv_nxt) - tp->rcv_nxt - 1;
1111 tp->rcv_nxt += len;
1114 toep->tid, len);
1115 while (toep->ddp.active_count > 0) {
1116 MPASS(toep->ddp.active_id != -1);
1117 db_idx = toep->ddp.active_id;
1121 MPASS((toep->ddp.flags & db_flag) != 0);
1122 db = &toep->ddp.db[db_idx];
1125 if (placed > db->drb->len - db->placed)
1126 placed = db->drb->len - db->placed;
1133 complete_ddp_buffer(toep, db, db_idx);
1134 len -= placed;
1137 job = db->job;
1138 copied = job->aio_received;
1140 if (placed > job->uaiocb.aio_nbytes - copied)
1141 placed = job->uaiocb.aio_nbytes - copied;
1143 job->msgrcv = 1;
1144 toep->ofld_rxq->rx_aio_ddp_jobs++;
1146 toep->ofld_rxq->rx_aio_ddp_octets += placed;
1153 job->aio_received += placed;
1156 __func__, toep->tid, db_idx, placed);
1159 len -= placed;
1160 complete_ddp_buffer(toep, db, db_idx);
1164 if ((toep->ddp.flags & DDP_AIO) != 0)
1178 struct adapter *sc = iq->adapter;
1185 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
1186 KASSERT(!(toep->flags & TPF_SYNQE),
1189 vld = be32toh(cpl->ddpvld);
1200 handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len));
1209 struct adapter *sc = iq->adapter;
1215 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
1216 KASSERT(!(toep->flags & TPF_SYNQE),
1219 handle_ddp_data(toep, cpl->ddp_report, cpl->rcv_nxt, 0);
1227 struct adapter *sc = toep->vi->adapter;
1233 if (!sc->tt.ddp)
1253 wr = alloc_wrqe(len, toep->ctrlq);
1257 CTR(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
1267 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 26,
1271 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 28,
1275 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 30,
1279 toep->params.ulp_mode = ULP_MODE_TCPDDP;
1280 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_ULP_TYPE,
1285 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_T_FLAGS,
1300 KASSERT((toep->ddp.flags & (DDP_ON | DDP_OK | DDP_SC_REQ)) == DDP_OK,
1302 __func__, toep, toep->ddp.flags));
1305 __func__, toep->tid, time_uptime);
1308 if ((toep->ddp.flags & DDP_AIO) != 0)
1312 toep->ddp.flags |= DDP_SC_REQ;
1313 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_RX_DDP_FLAGS,
1317 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS,
1349 return (howmany(npages >> (ddp_page_shift - PAGE_SHIFT), PPOD_PAGES));
1358 if (vmem_alloc(pr->pr_arena, PPOD_SZ(nppods), M_NOWAIT | M_FIRSTFIT,
1363 CTR5(KTR_CXGBE, "%-17s arena %p, addr 0x%08x, nppods %d, pgsz %d",
1364 __func__, pr->pr_arena, (uint32_t)addr & pr->pr_tag_mask,
1365 nppods, 1 << pr->pr_page_shift[pgsz_idx]);
1373 MPASS((addr & pr->pr_tag_mask) == addr);
1374 MPASS((addr & pr->pr_invalid_bit) == 0);
1376 prsv->prsv_pr = pr;
1377 prsv->prsv_tag = V_PPOD_PGSZ(pgsz_idx) | addr;
1378 prsv->prsv_nppods = nppods;
1399 while (i < npages - 1 &&
1407 if (hcf < (1 << pr->pr_page_shift[1])) {
1413 #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1)
1415 for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) {
1424 nppods = pages_to_nppods(npages, pr->pr_page_shift[idx]);
1427 MPASS(prsv->prsv_nppods > 0);
1435 struct ppod_reservation *prsv = &ps->prsv;
1437 KASSERT(prsv->prsv_nppods == 0,
1440 return (t4_alloc_page_pods_for_vmpages(pr, ps->pages, ps->npages,
1449 MPASS(bp->bio_flags & BIO_UNMAPPED);
1451 return (t4_alloc_page_pods_for_vmpages(pr, bp->bio_ma, bp->bio_ma_n,
1474 end_pva = trunc_page(buf + len - 1);
1486 if (hcf < (1 << pr->pr_page_shift[1])) {
1492 #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1)
1494 for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) {
1504 npages += (end_pva - start_pva) >> pr->pr_page_shift[idx];
1508 MPASS(prsv->prsv_nppods > 0);
1517 struct ppod_reservation *prsv = &drb->prsv;
1519 KASSERT(prsv->prsv_nppods == 0,
1522 return (t4_alloc_page_pods_for_buf(pr, (vm_offset_t)drb->buf, drb->len,
1546 for (i = entries - 1; i >= 0; i--) {
1548 buf = (vm_offset_t)sge->addr;
1549 len = sge->len;
1551 end_pva = trunc_page(buf + len - 1);
1564 if (hcf < (1 << pr->pr_page_shift[1])) {
1570 #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1)
1572 for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) {
1582 while (entries--) {
1584 start_pva = trunc_page((vm_offset_t)sgl->addr);
1585 end_pva = trunc_page((vm_offset_t)sgl->addr + sgl->len - 1);
1586 npages += (end_pva - start_pva) >> pr->pr_page_shift[idx];
1592 MPASS(prsv->prsv_nppods > 0);
1599 struct ppod_region *pr = prsv->prsv_pr;
1603 MPASS(prsv->prsv_nppods != 0);
1605 addr = prsv->prsv_tag & pr->pr_tag_mask;
1606 MPASS((addr & pr->pr_invalid_bit) == 0);
1609 CTR4(KTR_CXGBE, "%-17s arena %p, addr 0x%08x, nppods %d", __func__,
1610 pr->pr_arena, addr, prsv->prsv_nppods);
1613 vmem_free(pr->pr_arena, addr, PPOD_SZ(prsv->prsv_nppods));
1614 prsv->prsv_nppods = 0;
1629 uint32_t cmd;
1630 struct ppod_reservation *prsv = &ps->prsv;
1631 struct ppod_region *pr = prsv->prsv_pr;
1634 KASSERT(!(ps->flags & PS_PPODS_WRITTEN),
1636 MPASS(prsv->prsv_nppods > 0);
1638 cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));
1640 cmd |= htobe32(F_ULP_MEMIO_ORDER);
1642 cmd |= htobe32(F_T5_ULP_MEMIO_IMM);
1643 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
1644 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
1645 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
1647 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
1657 ulpmc->cmd = cmd;
1658 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32));
1659 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
1660 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
1663 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1664 ulpsc->len = htobe32(chunk);
1668 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
1669 V_PPOD_TID(tid) | prsv->prsv_tag);
1670 ppod->len_offset = htobe64(V_PPOD_LEN(ps->len) |
1671 V_PPOD_OFST(ps->offset));
1672 ppod->rsvd = 0;
1674 for (k = 0; k < nitems(ppod->addr); k++) {
1675 if (idx < ps->npages) {
1676 pa = VM_PAGE_TO_PHYS(ps->pages[idx]);
1677 ppod->addr[k] = htobe64(pa);
1680 ppod->addr[k] = 0;
1683 "%s: tid %d ppod[%d]->addr[%d] = %p",
1685 be64toh(ppod->addr[k]));
1693 ps->flags |= PS_PPODS_WRITTEN;
1708 uint32_t cmd;
1709 struct ppod_reservation *prsv = &drb->prsv;
1710 struct ppod_region *pr = prsv->prsv_pr;
1714 MPASS(prsv->prsv_nppods > 0);
1716 cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));
1718 cmd |= htobe32(F_ULP_MEMIO_ORDER);
1720 cmd |= htobe32(F_T5_ULP_MEMIO_IMM);
1721 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
1722 offset = (uintptr_t)drb->buf & PAGE_MASK;
1723 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
1724 pva = trunc_page((uintptr_t)drb->buf);
1725 end_pva = trunc_page((uintptr_t)drb->buf + drb->len - 1);
1726 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
1728 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
1739 ulpmc->cmd = cmd;
1740 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32));
1741 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
1742 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
1745 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1746 ulpsc->len = htobe32(chunk);
1750 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
1751 V_PPOD_TID(tid) | prsv->prsv_tag);
1752 ppod->len_offset = htobe64(V_PPOD_LEN(drb->len) |
1754 ppod->rsvd = 0;
1756 for (k = 0; k < nitems(ppod->addr); k++) {
1758 ppod->addr[k] = 0;
1761 ppod->addr[k] = htobe64(pa);
1766 "%s: tid %d ppod[%d]->addr[%d] = %p",
1768 be64toh(ppod->addr[k]));
1777 pva -= ddp_pgsz;
1801 m->m_pkthdr.len = len;
1802 m->m_len = len;
1816 uint32_t cmd;
1817 struct ppod_region *pr = prsv->prsv_pr;
1821 MPASS(bp->bio_flags & BIO_UNMAPPED);
1823 cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));
1825 cmd |= htobe32(F_ULP_MEMIO_ORDER);
1827 cmd |= htobe32(F_T5_ULP_MEMIO_IMM);
1828 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
1829 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
1830 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
1833 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
1843 INIT_ULPTX_WR(ulpmc, len, 0, toep->tid);
1844 ulpmc->cmd = cmd;
1845 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32));
1846 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
1847 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
1850 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1851 ulpsc->len = htobe32(chunk);
1855 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
1856 V_PPOD_TID(toep->tid) |
1857 (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ)));
1858 ppod->len_offset = htobe64(V_PPOD_LEN(bp->bio_bcount) |
1859 V_PPOD_OFST(bp->bio_ma_offset));
1860 ppod->rsvd = 0;
1862 for (k = 0; k < nitems(ppod->addr); k++) {
1863 if (idx < bp->bio_ma_n) {
1864 pa = VM_PAGE_TO_PHYS(bp->bio_ma[idx]);
1865 ppod->addr[k] = htobe64(pa);
1868 ppod->addr[k] = 0;
1871 "%s: tid %d ppod[%d]->addr[%d] = %p",
1872 __func__, toep->tid, i, k,
1873 be64toh(ppod->addr[k]));
1894 uint32_t cmd;
1895 struct ppod_region *pr = prsv->prsv_pr;
1900 cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));
1902 cmd |= htobe32(F_ULP_MEMIO_ORDER);
1904 cmd |= htobe32(F_T5_ULP_MEMIO_IMM);
1905 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
1907 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
1909 end_pva = trunc_page(buf + buflen - 1);
1910 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
1913 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
1923 INIT_ULPTX_WR(ulpmc, len, 0, toep->tid);
1924 ulpmc->cmd = cmd;
1925 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32));
1926 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
1927 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
1930 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1931 ulpsc->len = htobe32(chunk);
1935 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
1936 V_PPOD_TID(toep->tid) |
1937 (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ)));
1938 ppod->len_offset = htobe64(V_PPOD_LEN(buflen) |
1940 ppod->rsvd = 0;
1942 for (k = 0; k < nitems(ppod->addr); k++) {
1944 ppod->addr[k] = 0;
1947 ppod->addr[k] = htobe64(pa);
1952 "%s: tid %d ppod[%d]->addr[%d] = %p",
1953 __func__, toep->tid, i, k,
1954 be64toh(ppod->addr[k]));
1963 pva -= ddp_pgsz;
1984 uint32_t cmd;
1985 struct ppod_region *pr = prsv->prsv_pr;
1992 cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));
1994 cmd |= htobe32(F_ULP_MEMIO_ORDER);
1996 cmd |= htobe32(F_T5_ULP_MEMIO_IMM);
1997 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
1998 offset = (vm_offset_t)sgl->addr & PAGE_MASK;
1999 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
2000 pva = trunc_page((vm_offset_t)sgl->addr);
2001 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
2004 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
2014 INIT_ULPTX_WR(ulpmc, len, 0, toep->tid);
2015 ulpmc->cmd = cmd;
2016 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32));
2017 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
2018 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
2021 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
2022 ulpsc->len = htobe32(chunk);
2026 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
2027 V_PPOD_TID(toep->tid) |
2028 (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ)));
2029 ppod->len_offset = htobe64(V_PPOD_LEN(xferlen) |
2031 ppod->rsvd = 0;
2033 for (k = 0; k < nitems(ppod->addr); k++) {
2036 ppod->addr[k] = htobe64(pa);
2038 ppod->addr[k] = 0;
2042 "%s: tid %d ppod[%d]->addr[%d] = %p",
2043 __func__, toep->tid, i, k,
2044 be64toh(ppod->addr[k]));
2052 if (k + 1 == nitems(ppod->addr))
2063 if (sg_offset == sgl->len) {
2068 entries--;
2073 (vm_offset_t)sgl->addr);
2090 struct tom_data *td = sc->tom_softc;
2092 if (ps->prsv.prsv_nppods == 0 &&
2093 t4_alloc_page_pods_for_ps(&td->pr, ps) != 0) {
2096 if (!(ps->flags & PS_PPODS_WRITTEN) &&
2097 t4_write_page_pods_for_ps(sc, toep->ctrlq, toep->tid, ps) != 0) {
2111 MPASS(r->size > 0);
2113 pr->pr_start = r->start;
2114 pr->pr_len = r->size;
2115 pr->pr_page_shift[0] = 12 + G_HPZ0(psz);
2116 pr->pr_page_shift[1] = 12 + G_HPZ1(psz);
2117 pr->pr_page_shift[2] = 12 + G_HPZ2(psz);
2118 pr->pr_page_shift[3] = 12 + G_HPZ3(psz);
2120 /* The SGL -> page pod algorithm requires the sizes to be in order. */
2121 for (i = 1; i < nitems(pr->pr_page_shift); i++) {
2122 if (pr->pr_page_shift[i] <= pr->pr_page_shift[i - 1])
2126 pr->pr_tag_mask = ((1 << fls(r->size)) - 1) & V_PPOD_TAG(M_PPOD_TAG);
2127 pr->pr_alias_mask = V_PPOD_TAG(M_PPOD_TAG) & ~pr->pr_tag_mask;
2128 if (pr->pr_tag_mask == 0 || pr->pr_alias_mask == 0)
2130 pr->pr_alias_shift = fls(pr->pr_tag_mask);
2131 pr->pr_invalid_bit = 1 << (pr->pr_alias_shift - 1);
2133 pr->pr_arena = vmem_create(name, 0, pr->pr_len, PPOD_SIZE, 0,
2135 if (pr->pr_arena == NULL)
2147 if (pr->pr_arena)
2148 vmem_destroy(pr->pr_arena);
2157 if (ps->start != start || ps->npages != npages ||
2158 ps->offset != pgoff || ps->len != len)
2161 return (ps->vm != vm || ps->vm_timestamp != vm->vm_map.timestamp);
2180 vm = job->userproc->p_vmspace;
2181 map = &vm->vm_map;
2182 start = (uintptr_t)job->uaiocb.aio_buf;
2184 end = round_page(start + job->uaiocb.aio_nbytes);
2187 if (end - start > MAX_DDP_BUFFER_SIZE) {
2199 __func__, toep->tid, (unsigned long)job->uaiocb.aio_nbytes,
2200 (unsigned long)(end - (start + pgoff)));
2201 job->uaiocb.aio_nbytes = end - (start + pgoff);
2206 n = atop(end - start);
2211 TAILQ_FOREACH(ps, &toep->ddp.cached_pagesets, link) {
2213 job->uaiocb.aio_nbytes) == 0) {
2214 TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link);
2215 toep->ddp.cached_count--;
2225 KASSERT(toep->ddp.active_count + toep->ddp.cached_count <=
2226 nitems(toep->ddp.db), ("%s: too many wired pagesets", __func__));
2227 if (toep->ddp.active_count + toep->ddp.cached_count ==
2228 nitems(toep->ddp.db)) {
2229 KASSERT(toep->ddp.cached_count > 0,
2231 ps = TAILQ_LAST(&toep->ddp.cached_pagesets, pagesetq);
2232 TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link);
2233 toep->ddp.cached_count--;
2234 free_pageset(toep->td, ps);
2241 ps->pages = (vm_page_t *)(ps + 1);
2242 ps->vm_timestamp = map->timestamp;
2243 ps->npages = vm_fault_quick_hold_pages(map, start, end - start,
2244 VM_PROT_WRITE, ps->pages, n);
2247 if (ps->npages < 0) {
2252 KASSERT(ps->npages == n, ("hold_aio: page count mismatch: %d vs %d",
2253 ps->npages, n));
2255 ps->offset = pgoff;
2256 ps->len = job->uaiocb.aio_nbytes;
2257 refcount_acquire(&vm->vm_refcnt);
2258 ps->vm = vm;
2259 ps->start = start;
2262 __func__, toep->tid, ps, job, ps->npages);
2273 KASSERT((toep->ddp.flags & DDP_AIO) != 0, ("%s: DDP_RCVBUF", __func__));
2274 while (!TAILQ_EMPTY(&toep->ddp.aiojobq)) {
2275 job = TAILQ_FIRST(&toep->ddp.aiojobq);
2276 TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);
2277 toep->ddp.waiting_count--;
2293 copied = job->aio_received;
2310 if (!(toep->ddp.flags & DDP_DEAD) &&
2312 TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list);
2313 toep->ddp.waiting_count++;
2321 struct adapter *sc = td_adapter(toep->td);
2326 struct ddp_buffer *db;
2337 if (toep->ddp.flags & DDP_DEAD) {
2338 MPASS(toep->ddp.waiting_count == 0);
2339 MPASS(toep->ddp.active_count == 0);
2343 if (toep->ddp.waiting_count == 0 ||
2344 toep->ddp.active_count == nitems(toep->ddp.db)) {
2348 job = TAILQ_FIRST(&toep->ddp.aiojobq);
2349 so = job->fd_file->f_data;
2350 sb = &so->so_rcv;
2354 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
2360 KASSERT(toep->ddp.active_count == 0 || sbavail(sb) == 0,
2365 if (so->so_error && sbavail(sb) == 0) {
2366 toep->ddp.waiting_count--;
2367 TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);
2378 copied = job->aio_received;
2384 error = so->so_error;
2385 so->so_error = 0;
2387 aio_complete(job, -1, error);
2396 if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) {
2398 if (toep->ddp.active_count != 0)
2408 if (sbavail(sb) == 0 && (toep->ddp.flags & DDP_ON) == 0) {
2422 if ((toep->ddp.flags & DDP_SC_REQ) == 0)
2432 if (toep->ddp.queueing != NULL)
2436 toep->ddp.waiting_count--;
2437 TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);
2440 toep->ddp.queueing = job;
2446 toep->ddp.queueing = NULL;
2451 if (so->so_error && sbavail(sb) == 0) {
2452 copied = job->aio_received;
2457 toep->ddp.queueing = NULL;
2461 error = so->so_error;
2462 so->so_error = 0;
2465 aio_complete(job, -1, error);
2466 toep->ddp.queueing = NULL;
2470 if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) {
2473 if (toep->ddp.active_count != 0) {
2480 toep->ddp.queueing = NULL;
2485 toep->ddp.queueing = NULL;
2494 MPASS(!(toep->ddp.flags & DDP_DEAD));
2502 offset = ps->offset + job->aio_received;
2503 MPASS(job->aio_received <= job->uaiocb.aio_nbytes);
2504 resid = job->uaiocb.aio_nbytes - job->aio_received;
2505 m = sb->sb_mb;
2506 KASSERT(m == NULL || toep->ddp.active_count == 0,
2516 iov[0].iov_len = m->m_len;
2526 error = uiomove_fromphys(ps->pages, offset + copied,
2529 uiomove_fromphys(ps->pages, offset + copied, uio.uio_resid, &uio);
2533 resid -= uio.uio_offset;
2534 m = m->m_next;
2538 job->aio_received += copied;
2539 job->msgrcv = 1;
2540 copied = job->aio_received;
2561 t4_rcvd_locked(&toep->td->tod, intotcpcb(inp));
2563 if (resid == 0 || toep->ddp.flags & DDP_DEAD) {
2573 toep->ddp.queueing = NULL;
2582 if ((toep->ddp.flags & (DDP_ON | DDP_SC_REQ)) != DDP_ON) {
2586 toep->ddp.queueing = NULL;
2603 toep->ddp.queueing = NULL;
2614 if (toep->ddp.db[0].job == NULL) {
2617 MPASS(toep->ddp.db[1].job == NULL);
2625 if (so->so_state & SS_NBIO)
2633 if (so->so_state & SS_NBIO)
2640 MPASS((toep->ddp.flags & buf_flag) == 0);
2641 if ((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0) {
2643 MPASS(toep->ddp.active_id == -1);
2644 MPASS(toep->ddp.active_count == 0);
2656 wr = mk_update_tcb_for_ddp(sc, toep, db_idx, &ps->prsv,
2657 job->aio_received, ps->len, ddp_flags, ddp_flags_mask);
2661 toep->ddp.queueing = NULL;
2679 toep->ddp.queueing = NULL;
2686 toep->tid, job, db_idx, ddp_flags, ddp_flags_mask);
2688 /* Give the chip the go-ahead. */
2690 db = &toep->ddp.db[db_idx];
2691 db->cancel_pending = 0;
2692 db->job = job;
2693 db->ps = ps;
2694 toep->ddp.queueing = NULL;
2695 toep->ddp.flags |= buf_flag;
2696 toep->ddp.active_count++;
2697 if (toep->ddp.active_count == 1) {
2698 MPASS(toep->ddp.active_id == -1);
2699 toep->ddp.active_id = db_idx;
2701 toep->ddp.active_id);
2711 if (toep->ddp.flags & DDP_TASK_ACTIVE)
2713 toep->ddp.flags |= DDP_TASK_ACTIVE;
2715 soaio_enqueue(&toep->ddp.requeue_task);
2725 toep->ddp.flags &= ~DDP_TASK_ACTIVE;
2734 struct socket *so = job->fd_file->f_data;
2736 struct toepcb *toep = tp->t_toe;
2737 struct adapter *sc = td_adapter(toep->td);
2748 for (i = 0; i < nitems(toep->ddp.db); i++) {
2749 if (toep->ddp.db[i].job == job) {
2751 MPASS(toep->ddp.db[i].cancel_pending == 0);
2760 t4_set_tcb_field(sc, toep->ctrlq, toep,
2763 toep->ddp.db[i].cancel_pending = 1;
2775 struct socket *so = job->fd_file->f_data;
2777 struct toepcb *toep = tp->t_toe;
2781 TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);
2782 toep->ddp.waiting_count--;
2783 if (toep->ddp.waiting_count == 0)
2797 struct toepcb *toep = tp->t_toe;
2800 if (job->uaiocb.aio_lio_opcode != LIO_READ)
2818 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
2823 if ((toep->ddp.flags & DDP_AIO) == 0) {
2824 toep->ddp.flags |= DDP_AIO;
2825 TAILQ_INIT(&toep->ddp.cached_pagesets);
2826 TAILQ_INIT(&toep->ddp.aiojobq);
2827 TASK_INIT(&toep->ddp.requeue_task, 0, aio_ddp_requeue_task,
2838 CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid);
2842 TAILQ_INSERT_TAIL(&toep->ddp.aiojobq, job, list);
2843 toep->ddp.waiting_count++;
2865 if ((toep->ddp.flags & DDP_DEAD) != 0) {
2866 MPASS(toep->ddp.active_count == 0);
2871 if (toep->ddp.active_count == nitems(toep->ddp.db)) {
2875 inp = toep->inp;
2876 so = inp->inp_socket;
2877 sb = &so->so_rcv;
2892 if ((toep->ddp.flags & DDP_DEAD) != 0 ||
2893 toep->ddp.active_count == nitems(toep->ddp.db)) {
2900 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
2907 if (so->so_error != 0 || (sb->sb_state & SBS_CANTRCVMORE) != 0) {
2935 toep->ddp.flags &= ~DDP_TASK_ACTIVE;
2945 struct adapter *sc = td_adapter(toep->td);
2967 if ((toep->ddp.flags & DDP_AIO) != 0) {
2972 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
2977 toep->ddp.flags |= DDP_RCVBUF;
2978 TAILQ_INIT(&toep->ddp.cached_buffers);
2980 TASK_INIT(&toep->ddp.requeue_task, 0, ddp_rcvbuf_requeue_task, toep);