1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _RTE_OCTEONTX_ZIP_VF_H_ 6 #define _RTE_OCTEONTX_ZIP_VF_H_ 7 8 #include <unistd.h> 9 10 #include <bus_pci_driver.h> 11 #include <rte_comp.h> 12 #include <rte_compressdev.h> 13 #include <rte_compressdev_pmd.h> 14 #include <rte_malloc.h> 15 #include <rte_memory.h> 16 #include <rte_spinlock.h> 17 18 #include <zip_regs.h> 19 20 extern int octtx_zip_logtype_driver; 21 22 /* ZIP VF Control/Status registers (CSRs): */ 23 /* VF_BAR0: */ 24 #define ZIP_VQ_ENA (0x10) 25 #define ZIP_VQ_SBUF_ADDR (0x20) 26 #define ZIP_VF_PF_MBOXX(x) (0x400 | (x)<<3) 27 #define ZIP_VQ_DOORBELL (0x1000) 28 29 /**< Vendor ID */ 30 #define PCI_VENDOR_ID_CAVIUM 0x177D 31 /**< PCI device id of ZIP VF */ 32 #define PCI_DEVICE_ID_OCTEONTX_ZIPVF 0xA037 33 #define PCI_DEVICE_ID_OCTEONTX2_ZIPVF 0xA083 34 35 /* maximum number of zip vf devices */ 36 #define ZIP_MAX_VFS 8 37 38 /* max size of one chunk */ 39 #define ZIP_MAX_CHUNK_SIZE 8192 40 41 /* each instruction is fixed 128 bytes */ 42 #define ZIP_CMD_SIZE 128 43 44 #define ZIP_CMD_SIZE_WORDS (ZIP_CMD_SIZE >> 3) /* 16 64_bit words */ 45 46 /* size of next chunk buffer pointer */ 47 #define ZIP_MAX_NCBP_SIZE 8 48 49 /* size of instruction queue in units of instruction size */ 50 #define ZIP_MAX_NUM_CMDS ((ZIP_MAX_CHUNK_SIZE - ZIP_MAX_NCBP_SIZE) / \ 51 ZIP_CMD_SIZE) /* 63 */ 52 53 /* size of instruct queue in bytes */ 54 #define ZIP_MAX_CMDQ_SIZE ((ZIP_MAX_NUM_CMDS * ZIP_CMD_SIZE) + \ 55 ZIP_MAX_NCBP_SIZE)/* ~8072ull */ 56 57 #define ZIP_BUF_SIZE 256 58 #define ZIP_SGBUF_SIZE (5 * 1024) 59 #define ZIP_BURST_SIZE 64 60 61 #define ZIP_MAXSEG_SIZE 59460 62 #define ZIP_EXTRABUF_SIZE 4096 63 #define ZIP_MAX_SEGS 300 64 #define ZIP_MAX_DATA_SIZE (16*1024*1024) 65 66 #define ZIP_SGPTR_ALIGN 16 67 #define ZIP_CMDQ_ALIGN 128 68 #define MAX_SG_LEN ((ZIP_BUF_SIZE - ZIP_SGPTR_ALIGN) / sizeof(void *)) 69 70 /**< ZIP PMD specified queue pairs */ 71 #define ZIP_MAX_VF_QUEUE 1 72 73 #define ZIP_ALIGN_ROUNDUP(x, _align) \ 74 ((_align) * (((x) + (_align) - 1) / (_align))) 75 76 /**< ZIP PMD device name */ 77 #define COMPRESSDEV_NAME_ZIP_PMD compress_octeontx 78 79 #define ZIP_PMD_LOG(level, fmt, args...) \ 80 rte_log(RTE_LOG_ ## level, \ 81 octtx_zip_logtype_driver, "%s(): "fmt "\n", \ 82 __func__, ##args) 83 84 #define ZIP_PMD_INFO(fmt, args...) \ 85 ZIP_PMD_LOG(INFO, fmt, ## args) 86 #define ZIP_PMD_ERR(fmt, args...) \ 87 ZIP_PMD_LOG(ERR, fmt, ## args) 88 89 /* resources required to process stream */ 90 enum NUM_BUFS_PER_STREAM { 91 RES_BUF = 0, 92 CMD_BUF, 93 HASH_CTX_BUF, 94 DECOMP_CTX_BUF, 95 IN_DATA_BUF, 96 OUT_DATA_BUF, 97 HISTORY_DATA_BUF, 98 MAX_BUFS_PER_STREAM 99 }; 100 101 struct zip_stream; 102 struct zipvf_qp; 103 104 /* Algorithm handler function prototype */ 105 typedef int (*comp_func_t)(struct rte_comp_op *op, struct zipvf_qp *qp, 106 struct zip_stream *zstrm, int num); 107 108 /* Scatter gather list */ 109 struct zipvf_sginfo { 110 union zip_zptr_addr_s sg_addr; 111 union zip_zptr_ctl_s sg_ctl; 112 } __rte_aligned(16); 113 114 /** 115 * ZIP private stream structure 116 */ 117 struct zip_stream { 118 union zip_inst_s *inst[ZIP_BURST_SIZE]; 119 /* zip instruction pointer */ 120 comp_func_t func; 121 /* function to process comp operation */ 122 void *bufs[MAX_BUFS_PER_STREAM * ZIP_BURST_SIZE]; 123 } __rte_cache_aligned; 124 125 126 /** 127 * ZIP instruction Queue 128 */ 129 struct zipvf_cmdq { 130 rte_spinlock_t qlock; 131 /* queue lock */ 132 uint64_t *sw_head; 133 /* pointer to start of 8-byte word length queue-head */ 134 uint8_t *va; 135 /* pointer to instruction queue virtual address */ 136 rte_iova_t iova; 137 /* iova addr of cmdq head*/ 138 }; 139 140 /** 141 * ZIP device queue structure 142 */ 143 struct zipvf_qp { 144 struct zipvf_cmdq cmdq; 145 /* Hardware instruction queue structure */ 146 struct rte_ring *processed_pkts; 147 /* Ring for placing processed packets */ 148 struct rte_compressdev_stats qp_stats; 149 /* Queue pair statistics */ 150 uint16_t id; 151 /* Queue Pair Identifier */ 152 const char *name; 153 /* Unique Queue Pair Name */ 154 struct zip_vf *vf; 155 /* pointer to device, queue belongs to */ 156 struct zipvf_sginfo *g_info; 157 struct zipvf_sginfo *s_info; 158 /* SGL pointers */ 159 uint64_t num_sgbuf; 160 uint64_t enqed; 161 } __rte_cache_aligned; 162 163 /** 164 * ZIP VF device structure. 165 */ 166 struct zip_vf { 167 int vfid; 168 /* vf index */ 169 struct rte_pci_device *pdev; 170 /* pci device */ 171 void *vbar0; 172 /* CSR base address for underlying BAR0 VF.*/ 173 uint64_t dom_sdom; 174 /* Storing mbox domain and subdomain id for app rerun*/ 175 uint32_t max_nb_queue_pairs; 176 /* pointer to device qps */ 177 struct rte_mempool *zip_mp; 178 struct rte_mempool *sg_mp; 179 /* pointer to pools */ 180 } __rte_cache_aligned; 181 182 183 static inline int 184 zipvf_prepare_sgl(struct rte_mbuf *buf, int64_t offset, struct zipvf_sginfo *sg_list, 185 uint32_t data_len, const uint16_t max_segs, struct zipvf_qp *qp) 186 { 187 struct zipvf_sginfo *sginfo = (struct zipvf_sginfo *)sg_list; 188 uint32_t tot_buf_len, sgidx; 189 int ret = -EINVAL; 190 191 for (sgidx = tot_buf_len = 0; buf && sgidx < max_segs; buf = buf->next) { 192 if (offset >= rte_pktmbuf_data_len(buf)) { 193 offset -= rte_pktmbuf_data_len(buf); 194 continue; 195 } 196 197 sginfo[sgidx].sg_ctl.s.length = (uint16_t)(rte_pktmbuf_data_len(buf) - offset); 198 sginfo[sgidx].sg_addr.s.addr = rte_pktmbuf_iova_offset(buf, offset); 199 200 offset = 0; 201 tot_buf_len += sginfo[sgidx].sg_ctl.s.length; 202 203 if (tot_buf_len >= data_len) { 204 sginfo[sgidx].sg_ctl.s.length -= tot_buf_len - data_len; 205 ret = 0; 206 break; 207 } 208 209 ZIP_PMD_LOG(DEBUG, "ZIP SGL buf[%d], len = %d, iova = 0x%"PRIx64"\n", 210 sgidx, sginfo[sgidx].sg_ctl.s.length, sginfo[sgidx].sg_addr.s.addr); 211 ++sgidx; 212 } 213 214 if (unlikely(ret != 0)) { 215 if (sgidx == max_segs) 216 ZIP_PMD_ERR("Exceeded max segments in ZIP SGL (%u)", max_segs); 217 else 218 ZIP_PMD_ERR("Mbuf chain is too short"); 219 } 220 qp->num_sgbuf = ++sgidx; 221 222 ZIP_PMD_LOG(DEBUG, "Tot_buf_len:%d max_segs:%"PRIx64"\n", tot_buf_len, 223 qp->num_sgbuf); 224 return ret; 225 } 226 227 static inline int 228 zipvf_prepare_in_buf(union zip_inst_s *inst, struct zipvf_qp *qp, struct rte_comp_op *op) 229 { 230 uint32_t offset, inlen; 231 struct rte_mbuf *m_src; 232 int ret = 0; 233 234 inlen = op->src.length; 235 offset = op->src.offset; 236 m_src = op->m_src; 237 238 /* Gather input */ 239 if (op->m_src->next != NULL && inlen > ZIP_MAXSEG_SIZE) { 240 inst->s.dg = 1; 241 242 ret = zipvf_prepare_sgl(m_src, offset, qp->g_info, inlen, 243 op->m_src->nb_segs, qp); 244 245 inst->s.inp_ptr_addr.s.addr = rte_mem_virt2iova(qp->g_info); 246 inst->s.inp_ptr_ctl.s.length = qp->num_sgbuf; 247 inst->s.inp_ptr_ctl.s.fw = 0; 248 249 ZIP_PMD_LOG(DEBUG, "Gather(input): len(nb_segs):%d, iova: 0x%"PRIx64"\n", 250 inst->s.inp_ptr_ctl.s.length, inst->s.inp_ptr_addr.s.addr); 251 return ret; 252 } 253 254 /* Prepare direct input data pointer */ 255 inst->s.dg = 0; 256 inst->s.inp_ptr_addr.s.addr = rte_pktmbuf_iova_offset(m_src, offset); 257 inst->s.inp_ptr_ctl.s.length = inlen; 258 259 ZIP_PMD_LOG(DEBUG, "Direct input - inlen:%d\n", inlen); 260 return ret; 261 } 262 263 static inline int 264 zipvf_prepare_out_buf(union zip_inst_s *inst, struct zipvf_qp *qp, struct rte_comp_op *op) 265 { 266 uint32_t offset, outlen; 267 struct rte_mbuf *m_dst; 268 int ret = 0; 269 270 offset = op->dst.offset; 271 m_dst = op->m_dst; 272 outlen = rte_pktmbuf_pkt_len(m_dst) - op->dst.offset; 273 274 /* Scatter output */ 275 if (op->m_dst->next != NULL && outlen > ZIP_MAXSEG_SIZE) { 276 inst->s.ds = 1; 277 inst->s.totaloutputlength = outlen; 278 279 ret = zipvf_prepare_sgl(m_dst, offset, qp->s_info, inst->s.totaloutputlength, 280 m_dst->nb_segs, qp); 281 282 inst->s.out_ptr_addr.s.addr = rte_mem_virt2iova(qp->s_info); 283 inst->s.out_ptr_ctl.s.length = qp->num_sgbuf; 284 285 ZIP_PMD_LOG(DEBUG, "Scatter(output): nb_segs:%d, iova:0x%"PRIx64"\n", 286 inst->s.out_ptr_ctl.s.length, inst->s.out_ptr_addr.s.addr); 287 return ret; 288 } 289 290 /* Prepare direct output data pointer */ 291 inst->s.ds = 0; 292 inst->s.out_ptr_addr.s.addr = rte_pktmbuf_iova_offset(m_dst, offset); 293 inst->s.totaloutputlength = rte_pktmbuf_pkt_len(m_dst) - op->dst.offset; 294 if (inst->s.totaloutputlength == ZIP_MAXSEG_SIZE) 295 inst->s.totaloutputlength += ZIP_EXTRABUF_SIZE; /* DSTOP */ 296 297 inst->s.out_ptr_ctl.s.length = inst->s.totaloutputlength; 298 299 ZIP_PMD_LOG(DEBUG, "Direct output - outlen:%d\n", inst->s.totaloutputlength); 300 return ret; 301 } 302 303 static inline int 304 zipvf_prepare_cmd_stateless(struct rte_comp_op *op, struct zipvf_qp *qp, 305 union zip_inst_s *inst) 306 { 307 /* set flush flag to always 1*/ 308 inst->s.ef = 1; 309 310 if (inst->s.op == ZIP_OP_E_DECOMP) 311 inst->s.sf = 1; 312 else 313 inst->s.sf = 0; 314 315 /* Set input checksum */ 316 inst->s.adlercrc32 = op->input_chksum; 317 318 /* Prepare input/output buffers */ 319 if (zipvf_prepare_in_buf(inst, qp, op)) { 320 ZIP_PMD_ERR("Con't fill input SGL "); 321 return -EINVAL; 322 } 323 324 if (zipvf_prepare_out_buf(inst, qp, op)) { 325 ZIP_PMD_ERR("Con't fill output SGL "); 326 return -EINVAL; 327 } 328 329 return 0; 330 } 331 332 #ifdef ZIP_DBG 333 static inline void 334 zip_dump_instruction(void *inst) 335 { 336 union zip_inst_s *cmd83 = (union zip_inst_s *)inst; 337 338 printf("####### START ########\n"); 339 printf("ZIP Instr:0x%"PRIx64"\n", cmd83); 340 printf("doneint:%d totaloutputlength:%d\n", cmd83->s.doneint, 341 cmd83->s.totaloutputlength); 342 printf("exnum:%d iv:%d exbits:%d hmif:%d halg:%d\n", cmd83->s.exn, 343 cmd83->s.iv, cmd83->s.exbits, cmd83->s.hmif, cmd83->s.halg); 344 printf("flush:%d speed:%d cc:%d\n", cmd83->s.sf, 345 cmd83->s.ss, cmd83->s.cc); 346 printf("eof:%d bof:%d op:%d dscatter:%d dgather:%d hgather:%d\n", 347 cmd83->s.ef, cmd83->s.bf, cmd83->s.op, cmd83->s.ds, 348 cmd83->s.dg, cmd83->s.hg); 349 printf("historylength:%d adler32:%d\n", cmd83->s.historylength, 350 cmd83->s.adlercrc32); 351 printf("ctx_ptr.addr:0x%"PRIx64"\n", cmd83->s.ctx_ptr_addr.s.addr); 352 printf("ctx_ptr.len:%d\n", cmd83->s.ctx_ptr_ctl.s.length); 353 printf("history_ptr.addr:0x%"PRIx64"\n", cmd83->s.his_ptr_addr.s.addr); 354 printf("history_ptr.len:%d\n", cmd83->s.his_ptr_ctl.s.length); 355 printf("inp_ptr.addr:0x%"PRIx64"\n", cmd83->s.inp_ptr_addr.s.addr); 356 printf("inp_ptr.len:%d\n", cmd83->s.inp_ptr_ctl.s.length); 357 printf("out_ptr.addr:0x%"PRIx64"\n", cmd83->s.out_ptr_addr.s.addr); 358 printf("out_ptr.len:%d\n", cmd83->s.out_ptr_ctl.s.length); 359 printf("result_ptr.addr:0x%"PRIx64"\n", cmd83->s.res_ptr_addr.s.addr); 360 printf("result_ptr.len:%d\n", cmd83->s.res_ptr_ctl.s.length); 361 printf("####### END ########\n"); 362 } 363 #endif 364 365 int 366 zipvf_create(struct rte_compressdev *compressdev); 367 368 int 369 zipvf_destroy(struct rte_compressdev *compressdev); 370 371 int 372 zipvf_q_init(struct zipvf_qp *qp); 373 374 int 375 zipvf_q_term(struct zipvf_qp *qp); 376 377 void 378 zipvf_push_command(struct zipvf_qp *qp, union zip_inst_s *zcmd); 379 380 int 381 zip_process_op(struct rte_comp_op *op, struct zipvf_qp *qp, 382 struct zip_stream *zstrm, int num); 383 384 uint64_t 385 zip_reg_read64(uint8_t *hw_addr, uint64_t offset); 386 387 void 388 zip_reg_write64(uint8_t *hw_addr, uint64_t offset, uint64_t val); 389 390 #endif /* _RTE_ZIP_VF_H_ */ 391