1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _RTE_OCTEONTX_ZIP_VF_H_ 6 #define _RTE_OCTEONTX_ZIP_VF_H_ 7 8 #include <unistd.h> 9 10 #include <bus_pci_driver.h> 11 #include <rte_comp.h> 12 #include <rte_compressdev.h> 13 #include <rte_compressdev_pmd.h> 14 #include <rte_malloc.h> 15 #include <rte_memory.h> 16 #include <rte_spinlock.h> 17 18 #include <zip_regs.h> 19 20 extern int octtx_zip_logtype_driver; 21 #define RTE_LOGTYPE_OCTTX_ZIP_DRIVER octtx_zip_logtype_driver 22 23 /* ZIP VF Control/Status registers (CSRs): */ 24 /* VF_BAR0: */ 25 #define ZIP_VQ_ENA (0x10) 26 #define ZIP_VQ_SBUF_ADDR (0x20) 27 #define ZIP_VF_PF_MBOXX(x) (0x400 | (x)<<3) 28 #define ZIP_VQ_DOORBELL (0x1000) 29 30 /**< Vendor ID */ 31 #define PCI_VENDOR_ID_CAVIUM 0x177D 32 /**< PCI device id of ZIP VF */ 33 #define PCI_DEVICE_ID_OCTEONTX_ZIPVF 0xA037 34 #define PCI_DEVICE_ID_OCTEONTX2_ZIPVF 0xA083 35 36 /* maximum number of zip vf devices */ 37 #define ZIP_MAX_VFS 8 38 39 /* max size of one chunk */ 40 #define ZIP_MAX_CHUNK_SIZE 8192 41 42 /* each instruction is fixed 128 bytes */ 43 #define ZIP_CMD_SIZE 128 44 45 #define ZIP_CMD_SIZE_WORDS (ZIP_CMD_SIZE >> 3) /* 16 64_bit words */ 46 47 /* size of next chunk buffer pointer */ 48 #define ZIP_MAX_NCBP_SIZE 8 49 50 /* size of instruction queue in units of instruction size */ 51 #define ZIP_MAX_NUM_CMDS ((ZIP_MAX_CHUNK_SIZE - ZIP_MAX_NCBP_SIZE) / \ 52 ZIP_CMD_SIZE) /* 63 */ 53 54 /* size of instruct queue in bytes */ 55 #define ZIP_MAX_CMDQ_SIZE ((ZIP_MAX_NUM_CMDS * ZIP_CMD_SIZE) + \ 56 ZIP_MAX_NCBP_SIZE)/* ~8072ull */ 57 58 #define ZIP_BUF_SIZE 256 59 #define ZIP_SGBUF_SIZE (5 * 1024) 60 #define ZIP_BURST_SIZE 64 61 62 #define ZIP_MAXSEG_SIZE 59460 63 #define ZIP_EXTRABUF_SIZE 4096 64 #define ZIP_MAX_SEGS 300 65 #define ZIP_MAX_DATA_SIZE (16*1024*1024) 66 67 #define ZIP_SGPTR_ALIGN 16 68 #define ZIP_CMDQ_ALIGN 128 69 #define MAX_SG_LEN ((ZIP_BUF_SIZE - ZIP_SGPTR_ALIGN) / sizeof(void *)) 70 71 /**< ZIP PMD specified queue pairs */ 72 #define ZIP_MAX_VF_QUEUE 1 73 74 #define ZIP_ALIGN_ROUNDUP(x, _align) \ 75 ((_align) * (((x) + (_align) - 1) / (_align))) 76 77 /**< ZIP PMD device name */ 78 #define COMPRESSDEV_NAME_ZIP_PMD compress_octeontx 79 80 #define ZIP_PMD_LOG(level, ...) \ 81 RTE_LOG_LINE_PREFIX(level, OCTTX_ZIP_DRIVER, "%s(): ", __func__, __VA_ARGS__) 82 83 #define ZIP_PMD_INFO(fmt, ...) \ 84 ZIP_PMD_LOG(INFO, fmt, ## __VA_ARGS__) 85 #define ZIP_PMD_ERR(fmt, ...) \ 86 ZIP_PMD_LOG(ERR, fmt, ## __VA_ARGS__) 87 88 /* resources required to process stream */ 89 enum NUM_BUFS_PER_STREAM { 90 RES_BUF = 0, 91 CMD_BUF, 92 HASH_CTX_BUF, 93 DECOMP_CTX_BUF, 94 IN_DATA_BUF, 95 OUT_DATA_BUF, 96 HISTORY_DATA_BUF, 97 MAX_BUFS_PER_STREAM 98 }; 99 100 struct zip_stream; 101 struct zipvf_qp; 102 103 /* Algorithm handler function prototype */ 104 typedef int (*comp_func_t)(struct rte_comp_op *op, struct zipvf_qp *qp, 105 struct zip_stream *zstrm, int num); 106 107 /* Scatter gather list */ 108 struct __rte_aligned(16) zipvf_sginfo { 109 union zip_zptr_addr_s sg_addr; 110 union zip_zptr_ctl_s sg_ctl; 111 }; 112 113 /** 114 * ZIP private stream structure 115 */ 116 struct __rte_cache_aligned zip_stream { 117 union zip_inst_s *inst[ZIP_BURST_SIZE]; 118 /* zip instruction pointer */ 119 comp_func_t func; 120 /* function to process comp operation */ 121 void *bufs[MAX_BUFS_PER_STREAM * ZIP_BURST_SIZE]; 122 }; 123 124 125 /** 126 * ZIP instruction Queue 127 */ 128 struct zipvf_cmdq { 129 rte_spinlock_t qlock; 130 /* queue lock */ 131 uint64_t *sw_head; 132 /* pointer to start of 8-byte word length queue-head */ 133 uint8_t *va; 134 /* pointer to instruction queue virtual address */ 135 rte_iova_t iova; 136 /* iova addr of cmdq head*/ 137 }; 138 139 /** 140 * ZIP device queue structure 141 */ 142 struct __rte_cache_aligned zipvf_qp { 143 struct zipvf_cmdq cmdq; 144 /* Hardware instruction queue structure */ 145 struct rte_ring *processed_pkts; 146 /* Ring for placing processed packets */ 147 struct rte_compressdev_stats qp_stats; 148 /* Queue pair statistics */ 149 uint16_t id; 150 /* Queue Pair Identifier */ 151 const char *name; 152 /* Unique Queue Pair Name */ 153 struct zip_vf *vf; 154 /* pointer to device, queue belongs to */ 155 struct zipvf_sginfo *g_info; 156 struct zipvf_sginfo *s_info; 157 /* SGL pointers */ 158 uint64_t num_sgbuf; 159 uint64_t enqed; 160 }; 161 162 /** 163 * ZIP VF device structure. 164 */ 165 struct __rte_cache_aligned zip_vf { 166 int vfid; 167 /* vf index */ 168 struct rte_pci_device *pdev; 169 /* pci device */ 170 void *vbar0; 171 /* CSR base address for underlying BAR0 VF.*/ 172 uint64_t dom_sdom; 173 /* Storing mbox domain and subdomain id for app rerun*/ 174 uint32_t max_nb_queue_pairs; 175 /* pointer to device qps */ 176 struct rte_mempool *zip_mp; 177 struct rte_mempool *sg_mp; 178 /* pointer to pools */ 179 }; 180 181 182 static inline int 183 zipvf_prepare_sgl(struct rte_mbuf *buf, int64_t offset, struct zipvf_sginfo *sg_list, 184 uint32_t data_len, const uint16_t max_segs, struct zipvf_qp *qp) 185 { 186 struct zipvf_sginfo *sginfo = (struct zipvf_sginfo *)sg_list; 187 uint32_t tot_buf_len, sgidx; 188 int ret = -EINVAL; 189 190 for (sgidx = tot_buf_len = 0; buf && sgidx < max_segs; buf = buf->next) { 191 if (offset >= rte_pktmbuf_data_len(buf)) { 192 offset -= rte_pktmbuf_data_len(buf); 193 continue; 194 } 195 196 sginfo[sgidx].sg_ctl.s.length = (uint16_t)(rte_pktmbuf_data_len(buf) - offset); 197 sginfo[sgidx].sg_addr.s.addr = rte_pktmbuf_iova_offset(buf, offset); 198 199 offset = 0; 200 tot_buf_len += sginfo[sgidx].sg_ctl.s.length; 201 202 if (tot_buf_len >= data_len) { 203 sginfo[sgidx].sg_ctl.s.length -= tot_buf_len - data_len; 204 ret = 0; 205 break; 206 } 207 208 ZIP_PMD_LOG(DEBUG, "ZIP SGL buf[%d], len = %d, iova = 0x%"PRIx64, 209 sgidx, sginfo[sgidx].sg_ctl.s.length, sginfo[sgidx].sg_addr.s.addr); 210 ++sgidx; 211 } 212 213 if (unlikely(ret != 0)) { 214 if (sgidx == max_segs) 215 ZIP_PMD_ERR("Exceeded max segments in ZIP SGL (%u)", max_segs); 216 else 217 ZIP_PMD_ERR("Mbuf chain is too short"); 218 } 219 qp->num_sgbuf = ++sgidx; 220 221 ZIP_PMD_LOG(DEBUG, "Tot_buf_len:%d max_segs:%"PRIx64, tot_buf_len, 222 qp->num_sgbuf); 223 return ret; 224 } 225 226 static inline int 227 zipvf_prepare_in_buf(union zip_inst_s *inst, struct zipvf_qp *qp, struct rte_comp_op *op) 228 { 229 uint32_t offset, inlen; 230 struct rte_mbuf *m_src; 231 int ret = 0; 232 233 inlen = op->src.length; 234 offset = op->src.offset; 235 m_src = op->m_src; 236 237 /* Gather input */ 238 if (op->m_src->next != NULL && inlen > ZIP_MAXSEG_SIZE) { 239 inst->s.dg = 1; 240 241 ret = zipvf_prepare_sgl(m_src, offset, qp->g_info, inlen, 242 op->m_src->nb_segs, qp); 243 244 inst->s.inp_ptr_addr.s.addr = rte_mem_virt2iova(qp->g_info); 245 inst->s.inp_ptr_ctl.s.length = qp->num_sgbuf; 246 inst->s.inp_ptr_ctl.s.fw = 0; 247 248 ZIP_PMD_LOG(DEBUG, "Gather(input): len(nb_segs):%d, iova: 0x%"PRIx64, 249 inst->s.inp_ptr_ctl.s.length, inst->s.inp_ptr_addr.s.addr); 250 return ret; 251 } 252 253 /* Prepare direct input data pointer */ 254 inst->s.dg = 0; 255 inst->s.inp_ptr_addr.s.addr = rte_pktmbuf_iova_offset(m_src, offset); 256 inst->s.inp_ptr_ctl.s.length = inlen; 257 258 ZIP_PMD_LOG(DEBUG, "Direct input - inlen:%d", inlen); 259 return ret; 260 } 261 262 static inline int 263 zipvf_prepare_out_buf(union zip_inst_s *inst, struct zipvf_qp *qp, struct rte_comp_op *op) 264 { 265 uint32_t offset, outlen; 266 struct rte_mbuf *m_dst; 267 int ret = 0; 268 269 offset = op->dst.offset; 270 m_dst = op->m_dst; 271 outlen = rte_pktmbuf_pkt_len(m_dst) - op->dst.offset; 272 273 /* Scatter output */ 274 if (op->m_dst->next != NULL && outlen > ZIP_MAXSEG_SIZE) { 275 inst->s.ds = 1; 276 inst->s.totaloutputlength = outlen; 277 278 ret = zipvf_prepare_sgl(m_dst, offset, qp->s_info, inst->s.totaloutputlength, 279 m_dst->nb_segs, qp); 280 281 inst->s.out_ptr_addr.s.addr = rte_mem_virt2iova(qp->s_info); 282 inst->s.out_ptr_ctl.s.length = qp->num_sgbuf; 283 284 ZIP_PMD_LOG(DEBUG, "Scatter(output): nb_segs:%d, iova:0x%"PRIx64, 285 inst->s.out_ptr_ctl.s.length, inst->s.out_ptr_addr.s.addr); 286 return ret; 287 } 288 289 /* Prepare direct output data pointer */ 290 inst->s.ds = 0; 291 inst->s.out_ptr_addr.s.addr = rte_pktmbuf_iova_offset(m_dst, offset); 292 inst->s.totaloutputlength = rte_pktmbuf_pkt_len(m_dst) - op->dst.offset; 293 if (inst->s.totaloutputlength == ZIP_MAXSEG_SIZE) 294 inst->s.totaloutputlength += ZIP_EXTRABUF_SIZE; /* DSTOP */ 295 296 inst->s.out_ptr_ctl.s.length = inst->s.totaloutputlength; 297 298 ZIP_PMD_LOG(DEBUG, "Direct output - outlen:%d", inst->s.totaloutputlength); 299 return ret; 300 } 301 302 static inline int 303 zipvf_prepare_cmd_stateless(struct rte_comp_op *op, struct zipvf_qp *qp, 304 union zip_inst_s *inst) 305 { 306 /* set flush flag to always 1*/ 307 inst->s.ef = 1; 308 309 if (inst->s.op == ZIP_OP_E_DECOMP) 310 inst->s.sf = 1; 311 else 312 inst->s.sf = 0; 313 314 /* Set input checksum */ 315 inst->s.adlercrc32 = op->input_chksum; 316 317 /* Prepare input/output buffers */ 318 if (zipvf_prepare_in_buf(inst, qp, op)) { 319 ZIP_PMD_ERR("Con't fill input SGL "); 320 return -EINVAL; 321 } 322 323 if (zipvf_prepare_out_buf(inst, qp, op)) { 324 ZIP_PMD_ERR("Con't fill output SGL "); 325 return -EINVAL; 326 } 327 328 return 0; 329 } 330 331 #ifdef ZIP_DBG 332 static inline void 333 zip_dump_instruction(void *inst) 334 { 335 union zip_inst_s *cmd83 = (union zip_inst_s *)inst; 336 337 printf("####### START ########\n"); 338 printf("ZIP Instr:0x%"PRIx64"\n", cmd83); 339 printf("doneint:%d totaloutputlength:%d\n", cmd83->s.doneint, 340 cmd83->s.totaloutputlength); 341 printf("exnum:%d iv:%d exbits:%d hmif:%d halg:%d\n", cmd83->s.exn, 342 cmd83->s.iv, cmd83->s.exbits, cmd83->s.hmif, cmd83->s.halg); 343 printf("flush:%d speed:%d cc:%d\n", cmd83->s.sf, 344 cmd83->s.ss, cmd83->s.cc); 345 printf("eof:%d bof:%d op:%d dscatter:%d dgather:%d hgather:%d\n", 346 cmd83->s.ef, cmd83->s.bf, cmd83->s.op, cmd83->s.ds, 347 cmd83->s.dg, cmd83->s.hg); 348 printf("historylength:%d adler32:%d\n", cmd83->s.historylength, 349 cmd83->s.adlercrc32); 350 printf("ctx_ptr.addr:0x%"PRIx64"\n", cmd83->s.ctx_ptr_addr.s.addr); 351 printf("ctx_ptr.len:%d\n", cmd83->s.ctx_ptr_ctl.s.length); 352 printf("history_ptr.addr:0x%"PRIx64"\n", cmd83->s.his_ptr_addr.s.addr); 353 printf("history_ptr.len:%d\n", cmd83->s.his_ptr_ctl.s.length); 354 printf("inp_ptr.addr:0x%"PRIx64"\n", cmd83->s.inp_ptr_addr.s.addr); 355 printf("inp_ptr.len:%d\n", cmd83->s.inp_ptr_ctl.s.length); 356 printf("out_ptr.addr:0x%"PRIx64"\n", cmd83->s.out_ptr_addr.s.addr); 357 printf("out_ptr.len:%d\n", cmd83->s.out_ptr_ctl.s.length); 358 printf("result_ptr.addr:0x%"PRIx64"\n", cmd83->s.res_ptr_addr.s.addr); 359 printf("result_ptr.len:%d\n", cmd83->s.res_ptr_ctl.s.length); 360 printf("####### END ########\n"); 361 } 362 #endif 363 364 int 365 zipvf_create(struct rte_compressdev *compressdev); 366 367 int 368 zipvf_destroy(struct rte_compressdev *compressdev); 369 370 int 371 zipvf_q_init(struct zipvf_qp *qp); 372 373 int 374 zipvf_q_term(struct zipvf_qp *qp); 375 376 void 377 zipvf_push_command(struct zipvf_qp *qp, union zip_inst_s *zcmd); 378 379 int 380 zip_process_op(struct rte_comp_op *op, struct zipvf_qp *qp, 381 struct zip_stream *zstrm, int num); 382 383 uint64_t 384 zip_reg_read64(uint8_t *hw_addr, uint64_t offset); 385 386 void 387 zip_reg_write64(uint8_t *hw_addr, uint64_t offset, uint64_t val); 388 389 #endif /* _RTE_ZIP_VF_H_ */ 390