1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2024 Intel Corporation 3 */ 4 5 #include <rte_pci.h> 6 #include <rte_vfio.h> 7 8 #include "qat_device.h" 9 #include "qat_qp.h" 10 #include "adf_transport_access_macros_gen_lcevf.h" 11 #include "adf_pf2vf_msg.h" 12 #include "qat_pf2vf.h" 13 14 #include <stdint.h> 15 #include <sys/ioctl.h> 16 #include <unistd.h> 17 18 #define BITS_PER_ULONG (sizeof(unsigned long) * 8) 19 20 #define VFIO_PCI_LCE_DEVICE_CFG_REGION_INDEX VFIO_PCI_NUM_REGIONS 21 #define VFIO_PCI_LCE_CY_CFG_REGION_INDEX (VFIO_PCI_NUM_REGIONS + 2) 22 #define VFIO_PCI_LCE_RING_CFG_REGION_INDEX (VFIO_PCI_NUM_REGIONS + 4) 23 #define LCE_DEVICE_NAME_SIZE 64 24 #define LCE_DEVICE_MAX_BANKS 2080 25 #define LCE_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 26 #define LCE_DEVICE_BITMAP_SIZE LCE_DIV_ROUND_UP(LCE_DEVICE_MAX_BANKS, BITS_PER_ULONG) 27 28 /* QAT GEN_LCE specific macros */ 29 #define QAT_GEN_LCE_BUNDLE_NUM LCE_DEVICE_MAX_BANKS 30 #define QAT_GEN4_QPS_PER_BUNDLE_NUM 1 31 32 /** 33 * struct lce_vfio_dev_cap - LCE device capabilities 34 * 35 * Device level capabilities and service level capabilities 36 */ 37 struct lce_vfio_dev_cap { 38 uint16_t device_num; 39 uint16_t device_type; 40 uint32_t capability_mask; 41 uint32_t extended_capabilities; 42 uint16_t max_banks; 43 uint16_t max_rings_per_bank; 44 uint16_t arb_mask; 45 uint16_t services; 46 uint16_t pkg_id; 47 uint16_t node_id; 48 uint8_t device_name[LCE_DEVICE_NAME_SIZE]; 49 }; 50 51 /* struct lce_vfio_dev_cy_cap - CY capabilities of LCE device */ 52 struct lce_vfio_dev_cy_cap { 53 uint32_t nr_banks; 54 unsigned long bitmap[LCE_DEVICE_BITMAP_SIZE]; 55 }; 56 57 struct lce_qat_domain { 58 uint32_t nid :3; 59 uint32_t fid :7; 60 uint32_t ftype :2; 61 uint32_t vfid :13; 62 uint32_t rid :4; 63 uint32_t vld :1; 64 uint32_t desc_over :1; 65 uint32_t pasid_vld :1; 66 uint32_t pasid :20; 67 }; 68 69 struct lce_qat_buf_domain { 70 uint32_t bank_id: 20; 71 uint32_t type: 4; 72 uint32_t resv: 8; 73 struct lce_qat_domain dom; 74 }; 75 76 struct qat_dev_gen_lce_extra { 77 struct qat_qp_hw_data 78 qp_gen_lce_data[QAT_GEN_LCE_BUNDLE_NUM][QAT_GEN4_QPS_PER_BUNDLE_NUM]; 79 }; 80 81 static struct qat_pf2vf_dev qat_pf2vf_gen_lce = { 82 .pf2vf_offset = ADF_4XXXIOV_PF2VM_OFFSET, 83 .vf2pf_offset = ADF_4XXXIOV_VM2PF_OFFSET, 84 .pf2vf_type_shift = ADF_PFVF_2X_MSGTYPE_SHIFT, 85 .pf2vf_type_mask = ADF_PFVF_2X_MSGTYPE_MASK, 86 .pf2vf_data_shift = ADF_PFVF_2X_MSGDATA_SHIFT, 87 .pf2vf_data_mask = ADF_PFVF_2X_MSGDATA_MASK, 88 }; 89 90 static int 91 qat_select_valid_queue_gen_lce(struct qat_pci_device *qat_dev, int qp_id, 92 enum qat_service_type service_type) 93 { 94 int i = 0, valid_qps = 0; 95 struct qat_dev_gen_lce_extra *dev_extra = qat_dev->dev_private; 96 97 for (; i < QAT_GEN_LCE_BUNDLE_NUM; i++) { 98 if (dev_extra->qp_gen_lce_data[i][0].service_type == service_type) { 99 if (valid_qps == qp_id) 100 return i; 101 ++valid_qps; 102 } 103 } 104 return -1; 105 } 106 107 static const struct qat_qp_hw_data * 108 qat_qp_get_hw_data_gen_lce(struct qat_pci_device *qat_dev, 109 enum qat_service_type service_type, uint16_t qp_id) 110 { 111 struct qat_dev_gen_lce_extra *dev_extra = qat_dev->dev_private; 112 int ring_pair = qat_select_valid_queue_gen_lce(qat_dev, qp_id, service_type); 113 114 if (ring_pair < 0) 115 return NULL; 116 117 return &dev_extra->qp_gen_lce_data[ring_pair][0]; 118 } 119 120 static int 121 qat_qp_rings_per_service_gen_lce(struct qat_pci_device *qat_dev, 122 enum qat_service_type service) 123 { 124 int i = 0, count = 0, max_ops_per_srv = 0; 125 struct qat_dev_gen_lce_extra *dev_extra = qat_dev->dev_private; 126 127 max_ops_per_srv = QAT_GEN_LCE_BUNDLE_NUM; 128 for (i = 0, count = 0; i < max_ops_per_srv; i++) 129 if (dev_extra->qp_gen_lce_data[i][0].service_type == service) 130 count++; 131 return count; 132 } 133 134 static int qat_dev_read_config_gen_lce(struct qat_pci_device *qat_dev) 135 { 136 struct qat_dev_gen_lce_extra *dev_extra = qat_dev->dev_private; 137 struct qat_qp_hw_data *hw_data; 138 139 /** Enable only crypto ring: RP-0 */ 140 hw_data = &dev_extra->qp_gen_lce_data[0][0]; 141 memset(hw_data, 0, sizeof(*hw_data)); 142 143 hw_data->service_type = QAT_SERVICE_SYMMETRIC; 144 hw_data->tx_msg_size = 128; 145 hw_data->rx_msg_size = 32; 146 147 hw_data->tx_ring_num = 0; 148 hw_data->rx_ring_num = 1; 149 150 hw_data->hw_bundle_num = 0; 151 152 return 0; 153 } 154 155 static void qat_qp_build_ring_base_gen_lce(void *io_addr, struct qat_queue *queue) 156 { 157 uint64_t queue_base; 158 159 queue_base = BUILD_RING_BASE_ADDR_GEN_LCE(queue->base_phys_addr, queue->queue_size); 160 WRITE_CSR_RING_BASE_GEN_LCEVF(io_addr, queue->hw_bundle_number, 161 queue->hw_queue_number, queue_base); 162 } 163 164 static void 165 qat_qp_adf_arb_enable_gen_lce(const struct qat_queue *txq, 166 void *base_addr, rte_spinlock_t *lock) 167 { 168 uint32_t arb_csr_offset = 0, value; 169 170 rte_spinlock_lock(lock); 171 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + 172 (ADF_RING_BUNDLE_SIZE_GEN_LCE * txq->hw_bundle_number); 173 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN_LCEVF, arb_csr_offset); 174 value |= 0x01; 175 ADF_CSR_WR(base_addr, arb_csr_offset, value); 176 rte_spinlock_unlock(lock); 177 } 178 179 static void 180 qat_qp_adf_arb_disable_gen_lce(const struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock) 181 { 182 uint32_t arb_csr_offset = 0, value; 183 184 rte_spinlock_lock(lock); 185 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + 186 (ADF_RING_BUNDLE_SIZE_GEN_LCE * txq->hw_bundle_number); 187 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN_LCEVF, arb_csr_offset); 188 value &= ~(0x01); 189 ADF_CSR_WR(base_addr, arb_csr_offset, value); 190 rte_spinlock_unlock(lock); 191 } 192 193 static void 194 qat_qp_adf_configure_queues_gen_lce(struct qat_qp *qp) 195 { 196 uint32_t q_tx_config, q_resp_config; 197 struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q; 198 199 /* q_tx/rx->queue_size is initialized as per bundle config register */ 200 q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size); 201 202 q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size, 203 ADF_RING_NEAR_WATERMARK_512, 204 ADF_RING_NEAR_WATERMARK_0); 205 206 WRITE_CSR_RING_CONFIG_GEN_LCEVF(qp->mmap_bar_addr, q_tx->hw_bundle_number, 207 q_tx->hw_queue_number, q_tx_config); 208 WRITE_CSR_RING_CONFIG_GEN_LCEVF(qp->mmap_bar_addr, q_rx->hw_bundle_number, 209 q_rx->hw_queue_number, q_resp_config); 210 } 211 212 static void 213 qat_qp_csr_write_tail_gen_lce(struct qat_qp *qp, struct qat_queue *q) 214 { 215 WRITE_CSR_RING_TAIL_GEN_LCEVF(qp->mmap_bar_addr, q->hw_bundle_number, 216 q->hw_queue_number, q->tail); 217 } 218 219 static void 220 qat_qp_csr_write_head_gen_lce(struct qat_qp *qp, struct qat_queue *q, uint32_t new_head) 221 { 222 WRITE_CSR_RING_HEAD_GEN_LCEVF(qp->mmap_bar_addr, q->hw_bundle_number, 223 q->hw_queue_number, new_head); 224 } 225 226 static void 227 qat_qp_csr_setup_gen_lce(struct qat_pci_device *qat_dev, void *io_addr, struct qat_qp *qp) 228 { 229 qat_qp_build_ring_base_gen_lce(io_addr, &qp->tx_q); 230 qat_qp_build_ring_base_gen_lce(io_addr, &qp->rx_q); 231 qat_qp_adf_configure_queues_gen_lce(qp); 232 qat_qp_adf_arb_enable_gen_lce(&qp->tx_q, qp->mmap_bar_addr, &qat_dev->arb_csr_lock); 233 } 234 235 static struct qat_qp_hw_spec_funcs qat_qp_hw_spec_gen_lce = { 236 .qat_qp_rings_per_service = qat_qp_rings_per_service_gen_lce, 237 .qat_qp_build_ring_base = qat_qp_build_ring_base_gen_lce, 238 .qat_qp_adf_arb_enable = qat_qp_adf_arb_enable_gen_lce, 239 .qat_qp_adf_arb_disable = qat_qp_adf_arb_disable_gen_lce, 240 .qat_qp_adf_configure_queues = qat_qp_adf_configure_queues_gen_lce, 241 .qat_qp_csr_write_tail = qat_qp_csr_write_tail_gen_lce, 242 .qat_qp_csr_write_head = qat_qp_csr_write_head_gen_lce, 243 .qat_qp_csr_setup = qat_qp_csr_setup_gen_lce, 244 .qat_qp_get_hw_data = qat_qp_get_hw_data_gen_lce, 245 }; 246 247 static int 248 qat_reset_ring_pairs_gen_lce(struct qat_pci_device *qat_pci_dev __rte_unused) 249 { 250 return 0; 251 } 252 253 static const struct rte_mem_resource* 254 qat_dev_get_transport_bar_gen_lce(struct rte_pci_device *pci_dev) 255 { 256 return &pci_dev->mem_resource[0]; 257 } 258 259 static int 260 qat_dev_get_misc_bar_gen_lce(struct rte_mem_resource **mem_resource, 261 struct rte_pci_device *pci_dev) 262 { 263 *mem_resource = &pci_dev->mem_resource[2]; 264 return 0; 265 } 266 267 static int 268 qat_dev_get_extra_size_gen_lce(void) 269 { 270 return sizeof(struct qat_dev_gen_lce_extra); 271 } 272 273 static int 274 qat_dev_get_slice_map_gen_lce(uint32_t *map __rte_unused, 275 const struct rte_pci_device *pci_dev __rte_unused) 276 { 277 return 0; 278 } 279 280 static struct qat_dev_hw_spec_funcs qat_dev_hw_spec_gen_lce = { 281 .qat_dev_reset_ring_pairs = qat_reset_ring_pairs_gen_lce, 282 .qat_dev_get_transport_bar = qat_dev_get_transport_bar_gen_lce, 283 .qat_dev_get_misc_bar = qat_dev_get_misc_bar_gen_lce, 284 .qat_dev_read_config = qat_dev_read_config_gen_lce, 285 .qat_dev_get_extra_size = qat_dev_get_extra_size_gen_lce, 286 .qat_dev_get_slice_map = qat_dev_get_slice_map_gen_lce, 287 }; 288 289 RTE_INIT(qat_dev_gen_lce_init) 290 { 291 qat_qp_hw_spec[QAT_GEN_LCE] = &qat_qp_hw_spec_gen_lce; 292 qat_dev_hw_spec[QAT_GEN_LCE] = &qat_dev_hw_spec_gen_lce; 293 qat_gen_config[QAT_GEN_LCE].dev_gen = QAT_GEN_LCE; 294 qat_gen_config[QAT_GEN_LCE].pf2vf_dev = &qat_pf2vf_gen_lce; 295 } 296