1 /* 2 * SPDX-License-Identifier: BSD-3-Clause 3 * Copyright(c) 2023 Napatech A/S 4 */ 5 6 #include "rte_spinlock.h" 7 #include "nt_util.h" 8 #include "ntlog.h" 9 10 #include "nthw_drv.h" 11 #include "nthw_register.h" 12 #include "nthw_rac.h" 13 14 #define RAB_DMA_WAIT (1000000) 15 16 #define RAB_READ (0x01) 17 #define RAB_WRITE (0x02) 18 #define RAB_ECHO (0x08) 19 #define RAB_COMPLETION (0x0F) 20 21 #define RAB_OPR_LO (28) 22 23 #define RAB_CNT_LO (20) 24 #define RAB_CNT_BW (8) 25 26 #define RAB_BUSID_LO (16) 27 #define RAB_BUSID_BW (4) 28 29 #define RAB_ADDR_BW (16) 30 31 nthw_rac_t *nthw_rac_new(void) 32 { 33 nthw_rac_t *p = malloc(sizeof(nthw_rac_t)); 34 memset(p, 0, sizeof(nthw_rac_t)); 35 return p; 36 } 37 38 int nthw_rac_init(nthw_rac_t *p, nthw_fpga_t *p_fpga, struct fpga_info_s *p_fpga_info) 39 { 40 assert(p_fpga_info); 41 42 const char *const p_adapter_id_str = p_fpga_info->mp_adapter_id_str; 43 nthw_module_t *p_mod = nthw_fpga_query_module(p_fpga, MOD_RAC, 0); 44 45 if (p == NULL) 46 return p_mod == NULL ? -1 : 0; 47 48 if (p_mod == NULL) { 49 NT_LOG(ERR, NTHW, "%s: RAC %d: no such instance", p_adapter_id_str, 0); 50 return -1; 51 } 52 53 p->mp_fpga = p_fpga; 54 p->mp_mod_rac = p_mod; 55 56 p->mn_param_rac_rab_interfaces = 57 nthw_fpga_get_product_param(p->mp_fpga, NT_RAC_RAB_INTERFACES, 3); 58 NT_LOG(DBG, NTHW, "%s: NT_RAC_RAB_INTERFACES=%d", p_adapter_id_str, 59 p->mn_param_rac_rab_interfaces); 60 61 p->mn_param_rac_rab_ob_update = 62 nthw_fpga_get_product_param(p->mp_fpga, NT_RAC_RAB_OB_UPDATE, 0); 63 NT_LOG(DBG, NTHW, "%s: NT_RAC_RAB_OB_UPDATE=%d", p_adapter_id_str, 64 p->mn_param_rac_rab_ob_update); 65 66 /* Optional dummy test registers */ 67 p->mp_reg_dummy0 = nthw_module_query_register(p->mp_mod_rac, RAC_DUMMY0); 68 p->mp_reg_dummy1 = nthw_module_query_register(p->mp_mod_rac, RAC_DUMMY1); 69 p->mp_reg_dummy2 = nthw_module_query_register(p->mp_mod_rac, RAC_DUMMY2); 70 71 p->mp_reg_rab_init = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_INIT); 72 p->mp_fld_rab_init = nthw_register_get_field(p->mp_reg_rab_init, RAC_RAB_INIT_RAB); 73 p->mn_fld_rab_init_bw = nthw_field_get_bit_width(p->mp_fld_rab_init); 74 p->mn_fld_rab_init_mask = nthw_field_get_mask(p->mp_fld_rab_init); 75 76 /* RAC_RAB_INIT_RAB reg/field sanity checks: */ 77 assert(p->mn_fld_rab_init_mask == ((1UL << p->mn_fld_rab_init_bw) - 1)); 78 assert(p->mn_fld_rab_init_bw == p->mn_param_rac_rab_interfaces); 79 80 p->mp_reg_dbg_ctrl = nthw_module_query_register(p->mp_mod_rac, RAC_DBG_CTRL); 81 82 if (p->mp_reg_dbg_ctrl) 83 p->mp_fld_dbg_ctrl = nthw_register_query_field(p->mp_reg_dbg_ctrl, RAC_DBG_CTRL_C); 84 85 else 86 p->mp_fld_dbg_ctrl = NULL; 87 88 p->mp_reg_dbg_data = nthw_module_query_register(p->mp_mod_rac, RAC_DBG_DATA); 89 90 if (p->mp_reg_dbg_data) 91 p->mp_fld_dbg_data = nthw_register_query_field(p->mp_reg_dbg_data, RAC_DBG_DATA_D); 92 93 else 94 p->mp_reg_dbg_data = NULL; 95 96 p->mp_reg_rab_ib_data = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_IB_DATA); 97 p->mp_fld_rab_ib_data = nthw_register_get_field(p->mp_reg_rab_ib_data, RAC_RAB_IB_DATA_D); 98 99 p->mp_reg_rab_ob_data = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_OB_DATA); 100 p->mp_fld_rab_ob_data = nthw_register_get_field(p->mp_reg_rab_ob_data, RAC_RAB_OB_DATA_D); 101 102 p->mp_reg_rab_buf_free = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_BUF_FREE); 103 p->mp_fld_rab_buf_free_ib_free = 104 nthw_register_get_field(p->mp_reg_rab_buf_free, RAC_RAB_BUF_FREE_IB_FREE); 105 p->mp_fld_rab_buf_free_ib_ovf = 106 nthw_register_get_field(p->mp_reg_rab_buf_free, RAC_RAB_BUF_FREE_IB_OVF); 107 p->mp_fld_rab_buf_free_ob_free = 108 nthw_register_get_field(p->mp_reg_rab_buf_free, RAC_RAB_BUF_FREE_OB_FREE); 109 p->mp_fld_rab_buf_free_ob_ovf = 110 nthw_register_get_field(p->mp_reg_rab_buf_free, RAC_RAB_BUF_FREE_OB_OVF); 111 p->mp_fld_rab_buf_free_timeout = 112 nthw_register_get_field(p->mp_reg_rab_buf_free, RAC_RAB_BUF_FREE_TIMEOUT); 113 114 p->mp_reg_rab_buf_used = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_BUF_USED); 115 p->mp_fld_rab_buf_used_ib_used = 116 nthw_register_get_field(p->mp_reg_rab_buf_used, RAC_RAB_BUF_USED_IB_USED); 117 p->mp_fld_rab_buf_used_ob_used = 118 nthw_register_get_field(p->mp_reg_rab_buf_used, RAC_RAB_BUF_USED_OB_USED); 119 p->mp_fld_rab_buf_used_flush = 120 nthw_register_get_field(p->mp_reg_rab_buf_used, RAC_RAB_BUF_USED_FLUSH); 121 122 /* 123 * RAC_RAB_DMA regs are optional - only found in real 124 * NT4GA - not found in 9231/9232 and earlier 125 */ 126 p->mp_reg_rab_dma_ib_lo = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_DMA_IB_LO); 127 p->mp_fld_rab_dma_ib_lo_phy_addr = 128 nthw_register_get_field(p->mp_reg_rab_dma_ib_lo, RAC_RAB_DMA_IB_LO_PHYADDR); 129 130 p->mp_reg_rab_dma_ib_hi = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_DMA_IB_HI); 131 p->mp_fld_rab_dma_ib_hi_phy_addr = 132 nthw_register_get_field(p->mp_reg_rab_dma_ib_hi, RAC_RAB_DMA_IB_HI_PHYADDR); 133 134 p->mp_reg_rab_dma_ob_lo = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_DMA_OB_LO); 135 p->mp_fld_rab_dma_ob_lo_phy_addr = 136 nthw_register_get_field(p->mp_reg_rab_dma_ob_lo, RAC_RAB_DMA_OB_LO_PHYADDR); 137 138 p->mp_reg_rab_dma_ob_hi = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_DMA_OB_HI); 139 p->mp_fld_rab_dma_ob_hi_phy_addr = 140 nthw_register_get_field(p->mp_reg_rab_dma_ob_hi, RAC_RAB_DMA_OB_HI_PHYADDR); 141 142 p->mp_reg_rab_dma_ib_wr = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_DMA_IB_WR); 143 p->mp_fld_rab_dma_ib_wr_ptr = 144 nthw_register_get_field(p->mp_reg_rab_dma_ib_wr, RAC_RAB_DMA_IB_WR_PTR); 145 146 p->mp_reg_rab_dma_ib_rd = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_DMA_IB_RD); 147 p->mp_fld_rab_dma_ib_rd_ptr = 148 nthw_register_get_field(p->mp_reg_rab_dma_ib_rd, RAC_RAB_DMA_IB_RD_PTR); 149 150 p->mp_reg_rab_dma_ob_wr = nthw_module_get_register(p->mp_mod_rac, RAC_RAB_DMA_OB_WR); 151 p->mp_fld_rab_dma_ob_wr_ptr = 152 nthw_register_get_field(p->mp_reg_rab_dma_ob_wr, RAC_RAB_DMA_OB_WR_PTR); 153 154 p->RAC_RAB_INIT_ADDR = nthw_register_get_address(p->mp_reg_rab_init); 155 p->RAC_RAB_IB_DATA_ADDR = nthw_register_get_address(p->mp_reg_rab_ib_data); 156 p->RAC_RAB_OB_DATA_ADDR = nthw_register_get_address(p->mp_reg_rab_ob_data); 157 p->RAC_RAB_BUF_FREE_ADDR = nthw_register_get_address(p->mp_reg_rab_buf_free); 158 p->RAC_RAB_BUF_USED_ADDR = nthw_register_get_address(p->mp_reg_rab_buf_used); 159 160 /* 161 * RAC_RAB_DMA regs are optional - only found in real NT4GA - not found in 9231/9232 and 162 * earlier 163 */ 164 165 p->RAC_RAB_DMA_IB_LO_ADDR = nthw_register_get_address(p->mp_reg_rab_dma_ib_lo); 166 p->RAC_RAB_DMA_IB_HI_ADDR = nthw_register_get_address(p->mp_reg_rab_dma_ib_hi); 167 p->RAC_RAB_DMA_OB_LO_ADDR = nthw_register_get_address(p->mp_reg_rab_dma_ob_lo); 168 p->RAC_RAB_DMA_OB_HI_ADDR = nthw_register_get_address(p->mp_reg_rab_dma_ob_hi); 169 p->RAC_RAB_DMA_IB_RD_ADDR = nthw_register_get_address(p->mp_reg_rab_dma_ib_rd); 170 p->RAC_RAB_DMA_OB_WR_ADDR = nthw_register_get_address(p->mp_reg_rab_dma_ob_wr); 171 p->RAC_RAB_DMA_IB_WR_ADDR = nthw_register_get_address(p->mp_reg_rab_dma_ib_wr); 172 173 p->RAC_RAB_BUF_FREE_IB_FREE_MASK = nthw_field_get_mask(p->mp_fld_rab_buf_free_ib_free); 174 p->RAC_RAB_BUF_FREE_OB_FREE_MASK = nthw_field_get_mask(p->mp_fld_rab_buf_free_ob_free); 175 p->RAC_RAB_BUF_USED_IB_USED_MASK = nthw_field_get_mask(p->mp_fld_rab_buf_used_ib_used); 176 p->RAC_RAB_BUF_USED_OB_USED_MASK = nthw_field_get_mask(p->mp_fld_rab_buf_used_ob_used); 177 178 p->RAC_RAB_BUF_USED_FLUSH_MASK = nthw_field_get_mask(p->mp_fld_rab_buf_used_flush); 179 180 p->RAC_RAB_BUF_USED_OB_USED_LOW = 181 nthw_field_get_bit_pos_low(p->mp_fld_rab_buf_used_ob_used); 182 183 p->mp_reg_rab_nmb_rd = nthw_module_query_register(p->mp_mod_rac, RAC_NMB_RD_ADR); 184 185 if (p->mp_reg_rab_nmb_rd) 186 p->RAC_NMB_RD_ADR_ADDR = nthw_register_get_address(p->mp_reg_rab_nmb_rd); 187 188 p->mp_reg_rab_nmb_data = nthw_module_query_register(p->mp_mod_rac, RAC_NMB_DATA); 189 190 if (p->mp_reg_rab_nmb_data) 191 p->RAC_NMB_DATA_ADDR = nthw_register_get_address(p->mp_reg_rab_nmb_data); 192 193 p->mp_reg_rab_nmb_wr = nthw_module_query_register(p->mp_mod_rac, RAC_NMB_WR_ADR); 194 195 if (p->mp_reg_rab_nmb_wr) 196 p->RAC_NMB_WR_ADR_ADDR = nthw_register_get_address(p->mp_reg_rab_nmb_wr); 197 198 p->mp_reg_rab_nmb_status = nthw_module_query_register(p->mp_mod_rac, RAC_NMB_STATUS); 199 200 if (p->mp_reg_rab_nmb_status) 201 p->RAC_NMB_STATUS_ADDR = nthw_register_get_address(p->mp_reg_rab_nmb_status); 202 203 p->m_dma = NULL; 204 205 { 206 /* 207 * RAC is a primary communication channel - debug will be messy 208 * turn off debug by default - except for rac_rab_init 209 * NOTE: currently debug will not work - due to optimizations 210 */ 211 const int n_debug_mode = nthw_module_get_debug_mode(p->mp_mod_rac); 212 213 if (n_debug_mode && n_debug_mode <= 0xff) { 214 nthw_module_set_debug_mode(p->mp_mod_rac, 0); 215 nthw_register_set_debug_mode(p->mp_reg_rab_init, n_debug_mode); 216 } 217 } 218 219 rte_spinlock_init(&p->m_mutex); 220 221 return 0; 222 } 223 224 static int nthw_rac_get_rab_interface_count(const nthw_rac_t *p) 225 { 226 return p->mn_param_rac_rab_interfaces; 227 } 228 229 /* private function for internal RAC operations - 230 * improves log flexibility and prevents log flooding 231 */ 232 static void nthw_rac_reg_read32(const struct fpga_info_s *p_fpga_info, uint32_t reg_addr, 233 uint32_t *p_data) 234 { 235 *p_data = *(volatile uint32_t *)((uint8_t *)p_fpga_info->bar0_addr + reg_addr); 236 } 237 238 /* private function for internal RAC operations - 239 * improves log flexibility and prevents log flooding 240 */ 241 static void nthw_rac_reg_write32(const struct fpga_info_s *p_fpga_info, uint32_t reg_addr, 242 uint32_t n_data) 243 { 244 *(volatile uint32_t *)((uint8_t *)p_fpga_info->bar0_addr + reg_addr) = n_data; 245 } 246 247 static inline int _nthw_rac_wait_for_rab_done(const nthw_rac_t *p, uint32_t address, 248 uint32_t word_cnt) 249 { 250 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 251 const char *const p_adapter_id_str = p_fpga_info->mp_adapter_id_str; 252 uint32_t used = 0; 253 uint32_t retry; 254 255 for (retry = 0; retry < 100000; retry++) { 256 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_BUF_USED_ADDR, &used); 257 used = (used & p->RAC_RAB_BUF_USED_OB_USED_MASK) >> 258 p->RAC_RAB_BUF_USED_OB_USED_LOW; 259 260 if (used >= word_cnt) 261 break; 262 } 263 264 if (used < word_cnt) { 265 NT_LOG(ERR, NTHW, "%s: Fail rab bus r/w addr=0x%08X used=%x wordcount=%d", 266 p_adapter_id_str, address, used, word_cnt); 267 return -1; 268 } 269 270 return 0; 271 } 272 273 /* 274 * NT_PCI_REG_P9xyz_RAC_RAB_INIT 275 * 276 * Initializes (resets) the programmable registers on the Register Access Buses (RAB). 277 * This initialization must be performed by software as part of the driver load procedure. 278 * 279 * Bit n of this field initializes the programmable registers on RAB interface n. 280 * Software must write one to the bit and then clear the bit again. 281 * 282 * All RAB module registers will be reset to their defaults. 283 * This includes the product specific RESET module (eg RST9xyz) 284 * As a consequence of this behavior the official reset sequence 285 * must be excersised - as all RAB modules will be held in reset. 286 */ 287 int nthw_rac_rab_init(nthw_rac_t *p, uint32_t n_rab_intf_mask) 288 { 289 /* 290 * Write rac_rab_init 291 * Perform operation twice - first to get trace of operation - 292 * second to get things done... 293 */ 294 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 295 nthw_field_set_val_flush32(p->mp_fld_rab_init, n_rab_intf_mask); 296 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_INIT_ADDR, n_rab_intf_mask); 297 return 0; 298 } 299 300 int nthw_rac_rab_reset(nthw_rac_t *p) 301 { 302 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 303 const char *const p_adapter_id_str = p_fpga_info->mp_adapter_id_str; 304 (void)p_adapter_id_str; 305 306 /* RAC RAB bus "flip/flip" reset */ 307 const int n_rac_rab_bus_count = nthw_rac_get_rab_interface_count(p); 308 const int n_rac_rab_bus_mask = (1 << n_rac_rab_bus_count) - 1; 309 310 NT_LOG(DBG, NTHW, "%s: NT_RAC_RAB_INTERFACES=%d (0x%02X)", p_adapter_id_str, 311 n_rac_rab_bus_count, n_rac_rab_bus_mask); 312 assert(n_rac_rab_bus_count); 313 assert(n_rac_rab_bus_mask); 314 315 /* RAC RAB bus "flip/flip" reset first stage - new impl (ref RMT#37020) */ 316 nthw_rac_rab_init(p, 0); 317 nthw_rac_rab_init(p, n_rac_rab_bus_mask); 318 nthw_rac_rab_init(p, n_rac_rab_bus_mask & ~0x01); 319 320 return 0; 321 } 322 323 int nthw_rac_rab_setup(nthw_rac_t *p) 324 { 325 int rc = 0; 326 327 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 328 uint32_t n_dma_buf_size = 2L * RAB_DMA_BUF_CNT * sizeof(uint32_t); 329 const size_t align_size = nt_util_align_size(n_dma_buf_size); 330 int numa_node = p_fpga_info->numa_node; 331 uint64_t dma_addr; 332 uint32_t buf; 333 334 if (!p->m_dma) { 335 struct nt_dma_s *vfio_dma; 336 /* FPGA needs Page alignment (4K) */ 337 vfio_dma = nt_dma_alloc(align_size, 0x1000, numa_node); 338 339 if (vfio_dma == NULL) { 340 NT_LOG(ERR, NTNIC, "nt_dma_alloc failed"); 341 return -1; 342 } 343 344 p->m_dma_in_buf = (uint32_t *)vfio_dma->addr; 345 p->m_dma_out_buf = p->m_dma_in_buf + RAB_DMA_BUF_CNT; 346 p->m_dma = vfio_dma; 347 } 348 349 /* Setup DMA on the adapter */ 350 dma_addr = p->m_dma->iova; 351 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_DMA_IB_LO_ADDR, dma_addr & 0xffffffff); 352 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_DMA_IB_HI_ADDR, 353 (uint32_t)(dma_addr >> 32) & 0xffffffff); 354 dma_addr += RAB_DMA_BUF_CNT * sizeof(uint32_t); 355 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_DMA_OB_LO_ADDR, dma_addr & 0xffffffff); 356 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_DMA_OB_HI_ADDR, 357 (uint32_t)(dma_addr >> 32) & 0xffffffff); 358 359 /* Set initial value of internal pointers */ 360 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_DMA_IB_RD_ADDR, &buf); 361 p->m_dma_in_ptr_wr = (uint16_t)(buf / sizeof(uint32_t)); 362 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_DMA_OB_WR_ADDR, &buf); 363 p->m_dma_out_ptr_rd = (uint16_t)(buf / sizeof(uint32_t)); 364 p->m_in_free = RAB_DMA_BUF_CNT; 365 366 return rc; 367 } 368 369 void nthw_rac_bar0_read32(const struct fpga_info_s *p_fpga_info, uint32_t reg_addr, 370 uint32_t word_cnt, uint32_t *p_data) 371 { 372 volatile const uint32_t *const src_addr = 373 (uint32_t *)((uint8_t *)p_fpga_info->bar0_addr + reg_addr); 374 375 for (uint32_t i = 0; i < word_cnt; i++) 376 p_data[i] = src_addr[i]; 377 } 378 379 void nthw_rac_bar0_write32(const struct fpga_info_s *p_fpga_info, uint32_t reg_addr, 380 uint32_t word_cnt, const uint32_t *p_data) 381 { 382 volatile uint32_t *const dst_addr = 383 (uint32_t *)((uint8_t *)p_fpga_info->bar0_addr + reg_addr); 384 385 for (uint32_t i = 0; i < word_cnt; i++) 386 dst_addr[i] = p_data[i]; 387 } 388 389 int nthw_rac_rab_dma_begin(nthw_rac_t *p) 390 { 391 p->m_dma_active = true; 392 393 return 0; 394 } 395 396 static void nthw_rac_rab_dma_activate(nthw_rac_t *p) 397 { 398 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 399 const uint32_t completion = RAB_COMPLETION << RAB_OPR_LO; 400 401 /* Write completion word */ 402 p->m_dma_in_buf[p->m_dma_in_ptr_wr] = completion; 403 p->m_dma_in_ptr_wr = (uint16_t)((p->m_dma_in_ptr_wr + 1) & (RAB_DMA_BUF_CNT - 1)); 404 405 /* Clear output completion word */ 406 p->m_dma_out_buf[p->m_dma_out_ptr_rd] = 0; 407 408 /* Update DMA pointer and start transfer */ 409 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_DMA_IB_WR_ADDR, 410 (uint32_t)(p->m_dma_in_ptr_wr * sizeof(uint32_t))); 411 } 412 413 static int nthw_rac_rab_dma_wait(nthw_rac_t *p) 414 { 415 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 416 const char *const p_adapter_id_str = p_fpga_info->mp_adapter_id_str; 417 const uint32_t completion = RAB_COMPLETION << RAB_OPR_LO; 418 uint32_t i; 419 420 for (i = 0; i < RAB_DMA_WAIT; i++) { 421 nt_os_wait_usec_poll(1); 422 423 if ((p->m_dma_out_buf[p->m_dma_out_ptr_rd] & completion) == completion) 424 break; 425 } 426 427 if (i == RAB_DMA_WAIT) { 428 NT_LOG(ERR, NTHW, "%s: RAB: Unexpected value of completion (0x%08X)", 429 p_adapter_id_str, p->m_dma_out_buf[p->m_dma_out_ptr_rd]); 430 return -1; 431 } 432 433 p->m_dma_out_ptr_rd = (uint16_t)((p->m_dma_out_ptr_rd + 1) & (RAB_DMA_BUF_CNT - 1)); 434 p->m_in_free = RAB_DMA_BUF_CNT; 435 436 return 0; 437 } 438 439 int nthw_rac_rab_dma_commit(nthw_rac_t *p) 440 { 441 int ret; 442 443 nthw_rac_rab_dma_activate(p); 444 ret = nthw_rac_rab_dma_wait(p); 445 446 p->m_dma_active = false; 447 448 return ret; 449 } 450 451 uint32_t nthw_rac_rab_get_free(nthw_rac_t *p) 452 { 453 if (!p->m_dma_active) { 454 /* Expecting mutex not to be locked! */ 455 assert(0); /* alert developer that something is wrong */ 456 return -1; 457 } 458 459 return p->m_in_free; 460 } 461 462 int nthw_rac_rab_write32_dma(nthw_rac_t *p, nthw_rab_bus_id_t bus_id, uint32_t address, 463 uint32_t word_cnt, const uint32_t *p_data) 464 { 465 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 466 const char *const p_adapter_id_str = p_fpga_info->mp_adapter_id_str; 467 468 if (word_cnt == 0 || word_cnt > 256) { 469 NT_LOG(ERR, NTHW, 470 "%s: Failed rab dma write length check - bus: %d addr: 0x%08X wordcount: %d - inBufFree: 0x%08X", 471 p_adapter_id_str, bus_id, address, word_cnt, p->m_in_free); 472 assert(0); /* alert developer that something is wrong */ 473 return -1; 474 } 475 476 if (p->m_in_free < (word_cnt + 3)) { 477 /* 478 * No more memory available. 479 * nthw_rac_rab_dma_commit() needs to be called to start and finish pending 480 * transfers. 481 */ 482 return -1; 483 } 484 485 p->m_in_free -= (word_cnt + 1); 486 487 /* Write the command word */ 488 p->m_dma_in_buf[p->m_dma_in_ptr_wr] = (RAB_WRITE << RAB_OPR_LO) | 489 ((word_cnt & ((1 << RAB_CNT_BW) - 1)) << RAB_CNT_LO) | (bus_id << RAB_BUSID_LO) | 490 address; 491 p->m_dma_in_ptr_wr = (uint16_t)((p->m_dma_in_ptr_wr + 1) & (RAB_DMA_BUF_CNT - 1)); 492 493 for (uint32_t i = 0; i < word_cnt; i++) { 494 p->m_dma_in_buf[p->m_dma_in_ptr_wr] = p_data[i]; 495 p->m_dma_in_ptr_wr = (uint16_t)((p->m_dma_in_ptr_wr + 1) & (RAB_DMA_BUF_CNT - 1)); 496 } 497 498 return 0; 499 } 500 501 int nthw_rac_rab_read32_dma(nthw_rac_t *p, nthw_rab_bus_id_t bus_id, uint32_t address, 502 uint32_t word_cnt, struct dma_buf_ptr *buf_ptr) 503 { 504 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 505 const char *const p_adapter_id_str = p_fpga_info->mp_adapter_id_str; 506 507 if (word_cnt == 0 || word_cnt > 256) { 508 NT_LOG(ERR, NTHW, 509 "%s: Failed rab dma read length check - bus: %d addr: 0x%08X wordcount: %d - inBufFree: 0x%08X", 510 p_adapter_id_str, bus_id, address, word_cnt, p->m_in_free); 511 assert(0); /* alert developer that something is wrong */ 512 return -1; 513 } 514 515 if ((word_cnt + 3) > RAB_DMA_BUF_CNT) { 516 NT_LOG(ERR, NTHW, 517 "%s: Failed rab dma read length check - bus: %d addr: 0x%08X wordcount: %d", 518 p_adapter_id_str, bus_id, address, word_cnt); 519 return -1; 520 } 521 522 if (p->m_in_free < 3) { 523 /* 524 * No more memory available. 525 * nthw_rac_rab_dma_commit() needs to be called to start and finish pending 526 * transfers. 527 */ 528 return -1; 529 } 530 531 p->m_in_free -= 1; 532 533 /* Write the command word */ 534 p->m_dma_in_buf[p->m_dma_in_ptr_wr] = (RAB_READ << RAB_OPR_LO) | 535 ((word_cnt & ((1 << RAB_CNT_BW) - 1)) << RAB_CNT_LO) | (bus_id << RAB_BUSID_LO) | 536 address; 537 p->m_dma_in_ptr_wr = (uint16_t)((p->m_dma_in_ptr_wr + 1) & (RAB_DMA_BUF_CNT - 1)); 538 539 buf_ptr->index = p->m_dma_out_ptr_rd; 540 buf_ptr->size = RAB_DMA_BUF_CNT; 541 buf_ptr->base = p->m_dma_out_buf; 542 p->m_dma_out_ptr_rd = 543 (uint16_t)((p->m_dma_out_ptr_rd + word_cnt) & (RAB_DMA_BUF_CNT - 1U)); 544 545 return 0; 546 } 547 548 int nthw_rac_rab_write32(nthw_rac_t *p, bool trc, nthw_rab_bus_id_t bus_id, uint32_t address, 549 uint32_t word_cnt, const uint32_t *p_data) 550 { 551 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 552 const char *const p_adapter_id_str = p_fpga_info->mp_adapter_id_str; 553 uint32_t buf_used; 554 uint32_t buf_free; 555 uint32_t in_buf_free; 556 uint32_t out_buf_free; 557 int res = 0; 558 559 if (address > (1 << RAB_ADDR_BW)) { 560 NT_LOG(ERR, NTHW, "%s: RAB: Illegal address: value too large %d - max %d", 561 p_adapter_id_str, address, (1 << RAB_ADDR_BW)); 562 return -1; 563 } 564 565 if (bus_id > (1 << RAB_BUSID_BW)) { 566 NT_LOG(ERR, NTHW, "%s: RAB: Illegal bus id: value too large %d - max %d", 567 p_adapter_id_str, bus_id, (1 << RAB_BUSID_BW)); 568 return -1; 569 } 570 571 if (word_cnt == 0) { 572 NT_LOG(ERR, NTHW, "%s: RAB: Illegal word count: value is zero (%d)", 573 p_adapter_id_str, word_cnt); 574 return -1; 575 } 576 577 if (word_cnt > (1 << RAB_CNT_BW)) { 578 NT_LOG(ERR, NTHW, "%s: RAB: Illegal word count: value too large %d - max %d", 579 p_adapter_id_str, word_cnt, (1 << RAB_CNT_BW)); 580 return -1; 581 } 582 583 rte_spinlock_lock(&p->m_mutex); 584 585 if (p->m_dma_active) { 586 NT_LOG(ERR, NTHW, "%s: RAB: Illegal operation: DMA enabled", p_adapter_id_str); 587 res = -1; 588 goto exit_unlock_res; 589 } 590 591 /* Read buffer free register */ 592 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_BUF_FREE_ADDR, &buf_free); 593 594 in_buf_free = buf_free & p->RAC_RAB_BUF_FREE_IB_FREE_MASK; 595 out_buf_free = (buf_free & p->RAC_RAB_BUF_FREE_OB_FREE_MASK) >> 16; 596 597 /* Read buffer used register */ 598 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_BUF_USED_ADDR, &buf_used); 599 600 buf_used = 601 buf_used & (p->RAC_RAB_BUF_USED_IB_USED_MASK | p->RAC_RAB_BUF_USED_OB_USED_MASK); 602 603 /* 604 * Verify that output buffer can hold one completion word, 605 * input buffer can hold the number of words to be written + 606 * one write and one completion command 607 * and that the input and output "used" buffer is 0 608 */ 609 if (out_buf_free >= 1 && in_buf_free >= word_cnt + 2 && buf_used == 0) { 610 const uint32_t rab_oper_cmpl = (RAB_COMPLETION << RAB_OPR_LO); 611 uint32_t rab_echo_oper_cmpl; 612 uint32_t word_cnt_expected = 1; 613 uint32_t rab_oper_wr; 614 uint32_t i; 615 616 rab_oper_wr = (RAB_WRITE << RAB_OPR_LO) | 617 ((word_cnt & ((1 << RAB_CNT_BW) - 1)) << RAB_CNT_LO) | 618 (bus_id << RAB_BUSID_LO) | address; 619 620 if (trc) { 621 rab_oper_wr |= (RAB_ECHO << RAB_OPR_LO); 622 word_cnt_expected += word_cnt + 1; 623 } 624 625 /* Write command */ 626 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_IB_DATA_ADDR, rab_oper_wr); 627 628 /* Write data to input buffer */ 629 for (i = 0; i < word_cnt; i++) 630 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_IB_DATA_ADDR, p_data[i]); 631 632 /* Write completion command */ 633 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_IB_DATA_ADDR, rab_oper_cmpl); 634 635 /* Wait until done */ 636 if (_nthw_rac_wait_for_rab_done(p, address, word_cnt_expected)) { 637 res = -1; 638 goto exit_unlock_res; 639 } 640 641 if (trc) { 642 uint32_t rab_echo_oper_wr; 643 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, 644 &rab_echo_oper_wr); 645 646 if (p->mn_param_rac_rab_ob_update) 647 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, 0); 648 649 if (rab_oper_wr != rab_echo_oper_wr) { 650 NT_LOG(ERR, NTHW, 651 "%s: expected rab read echo oper (0x%08X) - read (0x%08X)", 652 p_adapter_id_str, rab_oper_wr, rab_echo_oper_wr); 653 } 654 } 655 656 { 657 /* Read data from output buffer */ 658 uint32_t data; 659 char *tmp_string; 660 661 if (trc) { 662 tmp_string = ntlog_helper_str_alloc("Register::write"); 663 ntlog_helper_str_add(tmp_string, 664 "(Dev: NA, Bus: RAB%u, Addr: 0x%08X, Cnt: %d, Data:", 665 bus_id, address, word_cnt); 666 } 667 668 for (i = 0; i < word_cnt; i++) { 669 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, &data); 670 671 if (p->mn_param_rac_rab_ob_update) { 672 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, 673 0); 674 } 675 676 if (trc) 677 ntlog_helper_str_add(tmp_string, " 0x%08X", data); 678 } 679 680 if (trc) { 681 ntlog_helper_str_add(tmp_string, ")"); 682 NT_LOG(DBG, NTHW, "%s", tmp_string); 683 ntlog_helper_str_free(tmp_string); 684 } 685 } 686 687 /* Read completion from out buffer */ 688 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, &rab_echo_oper_cmpl); 689 690 if (p->mn_param_rac_rab_ob_update) 691 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, 0); 692 693 if (rab_echo_oper_cmpl != rab_oper_cmpl) { 694 NT_LOG(ERR, NTHW, 695 "%s: RAB: Unexpected value of completion (0x%08X)- inBufFree: 0x%08X, outBufFree: 0x%08X, bufUsed: 0x%08X", 696 p_adapter_id_str, rab_echo_oper_cmpl, in_buf_free, out_buf_free, 697 buf_used); 698 res = -1; 699 goto exit_unlock_res; 700 } 701 702 /* Read buffer free register */ 703 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_BUF_FREE_ADDR, &buf_free); 704 705 if (buf_free & 0x80000000) { 706 /* Clear Timeout and overflow bits */ 707 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_BUF_FREE_ADDR, 0x0); 708 NT_LOG(ERR, NTHW, 709 "%s: RAB: timeout - Access outside register - bus: %d addr: 0x%08X - inBufFree: 0x%08X, outBufFree: 0x%08X, bufUsed: 0x%08X", 710 p_adapter_id_str, bus_id, address, in_buf_free, out_buf_free, 711 buf_used); 712 res = -1; 713 goto exit_unlock_res; 714 } 715 716 res = 0; 717 goto exit_unlock_res; 718 719 } else { 720 NT_LOG(ERR, NTHW, 721 "%s: RAB: Fail rab bus buffer check - bus: %d addr: 0x%08X wordcount: %d - inBufFree: 0x%08X, outBufFree: 0x%08X, bufUsed: 0x%08X", 722 p_adapter_id_str, bus_id, address, word_cnt, in_buf_free, out_buf_free, 723 buf_used); 724 res = -1; 725 goto exit_unlock_res; 726 } 727 728 exit_unlock_res: 729 rte_spinlock_unlock(&p->m_mutex); 730 return res; 731 } 732 733 int nthw_rac_rab_read32(nthw_rac_t *p, bool trc, nthw_rab_bus_id_t bus_id, uint32_t address, 734 uint32_t word_cnt, uint32_t *p_data) 735 { 736 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 737 const char *const p_adapter_id_str = p_fpga_info->mp_adapter_id_str; 738 uint32_t buf_used; 739 uint32_t buf_free; 740 uint32_t in_buf_free; 741 uint32_t out_buf_free; 742 int res = 0; 743 744 rte_spinlock_lock(&p->m_mutex); 745 746 if (address > (1 << RAB_ADDR_BW)) { 747 NT_LOG(ERR, NTHW, "%s: RAB: Illegal address: value too large %d - max %d", 748 p_adapter_id_str, address, (1 << RAB_ADDR_BW)); 749 res = -1; 750 goto exit_unlock_res; 751 } 752 753 if (bus_id > (1 << RAB_BUSID_BW)) { 754 NT_LOG(ERR, NTHW, "%s: RAB: Illegal bus id: value too large %d - max %d", 755 p_adapter_id_str, bus_id, (1 << RAB_BUSID_BW)); 756 res = -1; 757 goto exit_unlock_res; 758 } 759 760 if (word_cnt == 0) { 761 NT_LOG(ERR, NTHW, "%s: RAB: Illegal word count: value is zero (%d)", 762 p_adapter_id_str, word_cnt); 763 res = -1; 764 goto exit_unlock_res; 765 } 766 767 if (word_cnt > (1 << RAB_CNT_BW)) { 768 NT_LOG(ERR, NTHW, "%s: RAB: Illegal word count: value too large %d - max %d", 769 p_adapter_id_str, word_cnt, (1 << RAB_CNT_BW)); 770 res = -1; 771 goto exit_unlock_res; 772 } 773 774 /* Read buffer free register */ 775 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_BUF_FREE_ADDR, &buf_free); 776 777 in_buf_free = buf_free & p->RAC_RAB_BUF_FREE_IB_FREE_MASK; 778 out_buf_free = (buf_free & p->RAC_RAB_BUF_FREE_OB_FREE_MASK) >> 16; 779 780 /* Read buffer used register */ 781 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_BUF_USED_ADDR, &buf_used); 782 783 buf_used = 784 buf_used & (p->RAC_RAB_BUF_USED_IB_USED_MASK | p->RAC_RAB_BUF_USED_OB_USED_MASK); 785 786 /* 787 * Verify that output buffer can hold the number of words to be read, 788 * input buffer can hold one read command 789 * and that the input and output "used" buffer is 0 790 */ 791 if (out_buf_free >= word_cnt && in_buf_free >= 1 && buf_used == 0) { 792 const uint32_t rab_oper_cmpl = (RAB_COMPLETION << RAB_OPR_LO); 793 uint32_t rab_read_oper_cmpl; 794 uint32_t word_cnt_expected = word_cnt + 1; 795 uint32_t rab_oper_rd; 796 797 rab_oper_rd = (RAB_READ << RAB_OPR_LO) | 798 ((word_cnt & ((1 << RAB_CNT_BW) - 1)) << RAB_CNT_LO) | 799 (bus_id << RAB_BUSID_LO) | address; 800 801 if (trc) { 802 rab_oper_rd |= (RAB_ECHO << RAB_OPR_LO); 803 word_cnt_expected++; 804 } 805 806 /* Write command */ 807 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_IB_DATA_ADDR, rab_oper_rd); 808 809 /* Write completion command */ 810 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_IB_DATA_ADDR, rab_oper_cmpl); 811 812 /* Wait until done */ 813 if (_nthw_rac_wait_for_rab_done(p, address, word_cnt_expected)) { 814 res = -1; 815 goto exit_unlock_res; 816 } 817 818 if (trc) { 819 uint32_t rab_echo_oper_rd; 820 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, 821 &rab_echo_oper_rd); 822 823 if (p->mn_param_rac_rab_ob_update) 824 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, 0); 825 826 if (rab_oper_rd != rab_echo_oper_rd) { 827 NT_LOG(ERR, NTHW, 828 "%s: RAB: expected rab read echo oper (0x%08X) - read (0x%08X)", 829 p_adapter_id_str, rab_oper_rd, rab_echo_oper_rd); 830 } 831 } 832 833 { 834 /* Read data from output buffer */ 835 uint32_t i; 836 837 for (i = 0; i < word_cnt; i++) { 838 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, 839 &p_data[i]); 840 841 if (p->mn_param_rac_rab_ob_update) { 842 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, 843 0); 844 } 845 } 846 847 if (trc) { 848 char *tmp_string = ntlog_helper_str_alloc("Register::read"); 849 ntlog_helper_str_add(tmp_string, 850 "(Dev: NA, Bus: RAB%u, Addr: 0x%08X, Cnt: %d, Data:", 851 bus_id, address, word_cnt); 852 853 for (i = 0; i < word_cnt; i++) 854 ntlog_helper_str_add(tmp_string, " 0x%08X", p_data[i]); 855 856 ntlog_helper_str_add(tmp_string, ")"); 857 NT_LOG(DBG, NTHW, "%s", tmp_string); 858 ntlog_helper_str_free(tmp_string); 859 } 860 } 861 862 /* Read completion from out buffer */ 863 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, &rab_read_oper_cmpl); 864 865 if (p->mn_param_rac_rab_ob_update) 866 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_OB_DATA_ADDR, 0); 867 868 if (rab_read_oper_cmpl != rab_oper_cmpl) { 869 NT_LOG(ERR, NTHW, 870 "%s: RAB: Unexpected value of completion (0x%08X)- inBufFree: 0x%08X, outBufFree: 0x%08X, bufUsed: 0x%08X", 871 p_adapter_id_str, rab_read_oper_cmpl, in_buf_free, out_buf_free, 872 buf_used); 873 res = -1; 874 goto exit_unlock_res; 875 } 876 877 /* Read buffer free register */ 878 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_BUF_FREE_ADDR, &buf_free); 879 880 if (buf_free & 0x80000000) { 881 /* Clear Timeout and overflow bits */ 882 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_BUF_FREE_ADDR, 0x0); 883 NT_LOG(ERR, NTHW, 884 "%s: RAB: timeout - Access outside register - bus: %d addr: 0x%08X - inBufFree: 0x%08X, outBufFree: 0x%08X, bufUsed: 0x%08X", 885 p_adapter_id_str, bus_id, address, in_buf_free, out_buf_free, 886 buf_used); 887 res = -1; 888 goto exit_unlock_res; 889 } 890 891 res = 0; 892 goto exit_unlock_res; 893 894 } else { 895 NT_LOG(ERR, NTHW, 896 "%s: RAB: Fail rab bus buffer check - bus: %d addr: 0x%08X wordcount: %d - inBufFree: 0x%08X, outBufFree: 0x%08X, bufUsed: 0x%08X", 897 p_adapter_id_str, bus_id, address, word_cnt, in_buf_free, out_buf_free, 898 buf_used); 899 res = -1; 900 goto exit_unlock_res; 901 } 902 903 exit_unlock_res: 904 rte_spinlock_unlock(&p->m_mutex); 905 return res; 906 } 907 908 int nthw_rac_rab_flush(nthw_rac_t *p) 909 { 910 const struct fpga_info_s *const p_fpga_info = p->mp_fpga->p_fpga_info; 911 const char *const p_adapter_id_str = p_fpga_info->mp_adapter_id_str; 912 uint32_t data = 0; 913 uint32_t retry; 914 int res = 0; 915 916 rte_spinlock_lock(&p->m_mutex); 917 918 /* Set the flush bit */ 919 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_BUF_USED_ADDR, 920 p->RAC_RAB_BUF_USED_FLUSH_MASK); 921 922 /* Reset BUF FREE register */ 923 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_BUF_FREE_ADDR, 0x0); 924 925 /* Wait until OB_USED and IB_USED are 0 */ 926 for (retry = 0; retry < 100000; retry++) { 927 nthw_rac_reg_read32(p_fpga_info, p->RAC_RAB_BUF_USED_ADDR, &data); 928 929 if ((data & 0xFFFFFFFF) == p->RAC_RAB_BUF_USED_FLUSH_MASK) 930 break; 931 } 932 933 if (data != p->RAC_RAB_BUF_USED_FLUSH_MASK) { 934 NT_LOG(ERR, NTHW, "%s: RAB: Rab bus flush error.", p_adapter_id_str); 935 res = -1; 936 } 937 938 /* Clear flush bit when done */ 939 nthw_rac_reg_write32(p_fpga_info, p->RAC_RAB_BUF_USED_ADDR, 0x0); 940 941 rte_spinlock_unlock(&p->m_mutex); 942 return res; 943 } 944