1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include "timvf_evdev.h" 6 7 int otx_logtype_timvf; 8 9 RTE_INIT(otx_timvf_init_log) 10 { 11 otx_logtype_timvf = rte_log_register("pmd.event.octeontx.timer"); 12 if (otx_logtype_timvf >= 0) 13 rte_log_set_level(otx_logtype_timvf, RTE_LOG_NOTICE); 14 } 15 16 struct __rte_packed timvf_mbox_dev_info { 17 uint64_t ring_active[4]; 18 uint64_t clk_freq; 19 }; 20 21 /* Response messages */ 22 enum { 23 MBOX_RET_SUCCESS, 24 MBOX_RET_INVALID, 25 MBOX_RET_INTERNAL_ERR, 26 }; 27 28 static int 29 timvf_mbox_dev_info_get(struct timvf_mbox_dev_info *info) 30 { 31 struct octeontx_mbox_hdr hdr = {0}; 32 uint16_t len = sizeof(struct timvf_mbox_dev_info); 33 34 hdr.coproc = TIM_COPROC; 35 hdr.msg = TIM_GET_DEV_INFO; 36 hdr.vfid = 0; /* TIM DEV is always 0. TIM RING ID changes. */ 37 38 memset(info, 0, len); 39 return octeontx_mbox_send(&hdr, NULL, 0, info, len); 40 } 41 42 static void 43 timvf_ring_info_get(const struct rte_event_timer_adapter *adptr, 44 struct rte_event_timer_adapter_info *adptr_info) 45 { 46 struct timvf_ring *timr = adptr->data->adapter_priv; 47 adptr_info->max_tmo_ns = timr->max_tout; 48 adptr_info->min_resolution_ns = timr->tck_nsec; 49 rte_memcpy(&adptr_info->conf, &adptr->data->conf, 50 sizeof(struct rte_event_timer_adapter_conf)); 51 } 52 53 static int 54 timvf_ring_conf_set(struct timvf_ctrl_reg *rctl, uint8_t ring_id) 55 { 56 struct octeontx_mbox_hdr hdr = {0}; 57 uint16_t len = sizeof(struct timvf_ctrl_reg); 58 int ret; 59 60 hdr.coproc = TIM_COPROC; 61 hdr.msg = TIM_SET_RING_INFO; 62 hdr.vfid = ring_id; 63 64 ret = octeontx_mbox_send(&hdr, rctl, len, NULL, 0); 65 if (ret < 0 || hdr.res_code != MBOX_RET_SUCCESS) 66 return -EACCES; 67 return 0; 68 } 69 70 static int 71 timvf_get_start_cyc(uint64_t *now, uint8_t ring_id) 72 { 73 struct octeontx_mbox_hdr hdr = {0}; 74 75 hdr.coproc = TIM_COPROC; 76 hdr.msg = TIM_RING_START_CYC_GET; 77 hdr.vfid = ring_id; 78 *now = 0; 79 return octeontx_mbox_send(&hdr, NULL, 0, now, sizeof(uint64_t)); 80 } 81 82 static int 83 optimize_bucket_parameters(struct timvf_ring *timr) 84 { 85 uint32_t hbkts; 86 uint32_t lbkts; 87 uint64_t tck_nsec; 88 89 hbkts = rte_align32pow2(timr->nb_bkts); 90 tck_nsec = RTE_ALIGN_MUL_CEIL(timr->max_tout / (hbkts - 1), 10); 91 92 if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS)) 93 hbkts = 0; 94 95 lbkts = rte_align32prevpow2(timr->nb_bkts); 96 tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout / (lbkts - 1)), 10); 97 98 if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS)) 99 lbkts = 0; 100 101 if (!hbkts && !lbkts) 102 return 0; 103 104 if (!hbkts) { 105 timr->nb_bkts = lbkts; 106 goto end; 107 } else if (!lbkts) { 108 timr->nb_bkts = hbkts; 109 goto end; 110 } 111 112 timr->nb_bkts = (hbkts - timr->nb_bkts) < 113 (timr->nb_bkts - lbkts) ? hbkts : lbkts; 114 end: 115 timr->get_target_bkt = bkt_and; 116 timr->tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout / 117 (timr->nb_bkts - 1)), 10); 118 return 1; 119 } 120 121 static int 122 timvf_ring_start(const struct rte_event_timer_adapter *adptr) 123 { 124 int ret; 125 uint8_t use_fpa = 0; 126 uint64_t interval; 127 uintptr_t pool; 128 struct timvf_ctrl_reg rctrl; 129 struct timvf_mbox_dev_info dinfo; 130 struct timvf_ring *timr = adptr->data->adapter_priv; 131 132 ret = timvf_mbox_dev_info_get(&dinfo); 133 if (ret < 0 || ret != sizeof(struct timvf_mbox_dev_info)) 134 return -EINVAL; 135 136 /* Calculate the interval cycles according to clock source. */ 137 switch (timr->clk_src) { 138 case TIM_CLK_SRC_SCLK: 139 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq); 140 break; 141 case TIM_CLK_SRC_GPIO: 142 /* GPIO doesn't work on tck_nsec. */ 143 interval = 0; 144 break; 145 case TIM_CLK_SRC_GTI: 146 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq); 147 break; 148 case TIM_CLK_SRC_PTP: 149 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq); 150 break; 151 default: 152 timvf_log_err("Unsupported clock source configured %d", 153 timr->clk_src); 154 return -EINVAL; 155 } 156 157 if (!strcmp(rte_mbuf_best_mempool_ops(), "octeontx_fpavf")) 158 use_fpa = 1; 159 160 /*CTRL0 register.*/ 161 rctrl.rctrl0 = interval; 162 163 /*CTRL1 register.*/ 164 rctrl.rctrl1 = (uint64_t)(timr->clk_src) << 51 | 165 1ull << 48 /* LOCK_EN (Enable hw bucket lock mechanism) */ | 166 1ull << 47 /* ENA */ | 167 1ull << 44 /* ENA_LDWB */ | 168 (timr->nb_bkts - 1); 169 170 rctrl.rctrl2 = (uint64_t)(TIM_CHUNK_SIZE / 16) << 40; 171 172 if (use_fpa) { 173 pool = (uintptr_t)((struct rte_mempool *) 174 timr->chunk_pool)->pool_id; 175 ret = octeontx_fpa_bufpool_gaura(pool); 176 if (ret < 0) { 177 timvf_log_dbg("Unable to get gaura id"); 178 ret = -ENOMEM; 179 goto error; 180 } 181 timvf_write64((uint64_t)ret, 182 (uint8_t *)timr->vbar0 + TIM_VRING_AURA); 183 } else { 184 rctrl.rctrl1 |= 1ull << 43 /* ENA_DFB (Enable don't free) */; 185 } 186 187 timvf_write64((uintptr_t)timr->bkt, 188 (uint8_t *)timr->vbar0 + TIM_VRING_BASE); 189 timvf_set_chunk_refill(timr, use_fpa); 190 if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id)) { 191 ret = -EACCES; 192 goto error; 193 } 194 195 if (timvf_get_start_cyc(&timr->ring_start_cyc, 196 timr->tim_ring_id) < 0) { 197 ret = -EACCES; 198 goto error; 199 } 200 timr->tck_int = NSEC2CLK(timr->tck_nsec, rte_get_timer_hz()); 201 timr->fast_div = rte_reciprocal_value_u64(timr->tck_int); 202 timvf_log_info("nb_bkts %d min_ns %"PRIu64" min_cyc %"PRIu64"" 203 " maxtmo %"PRIu64"\n", 204 timr->nb_bkts, timr->tck_nsec, interval, 205 timr->max_tout); 206 207 return 0; 208 error: 209 rte_free(timr->bkt); 210 rte_mempool_free(timr->chunk_pool); 211 return ret; 212 } 213 214 static int 215 timvf_ring_stop(const struct rte_event_timer_adapter *adptr) 216 { 217 struct timvf_ring *timr = adptr->data->adapter_priv; 218 struct timvf_ctrl_reg rctrl = {0}; 219 rctrl.rctrl0 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL0); 220 rctrl.rctrl1 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL1); 221 rctrl.rctrl1 &= ~(1ull << 47); /* Disable */ 222 rctrl.rctrl2 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL2); 223 224 if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id)) 225 return -EACCES; 226 return 0; 227 } 228 229 static int 230 timvf_ring_create(struct rte_event_timer_adapter *adptr) 231 { 232 char pool_name[25]; 233 int ret; 234 uint8_t tim_ring_id; 235 uint64_t nb_timers; 236 struct rte_event_timer_adapter_conf *rcfg = &adptr->data->conf; 237 struct timvf_ring *timr; 238 const char *mempool_ops; 239 unsigned int mp_flags = 0; 240 241 tim_ring_id = timvf_get_ring(); 242 if (tim_ring_id == UINT8_MAX) 243 return -ENODEV; 244 245 timr = rte_zmalloc("octeontx_timvf_priv", 246 sizeof(struct timvf_ring), 0); 247 if (timr == NULL) 248 return -ENOMEM; 249 250 adptr->data->adapter_priv = timr; 251 /* Check config parameters. */ 252 if ((rcfg->clk_src != RTE_EVENT_TIMER_ADAPTER_CPU_CLK) && 253 (!rcfg->timer_tick_ns || 254 rcfg->timer_tick_ns < TIM_MIN_INTERVAL)) { 255 timvf_log_err("Too low timer ticks"); 256 goto cfg_err; 257 } 258 259 timr->clk_src = (int) rcfg->clk_src; 260 timr->tim_ring_id = tim_ring_id; 261 timr->tck_nsec = RTE_ALIGN_MUL_CEIL(rcfg->timer_tick_ns, 10); 262 timr->max_tout = rcfg->max_tmo_ns; 263 timr->nb_bkts = (timr->max_tout / timr->tck_nsec); 264 timr->vbar0 = timvf_bar(timr->tim_ring_id, 0); 265 timr->bkt_pos = (uint8_t *)timr->vbar0 + TIM_VRING_REL; 266 nb_timers = rcfg->nb_timers; 267 timr->get_target_bkt = bkt_mod; 268 269 timr->nb_chunks = nb_timers / nb_chunk_slots; 270 271 /* Try to optimize the bucket parameters. */ 272 if ((rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES) 273 && !rte_is_power_of_2(timr->nb_bkts)) { 274 if (optimize_bucket_parameters(timr)) { 275 timvf_log_info("Optimized configured values"); 276 timvf_log_dbg("nb_bkts : %"PRIu32"", timr->nb_bkts); 277 timvf_log_dbg("tck_nsec : %"PRIu64"", timr->tck_nsec); 278 } else 279 timvf_log_info("Failed to Optimize configured values"); 280 } 281 282 if (rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT) { 283 mp_flags = MEMPOOL_F_SP_PUT | MEMPOOL_F_SC_GET; 284 timvf_log_info("Using single producer mode"); 285 } 286 287 timr->bkt = rte_zmalloc("octeontx_timvf_bucket", 288 (timr->nb_bkts) * sizeof(struct tim_mem_bucket), 289 0); 290 if (timr->bkt == NULL) 291 goto mem_err; 292 293 snprintf(pool_name, sizeof(pool_name), "timvf_chunk_pool%d", 294 timr->tim_ring_id); 295 timr->chunk_pool = (void *)rte_mempool_create_empty(pool_name, 296 timr->nb_chunks, TIM_CHUNK_SIZE, 0, 0, rte_socket_id(), 297 mp_flags); 298 299 if (!timr->chunk_pool) { 300 rte_free(timr->bkt); 301 timvf_log_err("Unable to create chunkpool."); 302 return -ENOMEM; 303 } 304 305 mempool_ops = rte_mbuf_best_mempool_ops(); 306 ret = rte_mempool_set_ops_byname(timr->chunk_pool, 307 mempool_ops, NULL); 308 309 if (ret != 0) { 310 timvf_log_err("Unable to set chunkpool ops."); 311 goto mem_err; 312 } 313 314 ret = rte_mempool_populate_default(timr->chunk_pool); 315 if (ret < 0) { 316 timvf_log_err("Unable to set populate chunkpool."); 317 goto mem_err; 318 } 319 timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VRING_BASE); 320 timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT); 321 timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT_W1S); 322 timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1C); 323 timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1S); 324 325 return 0; 326 mem_err: 327 rte_free(timr); 328 return -ENOMEM; 329 cfg_err: 330 rte_free(timr); 331 return -EINVAL; 332 } 333 334 static int 335 timvf_ring_free(struct rte_event_timer_adapter *adptr) 336 { 337 struct timvf_ring *timr = adptr->data->adapter_priv; 338 339 rte_mempool_free(timr->chunk_pool); 340 rte_free(timr->bkt); 341 timvf_release_ring(timr->tim_ring_id); 342 rte_free(adptr->data->adapter_priv); 343 return 0; 344 } 345 346 static int 347 timvf_stats_get(const struct rte_event_timer_adapter *adapter, 348 struct rte_event_timer_adapter_stats *stats) 349 { 350 struct timvf_ring *timr = adapter->data->adapter_priv; 351 uint64_t bkt_cyc = rte_rdtsc() - timr->ring_start_cyc; 352 353 stats->evtim_exp_count = timr->tim_arm_cnt; 354 stats->ev_enq_count = timr->tim_arm_cnt; 355 stats->adapter_tick_count = rte_reciprocal_divide_u64(bkt_cyc, 356 &timr->fast_div); 357 return 0; 358 } 359 360 static int 361 timvf_stats_reset(const struct rte_event_timer_adapter *adapter) 362 { 363 struct timvf_ring *timr = adapter->data->adapter_priv; 364 365 timr->tim_arm_cnt = 0; 366 return 0; 367 } 368 369 static struct rte_event_timer_adapter_ops timvf_ops = { 370 .init = timvf_ring_create, 371 .uninit = timvf_ring_free, 372 .start = timvf_ring_start, 373 .stop = timvf_ring_stop, 374 .get_info = timvf_ring_info_get, 375 }; 376 377 int 378 timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags, 379 uint32_t *caps, const struct rte_event_timer_adapter_ops **ops, 380 uint8_t enable_stats) 381 { 382 RTE_SET_USED(dev); 383 384 if (enable_stats) { 385 timvf_ops.stats_get = timvf_stats_get; 386 timvf_ops.stats_reset = timvf_stats_reset; 387 } 388 389 if (flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT) 390 timvf_ops.arm_burst = enable_stats ? 391 timvf_timer_arm_burst_sp_stats : 392 timvf_timer_arm_burst_sp; 393 else 394 timvf_ops.arm_burst = enable_stats ? 395 timvf_timer_arm_burst_mp_stats : 396 timvf_timer_arm_burst_mp; 397 398 timvf_ops.arm_tmo_tick_burst = enable_stats ? 399 timvf_timer_arm_tmo_brst_stats : 400 timvf_timer_arm_tmo_brst; 401 timvf_ops.cancel_burst = timvf_timer_cancel_burst; 402 *caps = RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT; 403 *ops = &timvf_ops; 404 return 0; 405 } 406