1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 3 * All rights reserved. 4 */ 5 6 #ifndef DPDK_ENA_COM_ENA_PLAT_DPDK_H_ 7 #define DPDK_ENA_COM_ENA_PLAT_DPDK_H_ 8 9 #include <stdbool.h> 10 #include <stdlib.h> 11 #include <pthread.h> 12 #include <stdint.h> 13 #include <inttypes.h> 14 #include <string.h> 15 #include <errno.h> 16 17 #include <rte_atomic.h> 18 #include <rte_branch_prediction.h> 19 #include <rte_cycles.h> 20 #include <rte_io.h> 21 #include <rte_log.h> 22 #include <rte_malloc.h> 23 #include <rte_memzone.h> 24 #include <rte_prefetch.h> 25 #include <rte_spinlock.h> 26 27 #include <sys/time.h> 28 29 typedef uint64_t u64; 30 typedef uint32_t u32; 31 typedef uint16_t u16; 32 typedef uint8_t u8; 33 34 typedef uint64_t dma_addr_t; 35 #ifndef ETIME 36 #define ETIME ETIMEDOUT 37 #endif 38 39 #define ena_atomic32_t rte_atomic32_t 40 #define ena_mem_handle_t const struct rte_memzone * 41 42 #define SZ_256 (256U) 43 #define SZ_4K (4096U) 44 45 #define ENA_COM_OK 0 46 #define ENA_COM_NO_MEM -ENOMEM 47 #define ENA_COM_INVAL -EINVAL 48 #define ENA_COM_NO_SPACE -ENOSPC 49 #define ENA_COM_NO_DEVICE -ENODEV 50 #define ENA_COM_TIMER_EXPIRED -ETIME 51 #define ENA_COM_FAULT -EFAULT 52 #define ENA_COM_TRY_AGAIN -EAGAIN 53 #define ENA_COM_UNSUPPORTED -EOPNOTSUPP 54 55 #define ____cacheline_aligned __rte_cache_aligned 56 57 #define ENA_ABORT() abort() 58 59 #define ENA_MSLEEP(x) rte_delay_us_sleep(x * 1000) 60 #define ENA_USLEEP(x) rte_delay_us_sleep(x) 61 #define ENA_UDELAY(x) rte_delay_us_block(x) 62 63 #define ENA_TOUCH(x) ((void)(x)) 64 #define memcpy_toio memcpy 65 #define wmb rte_wmb 66 #define rmb rte_rmb 67 #define mb rte_mb 68 #define mmiowb rte_io_wmb 69 #define __iomem 70 71 #define US_PER_S 1000000 72 #define ENA_GET_SYSTEM_USECS() \ 73 (rte_get_timer_cycles() * US_PER_S / rte_get_timer_hz()) 74 75 extern int ena_logtype_com; 76 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG 77 #define ENA_ASSERT(cond, format, arg...) \ 78 do { \ 79 if (unlikely(!(cond))) { \ 80 rte_log(RTE_LOGTYPE_ERR, ena_logtype_com, \ 81 format, ##arg); \ 82 rte_panic("line %d\tassert \"" #cond "\"" \ 83 "failed\n", __LINE__); \ 84 } \ 85 } while (0) 86 #else 87 #define ENA_ASSERT(cond, format, arg...) do {} while (0) 88 #endif 89 90 #define ENA_MAX_T(type, x, y) RTE_MAX((type)(x), (type)(y)) 91 #define ENA_MAX32(x, y) ENA_MAX_T(uint32_t, (x), (y)) 92 #define ENA_MAX16(x, y) ENA_MAX_T(uint16_t, (x), (y)) 93 #define ENA_MAX8(x, y) ENA_MAX_T(uint8_t, (x), (y)) 94 #define ENA_MIN_T(type, x, y) RTE_MIN((type)(x), (type)(y)) 95 #define ENA_MIN32(x, y) ENA_MIN_T(uint32_t, (x), (y)) 96 #define ENA_MIN16(x, y) ENA_MIN_T(uint16_t, (x), (y)) 97 #define ENA_MIN8(x, y) ENA_MIN_T(uint8_t, (x), (y)) 98 99 #define BITS_PER_LONG_LONG (__SIZEOF_LONG_LONG__ * 8) 100 #define U64_C(x) x ## ULL 101 #define BIT(nr) (1UL << (nr)) 102 #define BITS_PER_LONG (__SIZEOF_LONG__ * 8) 103 #define GENMASK(h, l) (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) 104 #define GENMASK_ULL(h, l) (((~0ULL) - (1ULL << (l)) + 1) & \ 105 (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h)))) 106 107 #ifdef RTE_LIBRTE_ENA_COM_DEBUG 108 #define ena_trc_log(level, fmt, arg...) \ 109 rte_log(RTE_LOG_ ## level, ena_logtype_com, \ 110 "[ENA_COM: %s]" fmt, __func__, ##arg) 111 112 #define ena_trc_dbg(format, arg...) ena_trc_log(DEBUG, format, ##arg) 113 #define ena_trc_info(format, arg...) ena_trc_log(INFO, format, ##arg) 114 #define ena_trc_warn(format, arg...) ena_trc_log(WARNING, format, ##arg) 115 #define ena_trc_err(format, arg...) ena_trc_log(ERR, format, ##arg) 116 #else 117 #define ena_trc_dbg(format, arg...) do { } while (0) 118 #define ena_trc_info(format, arg...) do { } while (0) 119 #define ena_trc_warn(format, arg...) do { } while (0) 120 #define ena_trc_err(format, arg...) do { } while (0) 121 #endif /* RTE_LIBRTE_ENA_COM_DEBUG */ 122 123 #define ENA_WARN(cond, format, arg...) \ 124 do { \ 125 if (unlikely(cond)) { \ 126 ena_trc_err( \ 127 "Warn failed on %s:%s:%d:" format, \ 128 __FILE__, __func__, __LINE__, ##arg); \ 129 } \ 130 } while (0) 131 132 /* Spinlock related methods */ 133 #define ena_spinlock_t rte_spinlock_t 134 #define ENA_SPINLOCK_INIT(spinlock) rte_spinlock_init(&spinlock) 135 #define ENA_SPINLOCK_LOCK(spinlock, flags) \ 136 ({(void)flags; rte_spinlock_lock(&spinlock); }) 137 #define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ 138 ({(void)flags; rte_spinlock_unlock(&(spinlock)); }) 139 #define ENA_SPINLOCK_DESTROY(spinlock) ((void)spinlock) 140 141 #define q_waitqueue_t \ 142 struct { \ 143 pthread_cond_t cond; \ 144 pthread_mutex_t mutex; \ 145 } 146 147 #define ena_wait_queue_t q_waitqueue_t 148 149 #define ENA_WAIT_EVENT_INIT(waitqueue) \ 150 do { \ 151 pthread_mutex_init(&(waitqueue).mutex, NULL); \ 152 pthread_cond_init(&(waitqueue).cond, NULL); \ 153 } while (0) 154 155 #define ENA_WAIT_EVENT_WAIT(waitevent, timeout) \ 156 do { \ 157 struct timespec wait; \ 158 struct timeval now; \ 159 unsigned long timeout_us; \ 160 gettimeofday(&now, NULL); \ 161 wait.tv_sec = now.tv_sec + timeout / 1000000UL; \ 162 timeout_us = timeout % 1000000UL; \ 163 wait.tv_nsec = (now.tv_usec + timeout_us) * 1000UL; \ 164 pthread_mutex_lock(&waitevent.mutex); \ 165 pthread_cond_timedwait(&waitevent.cond, \ 166 &waitevent.mutex, &wait); \ 167 pthread_mutex_unlock(&waitevent.mutex); \ 168 } while (0) 169 #define ENA_WAIT_EVENT_SIGNAL(waitevent) pthread_cond_signal(&waitevent.cond) 170 /* pthread condition doesn't need to be rearmed after usage */ 171 #define ENA_WAIT_EVENT_CLEAR(...) 172 #define ENA_WAIT_EVENT_DESTROY(waitqueue) ((void)(waitqueue)) 173 174 #define ena_wait_event_t ena_wait_queue_t 175 #define ENA_MIGHT_SLEEP() 176 177 #define ena_time_t uint64_t 178 #define ENA_TIME_EXPIRE(timeout) (timeout < rte_get_timer_cycles()) 179 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ 180 (timeout_us * rte_get_timer_hz() / 1000000 + rte_get_timer_cycles()) 181 182 /* 183 * Each rte_memzone should have unique name. 184 * To satisfy it, count number of allocations and add it to name. 185 */ 186 extern rte_atomic32_t ena_alloc_cnt; 187 188 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, handle) \ 189 do { \ 190 const struct rte_memzone *mz = NULL; \ 191 ENA_TOUCH(dmadev); ENA_TOUCH(handle); \ 192 if (size > 0) { \ 193 char z_name[RTE_MEMZONE_NAMESIZE]; \ 194 snprintf(z_name, sizeof(z_name), \ 195 "ena_alloc_%d", \ 196 rte_atomic32_add_return(&ena_alloc_cnt, 1)); \ 197 mz = rte_memzone_reserve(z_name, size, \ 198 SOCKET_ID_ANY, \ 199 RTE_MEMZONE_IOVA_CONTIG); \ 200 handle = mz; \ 201 } \ 202 if (mz == NULL) { \ 203 virt = NULL; \ 204 phys = 0; \ 205 } else { \ 206 memset(mz->addr, 0, size); \ 207 virt = mz->addr; \ 208 phys = mz->iova; \ 209 } \ 210 } while (0) 211 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, handle) \ 212 ({ ENA_TOUCH(size); ENA_TOUCH(phys); \ 213 ENA_TOUCH(dmadev); \ 214 rte_memzone_free(handle); }) 215 216 #define ENA_MEM_ALLOC_COHERENT_NODE( \ 217 dmadev, size, virt, phys, mem_handle, node, dev_node) \ 218 do { \ 219 const struct rte_memzone *mz = NULL; \ 220 ENA_TOUCH(dmadev); ENA_TOUCH(dev_node); \ 221 if (size > 0) { \ 222 char z_name[RTE_MEMZONE_NAMESIZE]; \ 223 snprintf(z_name, sizeof(z_name), \ 224 "ena_alloc_%d", \ 225 rte_atomic32_add_return(&ena_alloc_cnt, 1)); \ 226 mz = rte_memzone_reserve(z_name, size, node, \ 227 RTE_MEMZONE_IOVA_CONTIG); \ 228 mem_handle = mz; \ 229 } \ 230 if (mz == NULL) { \ 231 virt = NULL; \ 232 phys = 0; \ 233 } else { \ 234 memset(mz->addr, 0, size); \ 235 virt = mz->addr; \ 236 phys = mz->iova; \ 237 } \ 238 } while (0) 239 240 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \ 241 do { \ 242 ENA_TOUCH(dmadev); ENA_TOUCH(dev_node); \ 243 virt = rte_zmalloc_socket(NULL, size, 0, node); \ 244 } while (0) 245 246 #define ENA_MEM_ALLOC(dmadev, size) rte_zmalloc(NULL, size, 1) 247 #define ENA_MEM_FREE(dmadev, ptr, size) \ 248 ({ ENA_TOUCH(dmadev); ENA_TOUCH(size); rte_free(ptr); }) 249 250 #define ENA_DB_SYNC(mem_handle) ((void)mem_handle) 251 252 #define ENA_REG_WRITE32(bus, value, reg) \ 253 ({ (void)(bus); rte_write32((value), (reg)); }) 254 #define ENA_REG_WRITE32_RELAXED(bus, value, reg) \ 255 ({ (void)(bus); rte_write32_relaxed((value), (reg)); }) 256 #define ENA_REG_READ32(bus, reg) \ 257 ({ (void)(bus); rte_read32_relaxed((reg)); }) 258 259 #define ATOMIC32_INC(i32_ptr) rte_atomic32_inc(i32_ptr) 260 #define ATOMIC32_DEC(i32_ptr) rte_atomic32_dec(i32_ptr) 261 #define ATOMIC32_SET(i32_ptr, val) rte_atomic32_set(i32_ptr, val) 262 #define ATOMIC32_READ(i32_ptr) rte_atomic32_read(i32_ptr) 263 264 #define msleep(x) rte_delay_us(x * 1000) 265 #define udelay(x) rte_delay_us(x) 266 267 #define dma_rmb() rmb() 268 269 #define MAX_ERRNO 4095 270 #define IS_ERR(x) (((unsigned long)x) >= (unsigned long)-MAX_ERRNO) 271 #define ERR_PTR(error) ((void *)(long)error) 272 #define PTR_ERR(error) ((long)(void *)error) 273 #define might_sleep() 274 275 #define prefetch(x) rte_prefetch0(x) 276 #define prefetchw(x) prefetch(x) 277 278 #define lower_32_bits(x) ((uint32_t)(x)) 279 #define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16)) 280 281 #define ENA_TIME_EXPIRE(timeout) (timeout < rte_get_timer_cycles()) 282 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ 283 (timeout_us * rte_get_timer_hz() / 1000000 + rte_get_timer_cycles()) 284 #define ENA_WAIT_EVENT_DESTROY(waitqueue) ((void)(waitqueue)) 285 286 #ifndef READ_ONCE 287 #define READ_ONCE(var) (*((volatile typeof(var) *)(&(var)))) 288 #endif 289 290 #define READ_ONCE8(var) READ_ONCE(var) 291 #define READ_ONCE16(var) READ_ONCE(var) 292 #define READ_ONCE32(var) READ_ONCE(var) 293 294 /* The size must be 8 byte align */ 295 #define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \ 296 do { \ 297 int count, i; \ 298 uint64_t *to = (uint64_t *)(dst); \ 299 const uint64_t *from = (const uint64_t *)(src); \ 300 count = (size) / 8; \ 301 for (i = 0; i < count; i++, from++, to++) \ 302 rte_write64_relaxed(*from, to); \ 303 } while(0) 304 305 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 306 307 #define ENA_FFS(x) ffs(x) 308 309 void ena_rss_key_fill(void *key, size_t size); 310 311 #define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size) 312 313 #define ENA_INTR_INITIAL_TX_INTERVAL_USECS_PLAT 0 314 315 #define ENA_PRIu64 PRIu64 316 317 #include "ena_includes.h" 318 #endif /* DPDK_ENA_COM_ENA_PLAT_DPDK_H_ */ 319