1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <rte_mempool.h> 6 7 #include "roc_api.h" 8 #include "cnxk_mempool.h" 9 10 static int __rte_hot 11 cn9k_mempool_enq(struct rte_mempool *mp, void *const *obj_table, unsigned int n) 12 { 13 /* Ensure mbuf init changes are written before the free pointers 14 * are enqueued to the stack. 15 */ 16 rte_io_wmb(); 17 roc_npa_aura_op_bulk_free(mp->pool_id, (const uint64_t *)obj_table, n, 18 0); 19 20 return 0; 21 } 22 23 static inline int __rte_hot 24 cn9k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n) 25 { 26 unsigned int count; 27 28 count = roc_npa_aura_op_bulk_alloc(mp->pool_id, (uint64_t *)obj_table, 29 n, 0, 1); 30 31 if (unlikely(count != n)) { 32 /* If bulk alloc failed to allocate all pointers, try 33 * allocating remaining pointers with the default alloc 34 * with retry scheme. 35 */ 36 if (cnxk_mempool_deq(mp, &obj_table[count], n - count)) { 37 cn9k_mempool_enq(mp, obj_table, count); 38 return -ENOENT; 39 } 40 } 41 42 return 0; 43 } 44 45 static int 46 cn9k_mempool_alloc(struct rte_mempool *mp) 47 { 48 size_t block_size, padding; 49 50 block_size = mp->elt_size + mp->header_size + mp->trailer_size; 51 /* Align header size to ROC_ALIGN */ 52 if (mp->header_size % ROC_ALIGN != 0) { 53 padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) - 54 mp->header_size; 55 mp->header_size += padding; 56 block_size += padding; 57 } 58 59 /* Align block size to ROC_ALIGN */ 60 if (block_size % ROC_ALIGN != 0) { 61 padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size; 62 mp->trailer_size += padding; 63 block_size += padding; 64 } 65 66 /* 67 * Marvell CN9k has 8 sets, 41 ways L1D cache, VA<9:7> bits dictate the 68 * set selection. Add additional padding to ensure that the element size 69 * always occupies odd number of cachelines to ensure even distribution 70 * of elements among L1D cache sets. 71 */ 72 padding = ((block_size / ROC_ALIGN) % 2) ? 0 : ROC_ALIGN; 73 mp->trailer_size += padding; 74 75 return cnxk_mempool_alloc(mp); 76 } 77 78 static struct rte_mempool_ops cn9k_mempool_ops = { 79 .name = "cn9k_mempool_ops", 80 .alloc = cn9k_mempool_alloc, 81 .free = cnxk_mempool_free, 82 .enqueue = cn9k_mempool_enq, 83 .dequeue = cn9k_mempool_deq, 84 .get_count = cnxk_mempool_get_count, 85 .calc_mem_size = cnxk_mempool_calc_mem_size, 86 .populate = cnxk_mempool_populate, 87 }; 88 89 RTE_MEMPOOL_REGISTER_OPS(cn9k_mempool_ops); 90