225 			  (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |  in ath10k_ce_src_ring_byte_swap_set()
226 			  ath10k_set_ring_byte(n, ctrl_regs->src_ring));  in ath10k_ce_src_ring_byte_swap_set()
429 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_send_nolock()  local
431 	unsigned int nentries_mask = src_ring->nentries_mask;  in _ath10k_ce_send_nolock()
432 	unsigned int sw_index = src_ring->sw_index;  in _ath10k_ce_send_nolock()
433 	unsigned int write_index = src_ring->write_index;  in _ath10k_ce_send_nolock()
448 	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,  in _ath10k_ce_send_nolock()
464 	src_ring->per_transfer_context[write_index] = per_transfer_context;  in _ath10k_ce_send_nolock()
473 	src_ring->write_index = write_index;  in _ath10k_ce_send_nolock()
486 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_send_nolock_64()  local
488 	unsigned int nentries_mask = src_ring->nentries_mask;  in _ath10k_ce_send_nolock_64()
490 	unsigned int write_index = src_ring->write_index;  in _ath10k_ce_send_nolock_64()
506 		sw_index = src_ring->sw_index;  in _ath10k_ce_send_nolock_64()
514 	desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,  in _ath10k_ce_send_nolock_64()
540 	src_ring->per_transfer_context[write_index] = per_transfer_context;  in _ath10k_ce_send_nolock_64()
554 	src_ring->write_index = write_index;  in _ath10k_ce_send_nolock_64()
575 	struct ath10k_ce_ring *src_ring = pipe->src_ring;  in __ath10k_ce_send_revert()  local
585 	if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))  in __ath10k_ce_send_revert()
588 	if (WARN_ON_ONCE(src_ring->write_index ==  in __ath10k_ce_send_revert()
592 	src_ring->write_index--;  in __ath10k_ce_send_revert()
593 	src_ring->write_index &= src_ring->nentries_mask;  in __ath10k_ce_send_revert()
595 	src_ring->per_transfer_context[src_ring->write_index] = NULL;  in __ath10k_ce_send_revert()
626 	delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,  in ath10k_ce_num_free_src_entries()
627 			      pipe->src_ring->write_index,  in ath10k_ce_num_free_src_entries()
628 			      pipe->src_ring->sw_index - 1);  in ath10k_ce_num_free_src_entries()
1005 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_completed_send_next_nolock()  local
1008 	unsigned int nentries_mask = src_ring->nentries_mask;  in _ath10k_ce_completed_send_next_nolock()
1009 	unsigned int sw_index = src_ring->sw_index;  in _ath10k_ce_completed_send_next_nolock()
1013 	if (src_ring->hw_index == sw_index) {  in _ath10k_ce_completed_send_next_nolock()
1027 		src_ring->hw_index = read_index;  in _ath10k_ce_completed_send_next_nolock()
1033 		read_index = src_ring->hw_index;  in _ath10k_ce_completed_send_next_nolock()
1040 			src_ring->per_transfer_context[sw_index];  in _ath10k_ce_completed_send_next_nolock()
1043 	src_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_completed_send_next_nolock()
1044 	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,  in _ath10k_ce_completed_send_next_nolock()
1050 	src_ring->sw_index = sw_index;  in _ath10k_ce_completed_send_next_nolock()
1058 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in _ath10k_ce_completed_send_next_nolock_64()  local
1061 	unsigned int nentries_mask = src_ring->nentries_mask;  in _ath10k_ce_completed_send_next_nolock_64()
1062 	unsigned int sw_index = src_ring->sw_index;  in _ath10k_ce_completed_send_next_nolock_64()
1066 	if (src_ring->hw_index == sw_index) {  in _ath10k_ce_completed_send_next_nolock_64()
1080 		src_ring->hw_index = read_index;  in _ath10k_ce_completed_send_next_nolock_64()
1086 		read_index = src_ring->hw_index;  in _ath10k_ce_completed_send_next_nolock_64()
1093 			src_ring->per_transfer_context[sw_index];  in _ath10k_ce_completed_send_next_nolock_64()
1096 	src_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_completed_send_next_nolock_64()
1097 	desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,  in _ath10k_ce_completed_send_next_nolock_64()
1103 	src_ring->sw_index = sw_index;  in _ath10k_ce_completed_send_next_nolock_64()
1117 					struct ath10k_ce_ring *src_ring,  in ath10k_ce_extract_desc_data()  argument
1123 		struct ce_desc *base = src_ring->base_addr_owner_space;  in ath10k_ce_extract_desc_data()
1134 					   struct ath10k_ce_ring *src_ring,  in ath10k_ce_extract_desc_data_64()  argument
1140 		struct ce_desc_64 *base = src_ring->base_addr_owner_space;  in ath10k_ce_extract_desc_data_64()
1158 	struct ath10k_ce_ring *src_ring;  in ath10k_ce_cancel_send_next()  local
1166 	src_ring = ce_state->src_ring;  in ath10k_ce_cancel_send_next()
1168 	if (!src_ring)  in ath10k_ce_cancel_send_next()
1176 	nentries_mask = src_ring->nentries_mask;  in ath10k_ce_cancel_send_next()
1177 	sw_index = src_ring->sw_index;  in ath10k_ce_cancel_send_next()
1178 	write_index = src_ring->write_index;  in ath10k_ce_cancel_send_next()
1181 		ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,  in ath10k_ce_cancel_send_next()
1187 				src_ring->per_transfer_context[sw_index];  in ath10k_ce_cancel_send_next()
1190 		src_ring->per_transfer_context[sw_index] = NULL;  in ath10k_ce_cancel_send_next()
1194 		src_ring->sw_index = sw_index;  in ath10k_ce_cancel_send_next()
1361 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in ath10k_ce_init_src_ring()  local
1367 		memset(src_ring->base_addr_owner_space, 0,  in ath10k_ce_init_src_ring()
1370 		memset(src_ring->base_addr_owner_space, 0,  in ath10k_ce_init_src_ring()
1373 	src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);  in ath10k_ce_init_src_ring()
1374 	src_ring->sw_index &= src_ring->nentries_mask;  in ath10k_ce_init_src_ring()
1375 	src_ring->hw_index = src_ring->sw_index;  in ath10k_ce_init_src_ring()
1377 	src_ring->write_index =  in ath10k_ce_init_src_ring()
1379 	src_ring->write_index &= src_ring->nentries_mask;  in ath10k_ce_init_src_ring()
1382 					 src_ring->base_addr_ce_space);  in ath10k_ce_init_src_ring()
1391 		   ce_id, nentries, src_ring->base_addr_owner_space);  in ath10k_ce_init_src_ring()
1435 				       struct ath10k_ce_ring *src_ring,  in ath10k_ce_alloc_shadow_base()  argument
1438 	src_ring->shadow_base_unaligned = kcalloc(nentries,  in ath10k_ce_alloc_shadow_base()
1441 	if (!src_ring->shadow_base_unaligned)  in ath10k_ce_alloc_shadow_base()
1444 	src_ring->shadow_base = (struct ce_desc_64 *)  in ath10k_ce_alloc_shadow_base()
1445 			PTR_ALIGN(src_ring->shadow_base_unaligned,  in ath10k_ce_alloc_shadow_base()
1454 	struct ath10k_ce_ring *src_ring;  in ath10k_ce_alloc_src_ring()  local
1461 	src_ring = kzalloc(struct_size(src_ring, per_transfer_context,  in ath10k_ce_alloc_src_ring()
1463 	if (src_ring == NULL)  in ath10k_ce_alloc_src_ring()
1466 	src_ring->nentries = nentries;  in ath10k_ce_alloc_src_ring()
1467 	src_ring->nentries_mask = nentries - 1;  in ath10k_ce_alloc_src_ring()
1473 	src_ring->base_addr_owner_space_unaligned =  in ath10k_ce_alloc_src_ring()
1478 	if (!src_ring->base_addr_owner_space_unaligned) {  in ath10k_ce_alloc_src_ring()
1479 		kfree(src_ring);  in ath10k_ce_alloc_src_ring()
1483 	src_ring->base_addr_ce_space_unaligned = base_addr;  in ath10k_ce_alloc_src_ring()
1485 	src_ring->base_addr_owner_space =  in ath10k_ce_alloc_src_ring()
1486 			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_src_ring()
1488 	src_ring->base_addr_ce_space =  in ath10k_ce_alloc_src_ring()
1489 			ALIGN(src_ring->base_addr_ce_space_unaligned,  in ath10k_ce_alloc_src_ring()
1493 		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);  in ath10k_ce_alloc_src_ring()
1498 					  src_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_src_ring()
1500 			kfree(src_ring);  in ath10k_ce_alloc_src_ring()
1505 	return src_ring;  in ath10k_ce_alloc_src_ring()
1512 	struct ath10k_ce_ring *src_ring;  in ath10k_ce_alloc_src_ring_64()  local
1519 	src_ring = kzalloc(struct_size(src_ring, per_transfer_context,  in ath10k_ce_alloc_src_ring_64()
1521 	if (!src_ring)  in ath10k_ce_alloc_src_ring_64()
1524 	src_ring->nentries = nentries;  in ath10k_ce_alloc_src_ring_64()
1525 	src_ring->nentries_mask = nentries - 1;  in ath10k_ce_alloc_src_ring_64()
1530 	src_ring->base_addr_owner_space_unaligned =  in ath10k_ce_alloc_src_ring_64()
1535 	if (!src_ring->base_addr_owner_space_unaligned) {  in ath10k_ce_alloc_src_ring_64()
1536 		kfree(src_ring);  in ath10k_ce_alloc_src_ring_64()
1540 	src_ring->base_addr_ce_space_unaligned = base_addr;  in ath10k_ce_alloc_src_ring_64()
1542 	src_ring->base_addr_owner_space =  in ath10k_ce_alloc_src_ring_64()
1543 			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_src_ring_64()
1545 	src_ring->base_addr_ce_space =  in ath10k_ce_alloc_src_ring_64()
1546 			ALIGN(src_ring->base_addr_ce_space_unaligned,  in ath10k_ce_alloc_src_ring_64()
1550 		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);  in ath10k_ce_alloc_src_ring_64()
1555 					  src_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_src_ring_64()
1557 			kfree(src_ring);  in ath10k_ce_alloc_src_ring_64()
1562 	return src_ring;  in ath10k_ce_alloc_src_ring_64()
1720 	if (ce_state->src_ring) {  in _ath10k_ce_free_pipe()
1722 			kfree(ce_state->src_ring->shadow_base_unaligned);  in _ath10k_ce_free_pipe()
1724 				  (ce_state->src_ring->nentries *  in _ath10k_ce_free_pipe()
1727 				  ce_state->src_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe()
1728 				  ce_state->src_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe()
1729 		kfree(ce_state->src_ring);  in _ath10k_ce_free_pipe()
1742 	ce_state->src_ring = NULL;  in _ath10k_ce_free_pipe()
1751 	if (ce_state->src_ring) {  in _ath10k_ce_free_pipe_64()
1753 			kfree(ce_state->src_ring->shadow_base_unaligned);  in _ath10k_ce_free_pipe_64()
1755 				  (ce_state->src_ring->nentries *  in _ath10k_ce_free_pipe_64()
1758 				  ce_state->src_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe_64()
1759 				  ce_state->src_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe_64()
1760 		kfree(ce_state->src_ring);  in _ath10k_ce_free_pipe_64()
1773 	ce_state->src_ring = NULL;  in _ath10k_ce_free_pipe_64()
1905 		ce_state->src_ring =  in ath10k_ce_alloc_pipe()
1907 		if (IS_ERR(ce_state->src_ring)) {  in ath10k_ce_alloc_pipe()
1908 			ret = PTR_ERR(ce_state->src_ring);  in ath10k_ce_alloc_pipe()
1911 			ce_state->src_ring = NULL;  in ath10k_ce_alloc_pipe()