236 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
247 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
249 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
252 	cnt = ATOMIC32_READ(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
253 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
255 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
259 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
261 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
267 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
278 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
280 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
283 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
284 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
286 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
287 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
289 	ENA_DB_SYNC(&admin_queue->sq.mem_handle);  in __ena_com_submit_admin_cmd()
290 	ENA_REG_WRITE32(admin_queue->bus, admin_queue->sq.tail,  in __ena_com_submit_admin_cmd()
291 			admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
317 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
326 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
327 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
328 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
331 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
336 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
337 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
464 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
473 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
476 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
486 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
490 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
497 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
498 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
500 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
509 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
513 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
518 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
521 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
522 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
523 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
524 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
553 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
558 	timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
561                 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
562                 ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
563                 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
571 			ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
572 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
573 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
574 			ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
585 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
586 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
587 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
597 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
671 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
677 			    admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_interrupts()
685 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
686 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
687 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
688 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
697 		admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
704 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
783 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
785 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
787 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
790 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
796 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
816 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
908 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
918 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
940 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
1105 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1147 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1262 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1271 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1284 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1286 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1299 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1324 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1372 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1376 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1379 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1380 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1392 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1401 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1402 	while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1403 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1405 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1407 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1413 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1423 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1437 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1442 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1445 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1446 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1447 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1464 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1484 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1491 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1582 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1583 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1584 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1589 	ENA_SPINLOCK_DESTROY(admin_queue->q_lock);  in ena_com_admin_destroy()
1591 	if (admin_queue->comp_ctx) {  in ena_com_admin_destroy()
1594 		for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_admin_destroy()
1595 			struct ena_comp_ctx *comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_admin_destroy()
1600 		s = admin_queue->q_depth * sizeof(struct ena_comp_ctx);  in ena_com_admin_destroy()
1601 		ENA_MEM_FREE(ena_dev->dmadev, admin_queue->comp_ctx, s);  in ena_com_admin_destroy()
1603 	admin_queue->comp_ctx = NULL;  in ena_com_admin_destroy()
1604 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1610 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1631 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1697 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1717 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1719 	admin_queue->bus = ena_dev->bus;  in ena_com_admin_init()
1720 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1721 	admin_queue->polling = false;  in ena_com_admin_init()
1722 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1724 	ATOMIC32_SET(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1727 		ENA_SPINLOCK_INIT(admin_queue->q_lock);  in ena_com_admin_init()
1729 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1733 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
1737 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
1741 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
1744 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1745 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1750 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1751 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1757 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
1763 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
1774 	admin_queue->running_state = true;  in ena_com_admin_init()
1937 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
2088 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
2090 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
2101 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
2104 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
2110 	ret =  ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
2184 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
2195 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
2202 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
2234 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2279 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2398 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2431 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2558 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2597 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
2743 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
2754 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
2777 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()