76 		/* Max Txqs: Half for each of the two ports :max_iq/2 */
82 		/* Max Rxqs: Half for each of the two ports :max_oq/2  */
88 		/* Num of desc for rx rings */
106 		/* Max Txqs: Half for each of the two ports :max_iq/2 */
112 		/* Max Rxqs: Half for each of the two ports :max_oq/2  */
118 		/* Num of desc for rx rings */
165 	"BEGIN", "PCI-ENABLE-DONE", "PCI-MAP-DONE", "DISPATCH-INIT-DONE",
166 	"IQ-INIT-DONE", "SCBUFF-POOL-INIT-DONE", "RESPLIST-INIT-DONE",
167 	"DROQ-INIT-DONE", "MBOX-SETUP-DONE", "MSIX-ALLOC-VECTOR-DONE",
168 	"INTR-SET-DONE", "IO-QUEUES-INIT-DONE", "CONSOLE-INIT-DONE",
169 	"HOST-READY", "CORE-READY", "RUNNING", "IN-RESET",
188 	if (conf_type < 0 || conf_type > (LIO_NUM_CFGS - 1))  in oct_set_config_info()
208 	uint32_t	oct_id = oct->octeon_id;  in __lio_retrieve_config_info()
212 		if (oct->chip_id == LIO_CN23XX_PF_VID) {  in __lio_retrieve_config_info()
251 		return (lio_app_str[app_mode - LIO_DRV_APP_START]);  in lio_get_app_string()
253 	return (lio_app_str[LIO_DRV_INVALID_APP - LIO_DRV_APP_START]);  in lio_get_app_string()
262 		if ((oct->io_qmask.oq & BIT_ULL(i)) && (oct->droq[i]))  in lio_free_device_mem()
263 			free(oct->droq[i], M_DEVBUF);  in lio_free_device_mem()
267 		if ((oct->io_qmask.iq & BIT_ULL(i)) && (oct->instr_queue[i]))  in lio_free_device_mem()
268 			free(oct->instr_queue[i], M_DEVBUF);  in lio_free_device_mem()
271 	i = oct->octeon_id;  in lio_free_device_mem()
272 	free(oct->chip, M_DEVBUF);  in lio_free_device_mem()
275 	octeon_device_count--;  in lio_free_device_mem()
297 		configsize += (8 - (configsize & 0x7));  in lio_allocate_device_mem()
307 	oct->chip = (void *)(buf);  in lio_allocate_device_mem()
308 	oct->dispatch.dlist = (struct lio_dispatch *)(buf + configsize);  in lio_allocate_device_mem()
338 	mtx_init(&oct->pci_win_lock, "pci_win_lock", NULL, MTX_DEF);  in lio_allocate_device()
339 	mtx_init(&oct->mem_access_lock, "mem_access_lock", NULL, MTX_DEF);  in lio_allocate_device()
341 	oct->octeon_id = oct_idx;  in lio_allocate_device()
342 	snprintf(oct->device_name, sizeof(oct->device_name), "%s%d",  in lio_allocate_device()
343 		 LIO_DRV_NAME, oct->octeon_id);  in lio_allocate_device()
350  *  @param oct        - pointer to the octeon device structure.
351  *  @param bus        - PCIe bus #
352  *  @param dev        - PCIe device #
353  *  @param func       - PCIe function #
354  *  @param is_pf      - TRUE for PF, FALSE for VF
363 	oct->loc.bus = bus;  in lio_register_device()
364 	oct->loc.dev = dev;  in lio_register_device()
365 	oct->loc.func = func;  in lio_register_device()
367 	oct->adapter_refcount = &lio_adapter_refcounts[oct->octeon_id];  in lio_register_device()
368 	atomic_store_rel_int(oct->adapter_refcount, 0);  in lio_register_device()
371 	for (idx = (int)oct->octeon_id - 1; idx >= 0; idx--) {  in lio_register_device()
376 			atomic_add_int(oct->adapter_refcount, 1);  in lio_register_device()
381 		if ((octeon_device[idx]->loc.bus == bus) &&  in lio_register_device()
382 		    (octeon_device[idx]->loc.dev == dev)) {  in lio_register_device()
383 			oct->adapter_refcount =  in lio_register_device()
384 				octeon_device[idx]->adapter_refcount;  in lio_register_device()
391 	atomic_add_int(oct->adapter_refcount, 1);  in lio_register_device()
392 	refcount = atomic_load_acq_int(oct->adapter_refcount);  in lio_register_device()
395 		    oct->loc.bus, oct->loc.dev, oct->loc.func, refcount);  in lio_register_device()
401  *  Deregister a device at de-initialization time.
402  *  @param oct - pointer to the octeon device structure.
410 	atomic_subtract_int(oct->adapter_refcount, 1);  in lio_deregister_device()
411 	refcount = atomic_load_acq_int(oct->adapter_refcount);  in lio_deregister_device()
414 		    oct->loc.bus, oct->loc.dev, oct->loc.func, refcount);  in lio_deregister_device()
426 		num_ioqs = oct->sriov_info.num_pf_rings;  in lio_allocate_ioq_vector()
430 	oct->ioq_vector = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);  in lio_allocate_ioq_vector()
431 	if (oct->ioq_vector == NULL)  in lio_allocate_ioq_vector()
435 		ioq_vector = &oct->ioq_vector[i];  in lio_allocate_ioq_vector()
436 		ioq_vector->oct_dev = oct;  in lio_allocate_ioq_vector()
437 		ioq_vector->droq_index = i;  in lio_allocate_ioq_vector()
439 		CPU_SETOF(cpu_num, &ioq_vector->affinity_mask);  in lio_allocate_ioq_vector()
441 		if (oct->chip_id == LIO_CN23XX_PF_VID)  in lio_allocate_ioq_vector()
442 			ioq_vector->ioq_num = i + oct->sriov_info.pf_srn;  in lio_allocate_ioq_vector()
444 			ioq_vector->ioq_num = i;  in lio_allocate_ioq_vector()
453 	free(oct->ioq_vector, M_DEVBUF);  in lio_free_ioq_vector()
454 	oct->ioq_vector = NULL;  in lio_free_ioq_vector()
470 	oct->num_iqs = 0;  in lio_setup_instr_queue0()
472 	oct->instr_queue[0]->q_index = 0;  in lio_setup_instr_queue0()
473 	oct->instr_queue[0]->app_ctx = (void *)(size_t)0;  in lio_setup_instr_queue0()
474 	oct->instr_queue[0]->ifidx = 0;  in lio_setup_instr_queue0()
477 	txpciq.s.pkind = oct->pfvf_hsword.pkind;  in lio_setup_instr_queue0()
486 	oct->num_iqs++;  in lio_setup_instr_queue0()
504 	oct->num_oqs = 0;  in lio_setup_output_queue0()
510 	oct->num_oqs++;  in lio_setup_output_queue0()
520 	oct->dispatch.count = 0;  in lio_init_dispatch_list()
523 		oct->dispatch.dlist[i].opcode = 0;  in lio_init_dispatch_list()
524 		STAILQ_INIT(&oct->dispatch.dlist[i].head);  in lio_init_dispatch_list()
527 	mtx_init(&oct->dispatch.lock, "dispatch_lock", NULL, MTX_DEF);  in lio_init_dispatch_list()
541 	mtx_lock(&oct->dispatch.lock);  in lio_delete_dispatch_list()
546 		dispatch = &oct->dispatch.dlist[i].head;  in lio_delete_dispatch_list()
549 			STAILQ_REMOVE_HEAD(&oct->dispatch.dlist[i].head,  in lio_delete_dispatch_list()
554 		oct->dispatch.dlist[i].opcode = 0;  in lio_delete_dispatch_list()
557 	oct->dispatch.count = 0;  in lio_delete_dispatch_list()
559 	mtx_unlock(&oct->dispatch.lock);  in lio_delete_dispatch_list()
578 	mtx_lock(&octeon_dev->dispatch.lock);  in lio_get_dispatch()
580 	if (octeon_dev->dispatch.count == 0) {  in lio_get_dispatch()
581 		mtx_unlock(&octeon_dev->dispatch.lock);  in lio_get_dispatch()
585 	if (!(octeon_dev->dispatch.dlist[idx].opcode)) {  in lio_get_dispatch()
586 		mtx_unlock(&octeon_dev->dispatch.lock);  in lio_get_dispatch()
590 	if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {  in lio_get_dispatch()
591 		fn = octeon_dev->dispatch.dlist[idx].dispatch_fn;  in lio_get_dispatch()
593 		STAILQ_FOREACH(dispatch, &octeon_dev->dispatch.dlist[idx].head,  in lio_get_dispatch()
595 			if (((struct lio_dispatch *)dispatch)->opcode ==  in lio_get_dispatch()
598 				      dispatch)->dispatch_fn;  in lio_get_dispatch()
604 	mtx_unlock(&octeon_dev->dispatch.lock);  in lio_get_dispatch()
611  *   octeon_id - id of the octeon device.
612  *   opcode    - opcode for which driver should call the registered function
613  *   subcode   - subcode for which driver should call the registered function
614  *   fn        - The function to call when a packet with "opcode" arrives in
616  *   fn_arg    - The argument to be passed when calling function "fn".
636 	mtx_lock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
638 	if (oct->dispatch.dlist[idx].opcode == 0) {  in lio_register_dispatch_fn()
639 		oct->dispatch.dlist[idx].opcode = combined_opcode;  in lio_register_dispatch_fn()
640 		oct->dispatch.dlist[idx].dispatch_fn = fn;  in lio_register_dispatch_fn()
641 		oct->dispatch.dlist[idx].arg = fn_arg;  in lio_register_dispatch_fn()
642 		oct->dispatch.count++;  in lio_register_dispatch_fn()
643 		mtx_unlock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
647 	mtx_unlock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
668 		dispatch->opcode = combined_opcode;  in lio_register_dispatch_fn()
669 		dispatch->dispatch_fn = fn;  in lio_register_dispatch_fn()
670 		dispatch->arg = fn_arg;  in lio_register_dispatch_fn()
676 		mtx_lock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
677 		STAILQ_INSERT_HEAD(&oct->dispatch.dlist[idx].head,  in lio_register_dispatch_fn()
678 				   &dispatch->node, entries);  in lio_register_dispatch_fn()
679 		oct->dispatch.count++;  in lio_register_dispatch_fn()
680 		mtx_unlock(&oct->dispatch.lock);  in lio_register_dispatch_fn()
694  *   oct       - octeon device
695  *   opcode    - driver should unregister the function for this opcode
696  *   subcode   - driver should unregister the function for this subcode
717 	mtx_lock(&oct->dispatch.lock);  in lio_unregister_dispatch_fn()
719 	if (oct->dispatch.count == 0) {  in lio_unregister_dispatch_fn()
720 		mtx_unlock(&oct->dispatch.lock);  in lio_unregister_dispatch_fn()
724 	if (oct->dispatch.dlist[idx].opcode == combined_opcode) {  in lio_unregister_dispatch_fn()
725 		dispatch_head = &oct->dispatch.dlist[idx].head;  in lio_unregister_dispatch_fn()
728 			oct->dispatch.dlist[idx].opcode =  in lio_unregister_dispatch_fn()
729 			    ((struct lio_dispatch *)dispatch)->opcode;  in lio_unregister_dispatch_fn()
730 			oct->dispatch.dlist[idx].dispatch_fn =  in lio_unregister_dispatch_fn()
731 			    ((struct lio_dispatch *)dispatch)->dispatch_fn;  in lio_unregister_dispatch_fn()
732 			oct->dispatch.dlist[idx].arg =  in lio_unregister_dispatch_fn()
733 			    ((struct lio_dispatch *)dispatch)->arg;  in lio_unregister_dispatch_fn()
737 			oct->dispatch.dlist[idx].opcode = 0;  in lio_unregister_dispatch_fn()
738 			oct->dispatch.dlist[idx].dispatch_fn = NULL;  in lio_unregister_dispatch_fn()
739 			oct->dispatch.dlist[idx].arg = NULL;  in lio_unregister_dispatch_fn()
744 				    &oct->dispatch.dlist[idx].head,  in lio_unregister_dispatch_fn()
746 			if (((struct lio_dispatch *)dispatch)->opcode ==  in lio_unregister_dispatch_fn()
748 				STAILQ_REMOVE(&oct->dispatch.dlist[idx].head,  in lio_unregister_dispatch_fn()
758 		oct->dispatch.count--;  in lio_unregister_dispatch_fn()
760 	mtx_unlock(&oct->dispatch.lock);  in lio_unregister_dispatch_fn()
770 	struct lio_recv_pkt	*recv_pkt = recv_info->recv_pkt;  in lio_core_drv_init()
780 	if (atomic_load_acq_int(&oct->status) >= LIO_DEV_RUNNING) {  in lio_core_drv_init()
782 			    atomic_load_acq_int(&oct->status));  in lio_core_drv_init()
788 				   recv_pkt->rh.r_core_drv_init.app_mode),  in lio_core_drv_init()
789 		sizeof(app_name) - 1);  in lio_core_drv_init()
790 	oct->app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;  in lio_core_drv_init()
791 	if (recv_pkt->rh.r_core_drv_init.app_mode == LIO_DRV_NIC_APP) {  in lio_core_drv_init()
792 		oct->fw_info.max_nic_ports =  in lio_core_drv_init()
793 		    (uint32_t)recv_pkt->rh.r_core_drv_init.max_nic_ports;  in lio_core_drv_init()
794 		oct->fw_info.num_gmx_ports =  in lio_core_drv_init()
795 		    (uint32_t)recv_pkt->rh.r_core_drv_init.num_gmx_ports;  in lio_core_drv_init()
798 	if (oct->fw_info.max_nic_ports < num_nic_ports) {  in lio_core_drv_init()
800 			    num_nic_ports, oct->fw_info.max_nic_ports);  in lio_core_drv_init()
804 	oct->fw_info.app_cap_flags = recv_pkt->rh.r_core_drv_init.app_cap_flags;  in lio_core_drv_init()
805 	oct->fw_info.app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;  in lio_core_drv_init()
806 	oct->pfvf_hsword.app_mode =  in lio_core_drv_init()
807 	    (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;  in lio_core_drv_init()
809 	oct->pfvf_hsword.pkind = recv_pkt->rh.r_core_drv_init.pkind;  in lio_core_drv_init()
811 	for (i = 0; i < oct->num_iqs; i++)  in lio_core_drv_init()
812 		oct->instr_queue[i]->txpciq.s.pkind = oct->pfvf_hsword.pkind;  in lio_core_drv_init()
814 	atomic_store_rel_int(&oct->status, LIO_DEV_CORE_OK);  in lio_core_drv_init()
816 	cs = &core_setup[oct->octeon_id];  in lio_core_drv_init()
818 	if (recv_pkt->buffer_size[0] != (sizeof(*cs) + LIO_DROQ_INFO_SIZE)) {  in lio_core_drv_init()
821 			    recv_pkt->buffer_size[0]);  in lio_core_drv_init()
824 	memcpy(cs, recv_pkt->buffer_ptr[0]->m_data + LIO_DROQ_INFO_SIZE,  in lio_core_drv_init()
826 	strncpy(oct->boardinfo.name, cs->boardname, LIO_BOARD_NAME);  in lio_core_drv_init()
827 	strncpy(oct->boardinfo.serial_number, cs->board_serial_number,  in lio_core_drv_init()
832 	oct->boardinfo.major = cs->board_rev_major;  in lio_core_drv_init()
833 	oct->boardinfo.minor = cs->board_rev_minor;  in lio_core_drv_init()
836 		     LIO_CAST64(cs->corefreq));  in lio_core_drv_init()
839 	for (i = 0; i < recv_pkt->buffer_count; i++)  in lio_core_drv_init()
840 		lio_recv_buffer_free(recv_pkt->buffer_ptr[i]);  in lio_core_drv_init()
851 	    (oct->io_qmask.iq & BIT_ULL(q_no)))  in lio_get_tx_qsize()
852 		return (oct->instr_queue[q_no]->max_count);  in lio_get_tx_qsize()
855 	return (-1);  in lio_get_tx_qsize()
863 	    (oct->io_qmask.oq & BIT_ULL(q_no)))  in lio_get_rx_qsize()
864 		return (oct->droq[q_no]->max_count);  in lio_get_rx_qsize()
866 	return (-1);  in lio_get_rx_qsize()
889  *  @param octeon_id  - The id for which the octeon device pointer is required.
909 	mtx_lock(&oct->pci_win_lock);  in lio_pci_readq()
916 	if (oct->chip_id == LIO_CN23XX_PF_VID)  in lio_pci_readq()
918 	lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_hi, addrhi);  in lio_pci_readq()
921 	(void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_hi);  in lio_pci_readq()
923 	lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_lo,  in lio_pci_readq()
925 	(void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_lo);  in lio_pci_readq()
927 	val64 = lio_read_csr64(oct, oct->reg_list.pci_win_rd_data);  in lio_pci_readq()
929 	mtx_unlock(&oct->pci_win_lock);  in lio_pci_readq()
938 	mtx_lock(&oct->pci_win_lock);  in lio_pci_writeq()
940 	lio_write_csr64(oct, oct->reg_list.pci_win_wr_addr, addr);  in lio_pci_writeq()
943 	lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_hi, val >> 32);  in lio_pci_writeq()
945 	(void)lio_read_csr32(oct, oct->reg_list.pci_win_wr_data_hi);  in lio_pci_writeq()
947 	lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_lo,  in lio_pci_writeq()
950 	mtx_unlock(&oct->pci_win_lock);  in lio_pci_writeq()
970 lio_wait_for_ddr_init(struct octeon_device *oct, unsigned long *timeout)  in lio_wait_for_ddr_init()  argument
973 	uint32_t	ms;  in lio_wait_for_ddr_init()  local
975 	if (timeout == NULL)  in lio_wait_for_ddr_init()
978 	for (ms = 0; ret && ((*timeout == 0) || (ms <= *timeout)); ms += 100) {  in lio_wait_for_ddr_init()
981 		/* wait 100 ms */  in lio_wait_for_ddr_init()
992  *  @param dev - octeon device pointer passed as a void *.
1003 			return (octeon_dev->octeon_id);  in lio_get_device_id()
1005 	return (-1);  in lio_get_device_id()
1017 		oct = droq->oct_dev;  in lio_enable_irq()
1018 		pkts_pend = atomic_load_acq_int(&droq->pkts_pending);  in lio_enable_irq()
1019 		mtx_lock(&droq->lock);  in lio_enable_irq()
1020 		lio_write_csr32(oct, droq->pkts_sent_reg,  in lio_enable_irq()
1021 				droq->pkt_count - pkts_pend);  in lio_enable_irq()
1022 		droq->pkt_count = pkts_pend;  in lio_enable_irq()
1025 		mtx_unlock(&droq->lock);  in lio_enable_irq()
1029 		oct = iq->oct_dev;  in lio_enable_irq()
1030 		mtx_lock(&iq->lock);  in lio_enable_irq()
1031 		lio_write_csr32(oct, iq->inst_cnt_reg, iq->pkt_in_done);  in lio_enable_irq()
1032 		iq->pkt_in_done = 0;  in lio_enable_irq()
1035 		mtx_unlock(&iq->lock);  in lio_enable_irq()
1049 			lio_write_csr64(oct, droq->pkts_sent_reg,  in lio_enable_irq()
1054 			instr_cnt = lio_read_csr64(oct, iq->inst_cnt_reg);  in lio_enable_irq()
1055 			lio_write_csr64(oct, iq->inst_cnt_reg,  in lio_enable_irq()