xref: /onnv-gate/usr/src/uts/common/io/xge/hal/xgehal/xgehal-fifo.c (revision 6937:a5e2c8b5c817)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright (c) 2002-2006 Neterion, Inc.
 */

#include "xgehal-fifo.h"
#include "xgehal-device.h"

static xge_hal_status_e
__hal_fifo_mempool_item_alloc(xge_hal_mempool_h mempoolh,
			      void *memblock,
			      int memblock_index,
			      xge_hal_mempool_dma_t *dma_object,
			      void *item,
			      int index,
			      int is_last,
			      void *userdata)
{
	int memblock_item_idx;
	xge_hal_fifo_txdl_priv_t *txdl_priv;
	xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)item;
	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata;

	xge_assert(item);
	txdl_priv = (xge_hal_fifo_txdl_priv_t *) \
                __hal_mempool_item_priv((xge_hal_mempool_t *) mempoolh,
                                        memblock_index,
                                        item,
                                        &memblock_item_idx);
	xge_assert(txdl_priv);

	/* pre-format HAL's TxDL's private */
	txdl_priv->dma_offset = (char*)item - (char*)memblock;
	txdl_priv->dma_addr = dma_object->addr + txdl_priv->dma_offset;
	txdl_priv->dma_handle = dma_object->handle;
	txdl_priv->memblock   = memblock;
	txdl_priv->first_txdp = (xge_hal_fifo_txd_t *)item;
	txdl_priv->next_txdl_priv = NULL;
	txdl_priv->dang_txdl = NULL;
	txdl_priv->dang_frags = 0;
	txdl_priv->alloc_frags = 0;

#ifdef XGE_DEBUG_ASSERT
	txdl_priv->dma_object = dma_object;
#endif
	txdp->host_control = (u64)(ulong_t)txdl_priv;

#ifdef XGE_HAL_ALIGN_XMIT
	txdl_priv->align_vaddr = NULL;
	txdl_priv->align_dma_addr = (dma_addr_t)0;

#ifndef XGE_HAL_ALIGN_XMIT_ALLOC_RT
	{
	xge_hal_status_e status;
	if (fifo->config->alignment_size) {
	        status =__hal_fifo_dtr_align_alloc_map(fifo, txdp);
		if (status != XGE_HAL_OK)  {
		        xge_debug_mm(XGE_ERR,
		              "align buffer[%d] %d bytes, status %d",
			      index,
			      fifo->align_size,
			      status);
		        return status;
		}
	}
	}
#endif
#endif

	if (fifo->channel.dtr_init) {
		fifo->channel.dtr_init(fifo, (xge_hal_dtr_h)txdp, index,
			   fifo->channel.userdata, XGE_HAL_CHANNEL_OC_NORMAL);
	}

	return XGE_HAL_OK;
}


static xge_hal_status_e
__hal_fifo_mempool_item_free(xge_hal_mempool_h mempoolh,
			      void *memblock,
			      int memblock_index,
			      xge_hal_mempool_dma_t *dma_object,
			      void *item,
			      int index,
			      int is_last,
			      void *userdata)
{
	int memblock_item_idx;
	xge_hal_fifo_txdl_priv_t *txdl_priv;
#ifdef XGE_HAL_ALIGN_XMIT
	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata;
#endif

	xge_assert(item);

	txdl_priv = (xge_hal_fifo_txdl_priv_t *) \
                __hal_mempool_item_priv((xge_hal_mempool_t *) mempoolh,
                                        memblock_index,
                                        item,
                                        &memblock_item_idx);
	xge_assert(txdl_priv);

#ifdef XGE_HAL_ALIGN_XMIT
	if (fifo->config->alignment_size) {
		if (txdl_priv->align_dma_addr != 0) {
			xge_os_dma_unmap(fifo->channel.pdev,
			       txdl_priv->align_dma_handle,
			       txdl_priv->align_dma_addr,
			       fifo->align_size,
			       XGE_OS_DMA_DIR_TODEVICE);

			txdl_priv->align_dma_addr = 0;
		}

		if (txdl_priv->align_vaddr != NULL) {
			xge_os_dma_free(fifo->channel.pdev,
			      txdl_priv->align_vaddr,
			      fifo->align_size,
			      &txdl_priv->align_dma_acch,
			      &txdl_priv->align_dma_handle);

			txdl_priv->align_vaddr = NULL;
		}
	}
#endif

	return XGE_HAL_OK;
}

xge_hal_status_e
__hal_fifo_open(xge_hal_channel_h channelh, xge_hal_channel_attr_t *attr)
{
	xge_hal_device_t *hldev;
	xge_hal_status_e status;
	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
	xge_hal_fifo_queue_t *queue;
	int i, txdl_size, max_arr_index, mid_point;
	xge_hal_dtr_h  dtrh;

	hldev = (xge_hal_device_t *)fifo->channel.devh;
	fifo->config = &hldev->config.fifo;
	queue = &fifo->config->queue[attr->post_qid];

#if defined(XGE_HAL_TX_MULTI_RESERVE)
	xge_os_spin_lock_init(&fifo->channel.reserve_lock, hldev->pdev);
#elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ)
	xge_os_spin_lock_init_irq(&fifo->channel.reserve_lock, hldev->irqh);
#endif
#if defined(XGE_HAL_TX_MULTI_POST)
	if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA)  {
                fifo->post_lock_ptr = &hldev->xena_post_lock;
	} else {
	        xge_os_spin_lock_init(&fifo->channel.post_lock, hldev->pdev);
                fifo->post_lock_ptr = &fifo->channel.post_lock;
	}
#elif defined(XGE_HAL_TX_MULTI_POST_IRQ)
	if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA)  {
                fifo->post_lock_ptr = &hldev->xena_post_lock;
	} else {
	        xge_os_spin_lock_init_irq(&fifo->channel.post_lock,
					hldev->irqh);
                fifo->post_lock_ptr = &fifo->channel.post_lock;
	}
#endif

	fifo->align_size =
		fifo->config->alignment_size * fifo->config->max_aligned_frags;

	/* Initializing the BAR1 address as the start of
	 * the FIFO queue pointer and as a location of FIFO control
	 * word. */
	fifo->hw_pair =
	        (xge_hal_fifo_hw_pair_t *) (void *)(hldev->bar1 +
		        (attr->post_qid * XGE_HAL_FIFO_HW_PAIR_OFFSET));

	/* apply "interrupts per txdl" attribute */
	fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_UTILZ;
	if (queue->intr) {
		fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_PER_LIST;
	}
	fifo->no_snoop_bits =
		(int)(XGE_HAL_TX_FIFO_NO_SNOOP(queue->no_snoop_bits));

	/*
	 * FIFO memory management strategy:
	 *
	 * TxDL splitted into three independent parts:
	 *	- set of TxD's
	 *	- TxD HAL private part
	 *	- upper layer private part
	 *
	 * Adaptative memory allocation used. i.e. Memory allocated on
	 * demand with the size which will fit into one memory block.
	 * One memory block may contain more than one TxDL. In simple case
	 * memory block size can be equal to CPU page size. On more
	 * sophisticated OS's memory block can be contigious across
	 * several pages.
	 *
	 * During "reserve" operations more memory can be allocated on demand
	 * for example due to FIFO full condition.
	 *
	 * Pool of memory memblocks never shrinks except __hal_fifo_close
	 * routine which will essentially stop channel and free the resources.
	 */

	/* TxDL common private size == TxDL private + ULD private */
	fifo->priv_size = sizeof(xge_hal_fifo_txdl_priv_t) +
	attr->per_dtr_space;
	fifo->priv_size = ((fifo->priv_size + __xge_os_cacheline_size -1) /
                               __xge_os_cacheline_size) *
                               __xge_os_cacheline_size;

	/* recompute txdl size to be cacheline aligned */
	fifo->txdl_size = fifo->config->max_frags * sizeof(xge_hal_fifo_txd_t);
	txdl_size = ((fifo->txdl_size + __xge_os_cacheline_size - 1) /
			__xge_os_cacheline_size) * __xge_os_cacheline_size;

	if (fifo->txdl_size != txdl_size)
	        xge_debug_fifo(XGE_ERR, "cacheline > 128 ( ?? ): %d, %d, %d, %d",
		fifo->config->max_frags, fifo->txdl_size, txdl_size,
		__xge_os_cacheline_size);

	fifo->txdl_size = txdl_size;

	/* since dtr_init() callback will be called from item_alloc(),
	 * the same way channels userdata might be used prior to
	 * channel_initialize() */
	fifo->channel.dtr_init = attr->dtr_init;
	fifo->channel.userdata = attr->userdata;
	fifo->txdl_per_memblock = fifo->config->memblock_size /
		fifo->txdl_size;

	fifo->mempool = __hal_mempool_create(hldev->pdev,
					     fifo->config->memblock_size,
					     fifo->txdl_size,
					     fifo->priv_size,
					     queue->initial,
					     queue->max,
					     __hal_fifo_mempool_item_alloc,
					     __hal_fifo_mempool_item_free,
					     fifo);
	if (fifo->mempool == NULL) {
		return XGE_HAL_ERR_OUT_OF_MEMORY;
	}

	status = __hal_channel_initialize(channelh, attr,
					(void **) __hal_mempool_items_arr(fifo->mempool),
					queue->initial, queue->max,
					fifo->config->reserve_threshold);
	if (status != XGE_HAL_OK) {
		__hal_fifo_close(channelh);
		return status;
	}
	xge_debug_fifo(XGE_TRACE,
		"DTR  reserve_length:%d reserve_top:%d\n"
		"max_frags:%d reserve_threshold:%d\n"
		"memblock_size:%d alignment_size:%d max_aligned_frags:%d",
		fifo->channel.reserve_length, fifo->channel.reserve_top,
		fifo->config->max_frags, fifo->config->reserve_threshold,
		fifo->config->memblock_size, fifo->config->alignment_size,
		fifo->config->max_aligned_frags);

#ifdef XGE_DEBUG_ASSERT
	for ( i = 0; i < fifo->channel.reserve_length; i++) {
		xge_debug_fifo(XGE_TRACE, "DTR before reversing index:%d"
		" handle:%p", i, fifo->channel.reserve_arr[i]);
	}
#endif

	xge_assert(fifo->channel.reserve_length);
	/* reverse the FIFO dtr array */
	max_arr_index	= fifo->channel.reserve_length - 1;
	max_arr_index	-=fifo->channel.reserve_top;
	xge_assert(max_arr_index);
	mid_point = (fifo->channel.reserve_length - fifo->channel.reserve_top)/2;
	for (i = 0; i < mid_point; i++) {
		dtrh =	fifo->channel.reserve_arr[i];
		fifo->channel.reserve_arr[i] =
			fifo->channel.reserve_arr[max_arr_index - i];
		fifo->channel.reserve_arr[max_arr_index  - i] = dtrh;
	}

#ifdef XGE_DEBUG_ASSERT
	for ( i = 0; i < fifo->channel.reserve_length; i++) {
		xge_debug_fifo(XGE_TRACE, "DTR after reversing index:%d"
		" handle:%p", i, fifo->channel.reserve_arr[i]);
	}
#endif

	return XGE_HAL_OK;
}

void
__hal_fifo_close(xge_hal_channel_h channelh)
{
	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
	xge_hal_device_t *hldev = (xge_hal_device_t *)fifo->channel.devh;

	if (fifo->mempool) {
		__hal_mempool_destroy(fifo->mempool);
	}

	__hal_channel_terminate(channelh);

#if defined(XGE_HAL_TX_MULTI_RESERVE)
	xge_os_spin_lock_destroy(&fifo->channel.reserve_lock, hldev->pdev);
#elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ)
	xge_os_spin_lock_destroy_irq(&fifo->channel.reserve_lock, hldev->pdev);
#endif
	if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC)  {
#if defined(XGE_HAL_TX_MULTI_POST)
		xge_os_spin_lock_destroy(&fifo->channel.post_lock, hldev->pdev);
#elif defined(XGE_HAL_TX_MULTI_POST_IRQ)
		xge_os_spin_lock_destroy_irq(&fifo->channel.post_lock,
					     hldev->pdev);
#endif
	}
}

void
__hal_fifo_hw_initialize(xge_hal_device_h devh)
{
	xge_hal_device_t *hldev = (xge_hal_device_t *)devh;
	xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)(void *)hldev->bar0;
	u64* tx_fifo_partitions[4];
	u64* tx_fifo_wrr[5];
	u64  tx_fifo_wrr_value[5];
	u64 val64, part0;
	int i;

	/*  Tx DMA Initialization */

	tx_fifo_partitions[0] = &bar0->tx_fifo_partition_0;
	tx_fifo_partitions[1] = &bar0->tx_fifo_partition_1;
	tx_fifo_partitions[2] = &bar0->tx_fifo_partition_2;
	tx_fifo_partitions[3] = &bar0->tx_fifo_partition_3;

	tx_fifo_wrr[0] = &bar0->tx_w_round_robin_0;
	tx_fifo_wrr[1] = &bar0->tx_w_round_robin_1;
	tx_fifo_wrr[2] = &bar0->tx_w_round_robin_2;
	tx_fifo_wrr[3] = &bar0->tx_w_round_robin_3;
	tx_fifo_wrr[4] = &bar0->tx_w_round_robin_4;

	tx_fifo_wrr_value[0] = XGE_HAL_FIFO_WRR_0;
	tx_fifo_wrr_value[1] = XGE_HAL_FIFO_WRR_1;
	tx_fifo_wrr_value[2] = XGE_HAL_FIFO_WRR_2;
	tx_fifo_wrr_value[3] = XGE_HAL_FIFO_WRR_3;
	tx_fifo_wrr_value[4] = XGE_HAL_FIFO_WRR_4;

	/* Note: WRR calendar must be configured before the transmit
	 *       FIFOs are enabled! page 6-77 user guide */

	if (!hldev->config.rts_qos_en) {
		/* all zeroes for Round-Robin */
		for (i = 0; i < XGE_HAL_FIFO_MAX_WRR; i++) {
			xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 0,
					       tx_fifo_wrr[i]);
		}

		/* reset all of them but '0' */
		for (i=1; i < XGE_HAL_FIFO_MAX_PARTITION; i++) {
			xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 0ULL,
					       tx_fifo_partitions[i]);
		}
	} else { /* Change the default settings */

		for (i = 0; i < XGE_HAL_FIFO_MAX_WRR; i++) {
			xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
				       tx_fifo_wrr_value[i], tx_fifo_wrr[i]);
		}
	}

	/* configure only configured FIFOs */
	val64 = 0; part0 = 0;
	for (i = 0; i < XGE_HAL_MAX_FIFO_NUM; i++) {
		int reg_half = i % 2;
		int reg_num = i / 2;

		if (hldev->config.fifo.queue[i].configured) {
			int priority = hldev->config.fifo.queue[i].priority;
			val64 |=
			    vBIT((hldev->config.fifo.queue[i].max-1),
				(((reg_half) * 32) + 19),
				13) | vBIT(priority, (((reg_half)*32) + 5), 3);
		}

		/* NOTE: do write operation for each second u64 half
		 *       or force for first one if configured number
		 *	 is even */
		if (reg_half) {
			if (reg_num == 0) {
				/* skip partition '0', must write it once at
				 * the end */
				part0 = val64;
			} else {
				xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
				     val64, tx_fifo_partitions[reg_num]);
				xge_debug_fifo(XGE_TRACE,
					"fifo partition_%d at: "
					"0x"XGE_OS_LLXFMT" is: 0x"XGE_OS_LLXFMT,
					reg_num, (unsigned long long)(ulong_t)
					tx_fifo_partitions[reg_num],
					(unsigned long long)val64);
			}
			val64 = 0;
		}
	}

	part0 |= BIT(0); /* to enable the FIFO partition. */
	__hal_pio_mem_write32_lower(hldev->pdev, hldev->regh0, (u32)part0,
	                     tx_fifo_partitions[0]);
	xge_os_wmb();
	__hal_pio_mem_write32_upper(hldev->pdev, hldev->regh0, (u32)(part0>>32),
	                     tx_fifo_partitions[0]);
	xge_debug_fifo(XGE_TRACE, "fifo partition_0 at: "
			"0x"XGE_OS_LLXFMT" is: 0x"XGE_OS_LLXFMT,
			(unsigned long long)(ulong_t)
				tx_fifo_partitions[0],
			(unsigned long long) part0);

	/*
	 * Initialization of Tx_PA_CONFIG register to ignore packet
	 * integrity checking.
	 */
	val64 = xge_os_pio_mem_read64(hldev->pdev, hldev->regh0,
	                            &bar0->tx_pa_cfg);
	val64 |= XGE_HAL_TX_PA_CFG_IGNORE_FRM_ERR |
		 XGE_HAL_TX_PA_CFG_IGNORE_SNAP_OUI |
		 XGE_HAL_TX_PA_CFG_IGNORE_LLC_CTRL |
		 XGE_HAL_TX_PA_CFG_IGNORE_L2_ERR;
	xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64,
	                     &bar0->tx_pa_cfg);

	if (hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX)
		return;

	/*
	 * Assign MSI-X vectors
	 */
	for (i = 0; i < XGE_HAL_MAX_FIFO_NUM; i++) {
		xge_list_t *item;
		xge_hal_channel_t *channel = NULL;

		if (!hldev->config.fifo.queue[i].configured ||
		    !hldev->config.fifo.queue[i].intr_vector)
			continue;

		/* find channel */
		xge_list_for_each(item, &hldev->free_channels) {
			xge_hal_channel_t *tmp;
			tmp = xge_container_of(item, xge_hal_channel_t,
						   item);
			if (tmp->type == XGE_HAL_CHANNEL_TYPE_FIFO &&
			    tmp->post_qid == i) {
				channel = tmp;
				break;
			}
		}

		if (channel) {
			(void) xge_hal_channel_msix_set(channel,
				hldev->config.fifo.queue[i].intr_vector);
		}
	}

	xge_debug_fifo(XGE_TRACE, "%s", "fifo channels initialized");
}

#ifdef XGE_HAL_ALIGN_XMIT
void
__hal_fifo_dtr_align_free_unmap(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
{
        xge_hal_fifo_txdl_priv_t *txdl_priv;
	xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh;
	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;

	txdl_priv = __hal_fifo_txdl_priv(txdp);

	if (txdl_priv->align_dma_addr != 0) {
		xge_os_dma_unmap(fifo->channel.pdev,
		       txdl_priv->align_dma_handle,
		       txdl_priv->align_dma_addr,
		       fifo->align_size,
		       XGE_OS_DMA_DIR_TODEVICE);

                txdl_priv->align_dma_addr = 0;
	}

        if (txdl_priv->align_vaddr != NULL) {
	        xge_os_dma_free(fifo->channel.pdev,
	              txdl_priv->align_vaddr,
	              fifo->align_size,
	              &txdl_priv->align_dma_acch,
	              &txdl_priv->align_dma_handle);


	        txdl_priv->align_vaddr = NULL;
        }
 }

xge_hal_status_e
__hal_fifo_dtr_align_alloc_map(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
{
        xge_hal_fifo_txdl_priv_t *txdl_priv;
	xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh;
	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;

	xge_assert(txdp);

	txdl_priv = __hal_fifo_txdl_priv(txdp);

	/* allocate alignment DMA-buffer */
	txdl_priv->align_vaddr = (char *)xge_os_dma_malloc(fifo->channel.pdev,
				fifo->align_size,
				XGE_OS_DMA_CACHELINE_ALIGNED |
				XGE_OS_DMA_STREAMING,
				&txdl_priv->align_dma_handle,
				&txdl_priv->align_dma_acch);
	if (txdl_priv->align_vaddr == NULL) {
		return XGE_HAL_ERR_OUT_OF_MEMORY;
	}

	/* map it */
	txdl_priv->align_dma_addr = xge_os_dma_map(fifo->channel.pdev,
		txdl_priv->align_dma_handle, txdl_priv->align_vaddr,
		fifo->align_size,
		XGE_OS_DMA_DIR_TODEVICE, XGE_OS_DMA_STREAMING);

	if (txdl_priv->align_dma_addr == XGE_OS_INVALID_DMA_ADDR) {
                __hal_fifo_dtr_align_free_unmap(channelh, dtrh);
		return XGE_HAL_ERR_OUT_OF_MAPPING;
	}

	return XGE_HAL_OK;
}
#endif