io/ioat/ioat.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/errno.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/open.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sysmacros.h>

#include <sys/ioat.h>

static int ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred);
static int ioat_close(dev_t devp, int flag, int otyp, cred_t *cred);
static int ioat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
static int ioat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
static int ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
    void **result);
static int ioat_quiesce(dev_info_t *dip);

static 	struct cb_ops ioat_cb_ops = {
	ioat_open,		/* cb_open */
	ioat_close,		/* cb_close */
	nodev,			/* cb_strategy */
	nodev,			/* cb_print */
	nodev,			/* cb_dump */
	nodev,			/* cb_read */
	nodev,			/* cb_write */
	ioat_ioctl,		/* cb_ioctl */
	nodev,			/* cb_devmap */
	nodev,			/* cb_mmap */
	nodev,			/* cb_segmap */
	nochpoll,		/* cb_chpoll */
	ddi_prop_op,		/* cb_prop_op */
	NULL,			/* cb_stream */
	D_NEW | D_MP | D_64BIT | D_DEVMAP,	/* cb_flag */
	CB_REV
};

static struct dev_ops ioat_dev_ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt */
	ioat_getinfo,		/* devo_getinfo */
	nulldev,		/* devo_identify */
	nulldev,		/* devo_probe */
	ioat_attach,		/* devo_attach */
	ioat_detach,		/* devo_detach */
	nodev,			/* devo_reset */
	&ioat_cb_ops,		/* devo_cb_ops */
	NULL,			/* devo_bus_ops */
	NULL,			/* devo_power */
	ioat_quiesce,		/* devo_quiesce */
};

static struct modldrv ioat_modldrv = {
	&mod_driverops,		/* Type of module.  This one is a driver */
	"ioat driver",		/* Name of the module. */
	&ioat_dev_ops,		/* driver ops */
};

static struct modlinkage ioat_modlinkage = {
	MODREV_1,
	(void *) &ioat_modldrv,
	NULL
};


void *ioat_statep;

static int ioat_chip_init(ioat_state_t *state);
static void ioat_chip_fini(ioat_state_t *state);
static int ioat_drv_init(ioat_state_t *state);
static void ioat_drv_fini(ioat_state_t *state);
static uint_t ioat_isr(caddr_t parm);
static void ioat_intr_enable(ioat_state_t *state);
static void ioat_intr_disable(ioat_state_t *state);
void ioat_detach_finish(ioat_state_t *state);


ddi_device_acc_attr_t ioat_acc_attr = {
	DDI_DEVICE_ATTR_V0,		/* devacc_attr_version */
	DDI_NEVERSWAP_ACC,		/* devacc_attr_endian_flags */
	DDI_STORECACHING_OK_ACC,	/* devacc_attr_dataorder */
	DDI_DEFAULT_ACC			/* devacc_attr_access */
};

/* dcopy callback interface */
dcopy_device_cb_t ioat_cb = {
	DCOPY_DEVICECB_V0,
	0,		/* reserved */
	ioat_channel_alloc,
	ioat_channel_free,
	ioat_cmd_alloc,
	ioat_cmd_free,
	ioat_cmd_post,
	ioat_cmd_poll,
	ioat_unregister_complete
};

/*
 * _init()
 */
int
_init(void)
{
	int e;

	e = ddi_soft_state_init(&ioat_statep, sizeof (ioat_state_t), 1);
	if (e != 0) {
		return (e);
	}

	e = mod_install(&ioat_modlinkage);
	if (e != 0) {
		ddi_soft_state_fini(&ioat_statep);
		return (e);
	}

	return (0);
}

/*
 * _info()
 */
int
_info(struct modinfo *modinfop)
{
	return (mod_info(&ioat_modlinkage, modinfop));
}

/*
 * _fini()
 */
int
_fini(void)
{
	int e;

	e = mod_remove(&ioat_modlinkage);
	if (e != 0) {
		return (e);
	}

	ddi_soft_state_fini(&ioat_statep);

	return (0);
}

/*
 * ioat_attach()
 */
static int
ioat_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	ioat_state_t *state;
	int instance;
	int e;


	switch (cmd) {
	case DDI_ATTACH:
		break;

	case DDI_RESUME:
		instance = ddi_get_instance(dip);
		state = ddi_get_soft_state(ioat_statep, instance);
		if (state == NULL) {
			return (DDI_FAILURE);
		}
		e = ioat_channel_resume(state);
		if (e != DDI_SUCCESS) {
			return (DDI_FAILURE);
		}
		ioat_intr_enable(state);
		return (DDI_SUCCESS);

	default:
		return (DDI_FAILURE);
	}

	instance = ddi_get_instance(dip);
	e = ddi_soft_state_zalloc(ioat_statep, instance);
	if (e != DDI_SUCCESS) {
		return (DDI_FAILURE);
	}
	state = ddi_get_soft_state(ioat_statep, instance);
	if (state == NULL) {
		goto attachfail_get_soft_state;
	}

	state->is_dip = dip;
	state->is_instance = instance;

	/* setup the registers, save away some device info */
	e = ioat_chip_init(state);
	if (e != DDI_SUCCESS) {
		goto attachfail_chip_init;
	}

	/* initialize driver state, must be after chip init */
	e = ioat_drv_init(state);
	if (e != DDI_SUCCESS) {
		goto attachfail_drv_init;
	}

	/* create the minor node (for the ioctl) */
	e = ddi_create_minor_node(dip, "ioat", S_IFCHR, instance, DDI_PSEUDO,
	    0);
	if (e != DDI_SUCCESS) {
		goto attachfail_minor_node;
	}

	/* Enable device interrupts */
	ioat_intr_enable(state);

	/* Report that driver was loaded */
	ddi_report_dev(dip);

	/* register with dcopy */
	e = dcopy_device_register(state, &state->is_deviceinfo,
	    &state->is_device_handle);
	if (e != DCOPY_SUCCESS) {
		goto attachfail_register;
	}

	return (DDI_SUCCESS);

attachfail_register:
	ioat_intr_disable(state);
	ddi_remove_minor_node(dip, NULL);
attachfail_minor_node:
	ioat_drv_fini(state);
attachfail_drv_init:
	ioat_chip_fini(state);
attachfail_chip_init:
attachfail_get_soft_state:
	(void) ddi_soft_state_free(ioat_statep, instance);

	return (DDI_FAILURE);
}

/*
 * ioat_detach()
 */
static int
ioat_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	ioat_state_t *state;
	int instance;
	int e;


	instance = ddi_get_instance(dip);
	state = ddi_get_soft_state(ioat_statep, instance);
	if (state == NULL) {
		return (DDI_FAILURE);
	}

	switch (cmd) {
	case DDI_DETACH:
		break;

	case DDI_SUSPEND:
		ioat_channel_suspend(state);
		return (DDI_SUCCESS);

	default:
		return (DDI_FAILURE);
	}

	/*
	 * try to unregister from dcopy.  Since this driver doesn't follow the
	 * traditional parent/child model, we may still be in use so we can't
	 * detach yet.
	 */
	e = dcopy_device_unregister(&state->is_device_handle);
	if (e != DCOPY_SUCCESS) {
		if (e == DCOPY_PENDING) {
			cmn_err(CE_NOTE, "device busy, performing asynchronous"
			    " detach\n");
		}
		return (DDI_FAILURE);
	}

	ioat_detach_finish(state);

	return (DDI_SUCCESS);
}

/*
 * ioat_getinfo()
 */
/*ARGSUSED*/
static int
ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
	ioat_state_t *state;
	int instance;
	dev_t dev;
	int e;


	dev = (dev_t)arg;
	instance = getminor(dev);

	switch (cmd) {
	case DDI_INFO_DEVT2DEVINFO:
		state = ddi_get_soft_state(ioat_statep, instance);
		if (state == NULL) {
			return (DDI_FAILURE);
		}
		*result = (void *)state->is_dip;
		e = DDI_SUCCESS;
		break;

	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)(uintptr_t)instance;
		e = DDI_SUCCESS;
		break;

	default:
		e = DDI_FAILURE;
		break;
	}

	return (e);
}


/*
 * ioat_open()
 */
/*ARGSUSED*/
static int
ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred)
{
	ioat_state_t *state;
	int instance;

	instance = getminor(*devp);
	state = ddi_get_soft_state(ioat_statep, instance);
	if (state == NULL) {
		return (ENXIO);
	}

	return (0);
}


/*
 * ioat_close()
 */
/*ARGSUSED*/
static int
ioat_close(dev_t devp, int flag, int otyp, cred_t *cred)
{
	return (0);
}


/*
 * ioat_chip_init()
 */
static int
ioat_chip_init(ioat_state_t *state)
{
	ddi_device_acc_attr_t attr;
	int e;


	attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
	attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
	attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;

	e =  ddi_regs_map_setup(state->is_dip, 1, (caddr_t *)&state->is_genregs,
	    0, 0, &attr, &state->is_reg_handle);
	if (e != DDI_SUCCESS) {
		goto chipinitfail_regsmap;
	}

	/* save away ioat chip info */
	state->is_num_channels = (uint_t)ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_CHANCNT]);

	/*
	 * If we get a bogus value, something is wrong with the H/W, fail to
	 * attach.
	 */
	if (state->is_num_channels == 0) {
		goto chipinitfail_numchan;
	}

	state->is_maxxfer = (uint_t)ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_XFERCAP]);
	state->is_chanoff = (uintptr_t)ddi_get16(state->is_reg_handle,
	    (uint16_t *)&state->is_genregs[IOAT_PERPORT_OFF]);
	state->is_cbver = (uint_t)ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_CBVER]);
	state->is_intrdelay = (uint_t)ddi_get16(state->is_reg_handle,
	    (uint16_t *)&state->is_genregs[IOAT_INTRDELAY]);
	state->is_status = (uint_t)ddi_get16(state->is_reg_handle,
	    (uint16_t *)&state->is_genregs[IOAT_CSSTATUS]);
	state->is_capabilities = (uint_t)ddi_get32(state->is_reg_handle,
	    (uint32_t *)&state->is_genregs[IOAT_DMACAPABILITY]);

	if (state->is_cbver & 0x10) {
		state->is_ver = IOAT_CBv1;
	} else if (state->is_cbver & 0x20) {
		state->is_ver = IOAT_CBv2;
	} else {
		goto chipinitfail_version;
	}

	return (DDI_SUCCESS);

chipinitfail_version:
chipinitfail_numchan:
	ddi_regs_map_free(&state->is_reg_handle);
chipinitfail_regsmap:
	return (DDI_FAILURE);
}


/*
 * ioat_chip_fini()
 */
static void
ioat_chip_fini(ioat_state_t *state)
{
	ddi_regs_map_free(&state->is_reg_handle);
}


/*
 * ioat_drv_init()
 */
static int
ioat_drv_init(ioat_state_t *state)
{
	ddi_acc_handle_t handle;
	int e;


	mutex_init(&state->is_mutex, NULL, MUTEX_DRIVER, NULL);

	state->is_deviceinfo.di_dip = state->is_dip;
	state->is_deviceinfo.di_num_dma = state->is_num_channels;
	state->is_deviceinfo.di_maxxfer = state->is_maxxfer;
	state->is_deviceinfo.di_capabilities = state->is_capabilities;
	state->is_deviceinfo.di_cb = &ioat_cb;

	e = pci_config_setup(state->is_dip, &handle);
	if (e != DDI_SUCCESS) {
		goto drvinitfail_config_setup;
	}

	/* read in Vendor ID */
	state->is_deviceinfo.di_id = (uint64_t)pci_config_get16(handle, 0);
	state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 16;

	/* read in Device ID */
	state->is_deviceinfo.di_id |= (uint64_t)pci_config_get16(handle, 2);
	state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 32;

	/* Add in chipset version */
	state->is_deviceinfo.di_id |= (uint64_t)state->is_cbver;
	pci_config_teardown(&handle);

	e = ddi_intr_hilevel(state->is_dip, 0);
	if (e != 0) {
		cmn_err(CE_WARN, "hilevel interrupt not supported\n");
		goto drvinitfail_hilevel;
	}

	/* we don't support MSIs for v2 yet */
	e = ddi_add_intr(state->is_dip, 0, NULL, NULL, ioat_isr,
	    (caddr_t)state);
	if (e != DDI_SUCCESS) {
		goto drvinitfail_add_intr;
	}

	e = ddi_get_iblock_cookie(state->is_dip, 0, &state->is_iblock_cookie);
	if (e != DDI_SUCCESS) {
		goto drvinitfail_iblock_cookie;
	}

	e = ioat_channel_init(state);
	if (e != DDI_SUCCESS) {
		goto drvinitfail_channel_init;
	}

	return (DDI_SUCCESS);

drvinitfail_channel_init:
drvinitfail_iblock_cookie:
	ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
drvinitfail_add_intr:
drvinitfail_hilevel:
drvinitfail_config_setup:
	mutex_destroy(&state->is_mutex);

	return (DDI_FAILURE);
}


/*
 * ioat_drv_fini()
 */
static void
ioat_drv_fini(ioat_state_t *state)
{
	ioat_channel_fini(state);
	ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
	mutex_destroy(&state->is_mutex);
}


/*
 * ioat_unregister_complete()
 */
void
ioat_unregister_complete(void *device_private, int status)
{
	ioat_state_t *state;


	state = device_private;

	if (status != DCOPY_SUCCESS) {
		cmn_err(CE_WARN, "asynchronous detach aborted\n");
		return;
	}

	cmn_err(CE_CONT, "detach completing\n");
	ioat_detach_finish(state);
}


/*
 * ioat_detach_finish()
 */
void
ioat_detach_finish(ioat_state_t *state)
{
	ioat_intr_disable(state);
	ddi_remove_minor_node(state->is_dip, NULL);
	ioat_drv_fini(state);
	ioat_chip_fini(state);
	(void) ddi_soft_state_free(ioat_statep, state->is_instance);
}


/*
 * ioat_intr_enable()
 */
static void
ioat_intr_enable(ioat_state_t *state)
{
	uint32_t intr_status;


	/* Clear any pending interrupts */
	intr_status = ddi_get32(state->is_reg_handle,
	    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);
	if (intr_status != 0) {
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS],
		    intr_status);
	}

	/* Enable interrupts on the device */
	ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
	    IOAT_INTRCTL_MASTER_EN);
}


/*
 * ioat_intr_disable()
 */
static void
ioat_intr_disable(ioat_state_t *state)
{
	/*
	 * disable interrupts on the device. A read of the interrupt control
	 * register clears the enable bit.
	 */
	(void) ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_INTRCTL]);
}


/*
 * ioat_isr()
 */
static uint_t
ioat_isr(caddr_t parm)
{
	uint32_t intr_status;
	ioat_state_t *state;
	uint8_t intrctrl;
	uint32_t chan;
	uint_t r;
	int i;

	state = (ioat_state_t *)parm;

	intrctrl = ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_INTRCTL]);
	/* master interrupt enable should always be set */
	ASSERT(intrctrl & IOAT_INTRCTL_MASTER_EN);

	/* If the interrupt status bit isn't set, it's not ours */
	if (!(intrctrl & IOAT_INTRCTL_INTR_STAT)) {
		/* re-set master interrupt enable (since it clears on read) */
		ddi_put8(state->is_reg_handle,
		    &state->is_genregs[IOAT_INTRCTL], intrctrl);
		return (DDI_INTR_UNCLAIMED);
	}

	/* see which channels generated the interrupt */
	intr_status = ddi_get32(state->is_reg_handle,
	    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);

	/* call the intr handler for the channels */
	r = DDI_INTR_UNCLAIMED;
	chan = 1;
	for (i = 0; i < state->is_num_channels; i++) {
		if (intr_status & chan) {
			ioat_channel_intr(&state->is_channel[i]);
			r = DDI_INTR_CLAIMED;
		}
		chan = chan << 1;
	}

	/*
	 * if interrupt status bit was set, there should have been an
	 * attention status bit set too.
	 */
	ASSERT(r == DDI_INTR_CLAIMED);

	/* re-set master interrupt enable (since it clears on read) */
	ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
	    intrctrl);

	return (r);
}

static int
ioat_quiesce(dev_info_t *dip)
{
	ioat_state_t *state;
	int instance;

	instance = ddi_get_instance(dip);
	state = ddi_get_soft_state(ioat_statep, instance);
	if (state == NULL) {
		return (DDI_FAILURE);
	}

	ioat_intr_disable(state);
	ioat_channel_quiesce(state);

	return (DDI_SUCCESS);
}