Home | History | Annotate | Download | only in sys
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 
     22 /*
     23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
     24  * Use is subject to license terms.
     25  */
     26 
     27 #ifndef _SYS_IOAT_H
     28 #define	_SYS_IOAT_H
     29 
     30 #ifdef __cplusplus
     31 extern "C" {
     32 #endif
     33 
     34 #include <sys/types.h>
     35 #include <sys/dcopy.h>
     36 #include <sys/dcopy_device.h>
     37 
     38 
     39 /* ioat ioctls */
     40 #define	IOATIOC			('T'<< 8)
     41 typedef enum {
     42 	IOAT_IOCTL_WRITE_REG	= (IOATIOC | 0x0),
     43 	IOAT_IOCTL_READ_REG	= (IOATIOC | 0x1),
     44 	IOAT_IOCTL_TEST		= (IOATIOC | 0x2)
     45 } ioat_ioctl_enum_t;
     46 
     47 typedef struct ioat_ioctl_reg_s {
     48 	uint_t		size;
     49 	uint_t		addr;
     50 	uint64_t	data;
     51 } ioat_ioctl_reg_t;
     52 typedef ioat_ioctl_reg_t ioat_ioctl_wrreg_t;
     53 typedef ioat_ioctl_reg_t ioat_ioctl_rdreg_t;
     54 
     55 #ifdef _KERNEL
     56 /* *** Driver Private Below *** */
     57 
     58 /* IOAT_DMACAPABILITY flags */
     59 #define	IOAT_DMACAP_PAGEBREAK	0x1
     60 #define	IOAT_DMACAP_CRC		0x2
     61 #define	IOAT_DMACAP_MARKERSKIP	0x4
     62 #define	IOAT_DMACAP_XOR		0x8
     63 #define	IOAT_DMACAP_DCA		0x10
     64 
     65 /* IOAT_INTRCTL bits */
     66 #define	IOAT_INTRCTL_MASTER_EN	0x1
     67 #define	IOAT_INTRCTL_INTR_STAT	0x2
     68 
     69 /* MMIO Registers */
     70 #define	IOAT_CHANCNT		0x0	/* 8-bit */
     71 #define	IOAT_XFERCAP		0x1	/* 8-bit */
     72 #define	IOAT_GENCTRL		0x2	/* 8-bit */
     73 #define	IOAT_INTRCTL		0x3	/* 8-bit */
     74 #define	IOAT_ATTNSTATUS		0x4	/* 32-bit */
     75 #define	IOAT_CBVER		0x8	/* 8-bit */
     76 #define	IOAT_PERPORT_OFF	0xA	/* 16-bit */
     77 #define	IOAT_INTRDELAY		0xC	/* 16-bit */
     78 #define	IOAT_CSSTATUS		0xE	/* 16-bit */
     79 #define	IOAT_DMACAPABILITY	0x10	/* 32-bit */
     80 
     81 #define	IOAT_CHANNELREG_OFFSET	0x80
     82 
     83 /* Channel Registers */
     84 #define	IOAT_CHAN_CTL		0x0	/* 16-bit */
     85 #define	IOAT_CHAN_COMP		0x2	/* 16-bit */
     86 #define	IOAT_CHAN_CMPL_LO	0x18	/* 32-bit */
     87 #define	IOAT_CHAN_CMPL_HI	0x1C	/* 32-bit */
     88 #define	IOAT_CHAN_ERR		0x28	/* 32-bit */
     89 #define	IOAT_CHAN_ERRMASK	0x2C	/* 32-bit */
     90 #define	IOAT_CHAN_DCACTRL	0x30	/* 32-bit */
     91 
     92 #define	IOAT_V1_CHAN_STS_LO	0x4	/* 32-bit */
     93 #define	IOAT_V1_CHAN_STS_HI	0x8	/* 32-bit */
     94 #define	IOAT_V1_CHAN_ADDR_LO	0x0C	/* 32-bit */
     95 #define	IOAT_V1_CHAN_ADDR_HI	0x10	/* 32-bit */
     96 #define	IOAT_V1_CHAN_CMD	0x14	/* 8-bit */
     97 
     98 #define	IOAT_V2_CHAN_CMD	0x4	/* 8-bit */
     99 #define	IOAT_V2_CHAN_CNT	0x6	/* 16-bit */
    100 #define	IOAT_V2_CHAN_STS_LO	0x8	/* 32-bit */
    101 #define	IOAT_V2_CHAN_STS_HI	0xC	/* 32-bit */
    102 #define	IOAT_V2_CHAN_ADDR_LO	0x10	/* 32-bit */
    103 #define	IOAT_V2_CHAN_ADDR_HI	0x14	/* 32-bit */
    104 
    105 #define	IOAT_CHAN_STS_ADDR_MASK		0xFFFFFFFFFFFFFFC0
    106 #define	IOAT_CHAN_STS_XFER_MASK		0x3F
    107 #define	IOAT_CHAN_STS_FAIL_MASK		0x6
    108 #define	IOAT_CMPL_INDEX(channel)	\
    109 	(((*channel->ic_cmpl & IOAT_CHAN_STS_ADDR_MASK) - \
    110 	ring->cr_phys_desc) >> 6)
    111 #define	IOAT_CMPL_FAILED(channel)	\
    112 	(*channel->ic_cmpl & IOAT_CHAN_STS_FAIL_MASK)
    113 
    114 
    115 typedef struct ioat_chan_desc_s {
    116 	uint32_t	dd_res0;
    117 	uint32_t	dd_ctrl;
    118 	uint64_t	dd_res1;
    119 	uint64_t	dd_res2;
    120 	uint64_t	dd_next_desc;
    121 	uint64_t	dd_res4;
    122 	uint64_t	dd_res5;
    123 	uint64_t	dd_res6;
    124 	uint64_t	dd_res7;
    125 } ioat_chan_desc_t;
    126 
    127 /* dca dd_ctrl bits */
    128 #define	IOAT_DESC_CTRL_OP_CNTX	((uint32_t)0xFF << 24)
    129 #define	IOAT_DESC_CTRL_CNTX_CHNG	0x1
    130 typedef struct ioat_chan_dca_desc_s {
    131 	uint32_t	dd_cntx;
    132 	uint32_t	dd_ctrl;
    133 	uint64_t	dd_res1;
    134 	uint64_t	dd_res2;
    135 	uint64_t	dd_next_desc;
    136 	uint64_t	dd_res4;
    137 	uint64_t	dd_res5;
    138 	uint64_t	dd_res6;
    139 	uint64_t	dd_res7;
    140 } ioat_chan_dca_desc_t;
    141 
    142 /* dma dd_ctrl bits */
    143 #define	IOAT_DESC_CTRL_OP_DMA	(0x0 << 24)
    144 #define	IOAT_DESC_DMACTRL_NULL	0x20
    145 #define	IOAT_DESC_CTRL_FENCE	0x10
    146 #define	IOAT_DESC_CTRL_CMPL	0x8
    147 #define	IOAT_DESC_CTRL_NODSTSNP	0x4
    148 #define	IOAT_DESC_CTRL_NOSRCSNP	0x2
    149 #define	IOAT_DESC_CTRL_INTR	0x1
    150 typedef struct ioat_chan_dma_desc_s {
    151 	uint32_t	dd_size;
    152 	uint32_t	dd_ctrl;
    153 	uint64_t	dd_src_paddr;
    154 	uint64_t	dd_dest_paddr;
    155 	uint64_t	dd_next_desc;
    156 	uint64_t	dd_next_src_paddr;	/* v2 only */
    157 	uint64_t	dd_next_dest_paddr;	/* v2 only */
    158 	uint64_t	dd_res6;
    159 	uint64_t	dd_res7;
    160 } ioat_chan_dma_desc_t;
    161 
    162 
    163 typedef enum {
    164 	IOAT_CBv1,
    165 	IOAT_CBv2
    166 } ioat_version_t;
    167 
    168 /* ioat private data per command */
    169 typedef struct ioat_cmd_private_s {
    170 	uint64_t	ip_generation;
    171 	uint64_t	ip_index;
    172 	uint64_t	ip_start;
    173 	dcopy_cmd_t	ip_next;
    174 } ioat_cmd_private_t;
    175 
    176 /* descriptor ring state */
    177 typedef struct ioat_channel_ring_s {
    178 	/* protects cr_cmpl_gen & cr_cmpl_last */
    179 	kmutex_t		cr_cmpl_mutex;
    180 
    181 	/* desc ring generation for the last completion we saw */
    182 	uint64_t		cr_cmpl_gen;
    183 
    184 	/* last descriptor index we saw complete */
    185 	uint64_t		cr_cmpl_last;
    186 
    187 	/* protects cr_desc_* */
    188 	kmutex_t		cr_desc_mutex;
    189 
    190 	/*
    191 	 * last descriptor posted. used to update its next pointer when we
    192 	 * add a new desc. Also used to tack the completion (See comment for
    193 	 * cr_desc_gen_prev).
    194 	 */
    195 	uint64_t		cr_desc_prev;
    196 
    197 	/* where to put the next descriptor */
    198 	uint64_t		cr_desc_next;
    199 
    200 	/* what the current desc ring generation is */
    201 	uint64_t		cr_desc_gen;
    202 
    203 	/*
    204 	 * used during cmd_post to track the last desc posted. cr_desc_next
    205 	 * and cr_desc_gen will be pointing to the next free desc after
    206 	 * writing the descriptor to the ring. But we want to track the
    207 	 * completion for the last descriptor posted.
    208 	 */
    209 	uint64_t		cr_desc_gen_prev;
    210 
    211 	/* the last desc in the ring (for wrap) */
    212 	uint64_t		cr_desc_last;
    213 
    214 	/* pointer to the head of the ring */
    215 	ioat_chan_desc_t	*cr_desc;
    216 
    217 	/* physical address of the head of the ring */
    218 	uint64_t		cr_phys_desc;
    219 
    220 	/* back pointer to the channel state */
    221 	struct ioat_channel_s	*cr_chan;
    222 
    223 	/* for CB v2, number of desc posted (written to IOAT_V2_CHAN_CNT) */
    224 	uint_t			cr_post_cnt;
    225 } ioat_channel_ring_t;
    226 
    227 /* track channel state so we can handle a failure */
    228 typedef enum {
    229 	IOAT_CHANNEL_OK = 0,
    230 	IOAT_CHANNEL_IN_FAILURE = 1
    231 } ic_channel_state_t;
    232 
    233 typedef struct ioat_channel_s *ioat_channel_t;
    234 struct ioat_channel_s {
    235 	/* channel's ring state */
    236 	ioat_channel_ring_t	*ic_ring;
    237 
    238 	/* IOAT_CBv1 || IOAT_CBv2 */
    239 	ioat_version_t		ic_ver;
    240 
    241 	/*
    242 	 * state to determine if it's OK to post the the channel and if all
    243 	 * future polls should return failure.
    244 	 */
    245 	ic_channel_state_t	ic_channel_state;
    246 
    247 	/* channel command cache (*_cmd_alloc, *_cmd_free, etc) */
    248 	kmem_cache_t		*ic_cmd_cache;
    249 
    250 	/* dcopy state for dcopy_device_channel_notify() call */
    251 	dcopy_handle_t		ic_dcopy_handle;
    252 
    253 	/* location in memory where completions are DMA'ed into */
    254 	volatile uint64_t	*ic_cmpl;
    255 
    256 	/* channel specific registers */
    257 	uint8_t			*ic_regs;
    258 
    259 	/* if this channel is using DCA */
    260 	boolean_t		ic_dca_active;
    261 
    262 	/* DCA ID the channel is currently pointing to */
    263 	uint32_t		ic_dca_current;
    264 
    265 	/* devices channel number */
    266 	uint_t			ic_chan_num;
    267 
    268 	/* number of descriptors in ring */
    269 	uint_t			ic_chan_desc_cnt;
    270 
    271 	/* descriptor ring alloc state */
    272 	ddi_dma_handle_t	ic_desc_dma_handle;
    273 	size_t			ic_desc_alloc_size;
    274 	ddi_acc_handle_t	ic_desc_handle;
    275 	ddi_dma_cookie_t	ic_desc_cookies;
    276 
    277 	/* completion buffer alloc state */
    278 	ddi_dma_handle_t	ic_cmpl_dma_handle;
    279 	size_t			ic_cmpl_alloc_size;
    280 	ddi_acc_handle_t	ic_cmpl_handle;
    281 	ddi_dma_cookie_t	ic_cmpl_cookie;
    282 	uint64_t		ic_phys_cmpl;
    283 
    284 	/* if inuse, we need to re-init the channel during resume */
    285 	boolean_t		ic_inuse;
    286 
    287 	/* backpointer to driver state */
    288 	struct ioat_state_s	*ic_state;
    289 };
    290 
    291 typedef struct ioat_rs_s *ioat_rs_hdl_t;
    292 
    293 /* driver state */
    294 typedef struct ioat_state_s {
    295 	dev_info_t		*is_dip;
    296 	int			is_instance;
    297 
    298 	kmutex_t		is_mutex;
    299 
    300 	/* register handle and pointer to registers */
    301 	ddi_acc_handle_t	is_reg_handle;
    302 	uint8_t			*is_genregs;
    303 
    304 	/* IOAT_CBv1 || IOAT_CBv2 */
    305 	ioat_version_t		is_ver;
    306 
    307 	/* channel state */
    308 	ioat_channel_t		is_channel;
    309 	size_t			is_chansize;
    310 	ioat_rs_hdl_t		is_channel_rs;
    311 
    312 	ddi_iblock_cookie_t	is_iblock_cookie;
    313 
    314 	/* device info */
    315 	uint_t			is_chanoff;
    316 	uint_t			is_num_channels;
    317 	uint_t			is_maxxfer;
    318 	uint_t			is_cbver;
    319 	uint_t			is_intrdelay;
    320 	uint_t			is_status;
    321 	uint_t			is_capabilities;
    322 
    323 	/* dcopy_device_register()/dcopy_device_unregister() state */
    324 	dcopy_device_handle_t	is_device_handle;
    325 	dcopy_device_info_t	is_deviceinfo;
    326 } ioat_state_t;
    327 
    328 
    329 int ioat_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred,
    330     int *rval);
    331 
    332 void ioat_rs_init(ioat_state_t *state, uint_t min_val, uint_t max_val,
    333     ioat_rs_hdl_t *handle);
    334 void ioat_rs_fini(ioat_rs_hdl_t *handle);
    335 int ioat_rs_alloc(ioat_rs_hdl_t handle, uint_t *rs);
    336 void ioat_rs_free(ioat_rs_hdl_t handle, uint_t rs);
    337 
    338 int ioat_channel_init(ioat_state_t *state);
    339 void ioat_channel_fini(ioat_state_t *state);
    340 void ioat_channel_suspend(ioat_state_t *state);
    341 int ioat_channel_resume(ioat_state_t *state);
    342 void ioat_channel_quiesce(ioat_state_t *);
    343 
    344 int ioat_channel_alloc(void *device_private, dcopy_handle_t handle, int flags,
    345     uint_t size, dcopy_query_channel_t *info, void *channel_private);
    346 void ioat_channel_free(void *channel_private);
    347 void ioat_channel_intr(ioat_channel_t channel);
    348 int ioat_cmd_alloc(void *channel, int flags, dcopy_cmd_t *cmd);
    349 void ioat_cmd_free(void *channel, dcopy_cmd_t *cmd);
    350 int ioat_cmd_post(void *channel, dcopy_cmd_t cmd);
    351 int ioat_cmd_poll(void *channel, dcopy_cmd_t cmd);
    352 void ioat_unregister_complete(void *device_private, int status);
    353 
    354 
    355 #endif /* _KERNEL */
    356 
    357 #ifdef __cplusplus
    358 }
    359 #endif
    360 
    361 #endif /* _SYS_IOAT_H */
    362