Home | History | Annotate | Download | only in ibdm
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 /*
     22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
     23  * Use is subject to license terms.
     24  */
     25 
     26 #ifndef _SYS_IB_MGT_IBDM_IBDM_IMPL_H
     27 #define	_SYS_IB_MGT_IBDM_IBDM_IMPL_H
     28 
     29 /*
     30  * ibdm_impl.h
     31  *
     32  *	This file contains definitions of the data structures, macros etc
     33  *	related to the IBDM module.
     34  */
     35 
     36 #include <sys/ib/mgt/ibdm/ibdm_ibnex.h>
     37 #include <sys/ib/ibtl/impl/ibtl_util.h>
     38 
     39 #ifdef __cplusplus
     40 extern "C" {
     41 #endif
     42 
     43 /* values for "cb_req_type" */
     44 #define	IBDM_REQ_TYPE_INVALID		0x0
     45 #define	IBDM_REQ_TYPE_CLASSPORTINFO	0x1
     46 #define	IBDM_REQ_TYPE_IOUINFO		0x2
     47 #define	IBDM_REQ_TYPE_IOCINFO		0x4
     48 #define	IBDM_REQ_TYPE_SRVENTS		0x8
     49 #define	IBDM_REQ_TYPE_IOU_DIAGCODE	0x10
     50 #define	IBDM_REQ_TYPE_IOC_DIAGCODE	0x20
     51 
     52 typedef struct ibdm_taskq_args_s {
     53 	ibmf_handle_t		tq_ibmf_handle;
     54 	ibmf_msg_t		*tq_ibmf_msg;
     55 	void			*tq_args;
     56 } ibdm_taskq_args_t;
     57 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ibdm_taskq_args_t))
     58 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ib_mad_hdr_t))
     59 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ibmf_msg_t))
     60 
     61 #define	IBDM_GID_PRESENT		0x1
     62 #define	IBDM_GID_NOT_PRESENT		0x0
     63 
     64 #define	IBDM_IBMF_PKT_DUP_RESP		0x1
     65 #define	IBDM_IBMF_PKT_REUSED		0x2
     66 #define	IBDM_IBMF_PKT_UNEXP_RESP	0x4
     67 
     68 #define	IBDM_MAX_SERV_ENTRIES_PER_REQ	4
     69 
     70 typedef struct ibdm_gid_s {
     71 	uint64_t		gid_dgid_hi;
     72 	uint64_t		gid_dgid_lo;
     73 	struct ibdm_gid_s	*gid_next;
     74 } ibdm_gid_t;
     75 
     76 #define	IBDM_GID_PROBE_NOT_DONE		0x00
     77 #define	IBDM_GET_CLASSPORTINFO		0x01
     78 #define	IBDM_GET_IOUNITINFO		0x02
     79 #define	IBDM_GET_IOC_DETAILS		0x04
     80 #define	IBDM_GID_PROBING_COMPLETE	0x08
     81 #define	IBDM_GID_PROBING_SKIPPED	0x10
     82 #define	IBDM_GID_PROBING_FAILED		0x20
     83 #define	IBDM_SET_CLASSPORTINFO		0x40
     84 
     85 /*
     86  * Identifiers to distinguish a Cisco FC GW from others.
     87  * Used to filter a setclassportinfo request.
     88  */
     89 #define	IBDM_CISCO_COMPANY_ID		(0x5ad)
     90 #define	IBDM_CISCO_DEVICE_ID		(0xa87c)
     91 
     92 /*
     93  * the bit-shift value for OUI in GUID
     94  * A 64 bit globally unique identifier (GUID) composed of a 24 bit company id
     95  * and an 48 bit extension identifier, and this value is used to extract
     96  * the company id from the GUID.
     97  */
     98 #define	IBDM_OUI_GUID_SHIFT		(40)
     99 
    100 /*
    101  * The state diagram for the gl_state
    102  *
    103  *                          (in case of Cisco FC GW)
    104  * IBDM_GID_PROBE_NOT_DONE  ---------- 40 -> IBDM_SET_CLASSPORTINFO
    105  *                          ----.	        |
    106  *    |      |			| (others)      |
    107  *    |      |			1		|
    108  *    |      |			|               1
    109  *    |      |			`-------------.	|
    110  *    |      |                                v v
    111  *    |	     |				     IBDM_GET_CLASSPORTINFO
    112  *    |      |
    113  *    |      |                                  |
    114  *    |      2                                  3
    115  *    |      |                                  |
    116  *    |      v                                  v
    117  *    |     IBDM_GID_PROBING_FAILED          IBDM_GET_IOUNITINFO
    118  *    |                                         |
    119  *    6                                         4
    120  *    |                                         |
    121  *    v                                         v
    122  *  IBDM_GID_PROBING_SKIPPLED                IBDM_GET_IOC_DETAILS
    123  *                                              |
    124  *                                              5
    125  *                                              |
    126  *                                              v
    127  *                                           IBDM_GID_PROBE_COMPLETE
    128  *
    129  * Initial state : IBDM_GID_PROBE_NOT_DONE
    130  *     40 = Port sends setClassPortInfo to activate Cisco FC GW
    131  *	1 = Port supports DM MAD's and a request to ClassportInfo is sent
    132  *	3 = Received ClassPortInfo and sent IOUnitInfo
    133  *	4 = Recevied IOUunitInfo and sent IOC profile, diagcodes, and
    134  *		service entries requests
    135  *	5 = Received all the IOC information
    136  *	2 = Failed to probe the GID
    137  *		Port does not support DM MAD's
    138  *		Port did not respond property
    139  *	6 = A different GID for the same port, skip the probe
    140  *
    141  * Reprobe state transition :
    142  *
    143  * IBDM_GID_PROBE_COMPLETE
    144  *	|
    145  *	7
    146  *	|
    147  *	v
    148  * IBDM_GET_IOC_DETAILS
    149  *	|
    150  *	8
    151  *	|
    152  *	v
    153  * IBDM_GID_PROBE_COMPLETE
    154  *
    155  *	7 = Reprobe request for one or more IOCs initiated.
    156  *	8 = Reprobe done(IOC COntroller Profile & Service entries)
    157  */
    158 
    159 typedef struct ibdm_dp_gidinfo_s {
    160 	kmutex_t		gl_mutex;
    161 	uint_t			gl_state;
    162 	int			gl_reprobe_flag;	/* pass this to taskq */
    163 	struct ibdm_dp_gidinfo_s *gl_next;
    164 	struct ibdm_dp_gidinfo_s *gl_prev;
    165 	ibdm_iou_info_t		*gl_iou;
    166 	int			gl_pending_cmds;
    167 	ibmf_qp_handle_t	gl_qp_hdl;
    168 	uint64_t		gl_transactionID;
    169 	ibdm_timeout_cb_args_t	gl_iou_cb_args;
    170 	ib_lid_t		gl_dlid;
    171 	ib_lid_t		gl_slid;
    172 	uint64_t		gl_dgid_hi;
    173 	uint64_t		gl_dgid_lo;
    174 	uint64_t		gl_sgid_hi;
    175 	uint64_t		gl_sgid_lo;
    176 	ib_guid_t		gl_nodeguid;
    177 	ib_guid_t		gl_portguid;
    178 	ib_pkey_t		gl_p_key;
    179 	boolean_t		gl_is_dm_capable;
    180 	boolean_t		gl_redirected;
    181 	uint32_t		gl_redirect_dlid;
    182 	uint32_t		gl_redirect_QP;
    183 	ib_pkey_t		gl_redirect_pkey;
    184 	ib_qkey_t		gl_redirect_qkey;
    185 	uint64_t		gl_redirectGID_hi;
    186 	uint64_t		gl_redirectGID_lo;
    187 	ibmf_handle_t		gl_ibmf_hdl;
    188 	ibmf_saa_handle_t	gl_sa_hdl;
    189 	timeout_id_t		gl_timeout_id;
    190 	ibdm_timeout_cb_args_t	gl_cpi_cb_args;
    191 	uint32_t		gl_ngids;
    192 	ibdm_gid_t		*gl_gid;
    193 	uint32_t		gl_resp_timeout;
    194 	int			gl_num_iocs;
    195 	ibdm_hca_list_t		*gl_hca_list;
    196 	int			gl_disconnected;
    197 	uint64_t		gl_min_transactionID;
    198 	uint64_t		gl_max_transactionID;
    199 	ibdm_iou_info_t		*gl_prev_iou;
    200 	uint16_t		gl_devid;	/* device ID info */
    201 	kcondvar_t		gl_probe_cv;	/* sync for Cisco FC GW */
    202 	uint32_t		gl_flag;
    203 	uint8_t			gl_SL:4; 	/* SL from path_record */
    204 	uint8_t			gl_redirectSL:4; /* SL from redirection */
    205 } ibdm_dp_gidinfo_t;
    206 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex,
    207 	ibdm_dp_gidinfo_s::{gl_state gl_timeout_id gl_pending_cmds}))
    208 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", ibdm_dp_gidinfo_s))
    209 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibdm_dp_gidinfo_s::{gl_ibmf_hdl gl_sa_hdl}))
    210 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex,
    211 	ibdm_ioc_info_s::{ioc_timeout_id ioc_dc_timeout_id}))
    212 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex,
    213 	ibdm_srvents_info_s::se_timeout_id))
    214 
    215 /*
    216  * The transaction ID for the GID contains of two parts :
    217  *	1. Upper 32 bits which is unique for each GID.
    218  *	2. Lower 32 bits which is unique for each MAD.
    219  * The assumptions are :
    220  *	1. At most 2 power 32 DM capable GIDs on the IB fabric
    221  *	2. IBDM sends maximum of 2 power 32 MADs to the same DM GID
    222  * The limits are sufficient for practical configurations.
    223  */
    224 #define	IBDM_GID_TRANSACTIONID_SHIFT	((ulong_t)32)
    225 #define	IBDM_GID_TRANSACTIONID_MASK	0xFFFFFFFF00000000ULL
    226 
    227 typedef struct ibdm_s {
    228 	/* Protects IBDM's critical data */
    229 	kmutex_t		ibdm_mutex;
    230 	uint32_t		ibdm_hca_count;
    231 	kmutex_t		ibdm_hl_mutex;
    232 	kmutex_t		ibdm_ibnex_mutex;
    233 	ibdm_hca_list_t		*ibdm_hca_list_head;
    234 	ibdm_hca_list_t		*ibdm_hca_list_tail;
    235 
    236 	ibdm_dp_gidinfo_t	*ibdm_dp_gidlist_head;
    237 	ibdm_dp_gidinfo_t	*ibdm_dp_gidlist_tail;
    238 
    239 	kcondvar_t		ibdm_probe_cv;
    240 	kcondvar_t		ibdm_busy_cv;
    241 	kcondvar_t		ibdm_port_settle_cv;
    242 	uint32_t		ibdm_ngid_probes_in_progress;
    243 	uint64_t		ibdm_transactionID;
    244 	uint32_t		ibdm_ngids;
    245 	uint32_t		ibdm_busy;
    246 	int			ibdm_state;
    247 	ibt_clnt_hdl_t		ibdm_ibt_clnt_hdl;
    248 
    249 	/*
    250 	 * These are callback routines registered by the IB nexus driver.
    251 	 * These callbacks are used to inform the IB nexus driver about
    252 	 * the arrival/removal of HCA and IOC's
    253 	 */
    254 	ibdm_callback_t		ibdm_ibnex_callback;
    255 
    256 	/* Flag indicating - prev_iou during sweep */
    257 	int			ibdm_prev_iou;
    258 } ibdm_t;
    259 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_mutex, ibdm_s::{ibdm_ibt_clnt_hdl
    260 	ibdm_busy ibdm_state}))
    261 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibdm_s::ibdm_ibt_clnt_hdl))
    262 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_hl_mutex,
    263 	ibdm_s::{ibdm_hca_list_head ibdm_hca_list_tail}))
    264 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_ibnex_mutex,
    265 	ibdm_s::ibdm_ibnex_callback))
    266 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", ibdm_s))
    267 _NOTE(LOCK_ORDER(ibdm_s::ibdm_mutex ibdm_dp_gidinfo_s::gl_mutex))
    268 
    269 /* valid values for ibdm_state */
    270 #define	IBDM_LOCKS_ALLOCED	0x01		/* global locks alloced */
    271 #define	IBDM_CVS_ALLOCED	0x02		/* global "cv"s alloced */
    272 #define	IBDM_IBT_ATTACHED	0x04		/* ibt_attach() called */
    273 #define	IBDM_HCA_ATTACHED	0x08		/* ibdm_handle_hca() called */
    274 
    275 #define	IBDM_8_BIT_MASK		0x00FF
    276 #define	IBDM_16_BIT_MASK	0xFFFF
    277 #define	IBDM_RETRY_COUNT	0x2
    278 
    279 #define	IBDM_BUSY		0x1
    280 #define	IBDM_PROBE_IN_PROGRESS	0x2
    281 #define	IBDM_CISCO_PROBE	0x4
    282 #define	IBDM_CISCO_PROBE_DONE	0x8
    283 
    284 /*
    285  * Device Management MAD packet format
    286  * +--------+------------+------------+------------+------------+
    287  * | offset |   byte 0   |   byte 1   |   byte 2   |   byte 3   |
    288  * +--------+------------+------------+------------+------------+ --
    289  * |   0    |                                                   |  ^
    290  * +--------+                                                   | sizeof(
    291  * |   ...  |              Common MAD Header                    | ib_mad_hdr_t)
    292  * +--------+                                                   |  | (A)
    293  * |   20   |                                                   |  v
    294  * +--------+------------+------------+------------+------------+ --
    295  * |   24   |                                                   |  ^
    296  * +--------+                                                   |  |
    297  * |   ...  |              RMPP Header                          |  |
    298  * +--------+                                                   |  |
    299  * |   32   |                                                   |  |
    300  * +--------+------------+------------+------------+------------+  |
    301  * |   36   |                                                   |  |
    302  * +--------+              Access_Key                           |
    303  * |   40   |                                                   | IBDM_DM_MAD_
    304  * +--------+------------+------------+------------+------------+ HDR_SZ
    305  * |   44   |  KeyType   |              reserved                |    (B)
    306  * +--------+------------+------------+------------+------------+  |
    307  * |   48   |                                                   |  |
    308  * +--------+                                                   |  |
    309  * |   52   |              Reserved                             |  |
    310  * +--------+                                                   |  |
    311  * |   56   |                                                   |  |
    312  * +--------+------------+------------+------------+------------+  |
    313  * |   60   |       Change_ID         |     ComponentMask       |  v
    314  * +--------+------------+------------+------------+------------+ --
    315  * |   64   |                                                   |  ^
    316  * +--------+                                                   | IBDM_MAD_SIZE
    317  * |   ...  |              Device Management Data               | - (A) - (B)
    318  * +--------+                                                   |  |
    319  * |  252   |                                                   |  v
    320  * +--------+------------+------------+------------+------------+ --
    321  */
    322 #define	IBDM_MAD_SIZE		256
    323 #define	IBDM_DM_MAD_HDR_SZ	40
    324 
    325 #define	IBDM_DFT_TIMEOUT	4
    326 #define	IBDM_DFT_NRETRIES	3
    327 
    328 #define	IBDM_ENABLE_TASKQ_HANDLING	1
    329 #define	IBDM_DISABLE_TASKQ_HANLDING	0
    330 
    331 typedef struct ibdm_saa_event_arg_s {
    332 	ibmf_saa_handle_t ibmf_saa_handle;
    333 	ibmf_saa_subnet_event_t ibmf_saa_event;
    334 	ibmf_saa_event_details_t event_details;
    335 	void *callback_arg;
    336 } ibdm_saa_event_arg_t;
    337 
    338 #define	IBDM_TIMEOUT_VALUE(t)	(drv_usectohz(t * 1000000))
    339 
    340 #define	IBDM_OUT_IBMFMSG_MADHDR(msg)\
    341 		(msg->im_msgbufs_send.im_bufs_mad_hdr)
    342 
    343 #define	IBDM_IN_IBMFMSG_MADHDR(msg)\
    344 		(msg->im_msgbufs_recv.im_bufs_mad_hdr)
    345 
    346 #define	IBDM_IN_IBMFMSG_STATUS(msg)\
    347 		b2h16(msg->im_msgbufs_recv.im_bufs_mad_hdr->Status)
    348 
    349 #define	IBDM_IN_IBMFMSG_ATTR(msg)\
    350 		b2h16(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID)
    351 
    352 #define	IBDM_IN_IBMFMSG_ATTRMOD(msg)\
    353 		b2h32(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier)
    354 
    355 #define	IBDM_IN_IBMFMSG2IOU(msg)	(ib_dm_io_unitinfo_t *)\
    356 		(msg->im_msgbufs_recv.im_bufs_cl_data)
    357 
    358 #define	IBDM_IN_IBMFMSG2IOC(msg)	(ib_dm_ioc_ctrl_profile_t *)\
    359 		(msg->im_msgbufs_recv.im_bufs_cl_data)
    360 
    361 #define	IBDM_IN_IBMFMSG2SRVENT(msg)	(ib_dm_srv_t *)\
    362 		(msg->im_msgbufs_recv.im_bufs_cl_data)
    363 
    364 #define	IBDM_IN_IBMFMSG2DIAGCODE(msg)	(uint32_t *)\
    365 		(msg->im_msgbufs_recv.im_bufs_cl_data)
    366 
    367 #define	IBDM_GIDINFO2IOCINFO(gid_info, idx) \
    368 		(ibdm_ioc_info_t *)&gid_info->gl_iou->iou_ioc_info[idx];
    369 
    370 #define	IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)\
    371 		((ioc_no < 1) || (ioc_no >\
    372 			gid_info->gl_iou->iou_info.iou_num_ctrl_slots))
    373 
    374 #define	IBDM_INVALID_PKEY(pkey)	\
    375 		(((pkey) == IB_PKEY_INVALID_FULL) || \
    376 		((pkey) == IB_PKEY_INVALID_LIMITED))
    377 
    378 #ifdef DEBUG
    379 
    380 void	ibdm_dump_mad_hdr(ib_mad_hdr_t *);
    381 void	ibdm_dump_ibmf_msg(ibmf_msg_t *, int);
    382 void	ibdm_dump_path_info(sa_path_record_t *);
    383 void	ibdm_dump_classportinfo(ib_mad_classportinfo_t *);
    384 void	ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *);
    385 void	ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *);
    386 void	ibdm_dump_service_entries(ib_dm_srv_t *);
    387 void	ibdm_dump_sweep_fabric_timestamp(int);
    388 
    389 #define	ibdm_dump_mad_hdr(a)		ibdm_dump_mad_hdr(a)
    390 #define	ibdm_dump_ibmf_msg(a, b)	ibdm_dump_ibmf_msg(a, b)
    391 #define	ibdm_dump_path_info(a)		ibdm_dump_path_info(a)
    392 #define	ibdm_dump_classportinfo(a)	ibdm_dump_classportinfo(a)
    393 #define	ibdm_dump_iounitinfo(a)		ibdm_dump_iounitinfo(a)
    394 #define	ibdm_dump_ioc_profile(a)	ibdm_dump_ioc_profile(a)
    395 #define	ibdm_dump_service_entries(a)	ibdm_dump_service_entries(a)
    396 
    397 #else
    398 
    399 #define	ibdm_dump_mad_hdr(a)
    400 #define	ibdm_dump_ibmf_msg(a, b)
    401 #define	ibdm_dump_path_info(a)
    402 #define	ibdm_dump_classportinfo(a)
    403 #define	ibdm_dump_iounitinfo(a)
    404 #define	ibdm_dump_ioc_profile(a)
    405 #define	ibdm_dump_service_entries(a)
    406 #define	ibdm_dump_sweep_fabric_timestamp(a)
    407 
    408 #endif
    409 
    410 #ifdef __cplusplus
    411 }
    412 #endif
    413 
    414 #endif	/* _SYS_IB_MGT_IBDM_IBDM_IMPL_H */
    415