4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * ibdm.c
27 *
28 * This file contains the InifiniBand Device Manager (IBDM) support functions.
29 * IB nexus driver will only be the client for the IBDM module.
30 *
31 * IBDM registers with IBTF for HCA arrival/removal notification.
32 * IBDM registers with SA access to send DM MADs to discover the IOC's behind
33 * the IOU's.
34 *
35 * IB nexus driver registers with IBDM to find the information about the
36 * HCA's and IOC's (behind the IOU) present on the IB fabric.
37 */
38
39 #include <sys/systm.h>
40 #include <sys/taskq.h>
41 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
42 #include <sys/ib/mgt/ibmf/ibmf_impl.h>
43 #include <sys/ib/ibtl/impl/ibtl_ibnex.h>
44 #include <sys/modctl.h>
45
46 /* Function Prototype declarations */
47 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
48 static int ibdm_fini(void);
49 static int ibdm_init(void);
50 static int ibdm_get_reachable_ports(ibdm_port_attr_t *,
51 ibdm_hca_list_t *);
52 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
53 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
54 static boolean_t ibdm_is_cisco(ib_guid_t);
55 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *);
56 static void ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *);
57 static int ibdm_set_classportinfo(ibdm_dp_gidinfo_t *);
58 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
954 (void) ibt_query_hca(hca_hdl, hca_attr);
955
956 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
957 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
958 hca_attr->hca_version_id, hca_attr->hca_nports);
959
960 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
961 &size)) != IBT_SUCCESS) {
962 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
963 "ibt_query_hca_ports failed, status 0x%x", status);
964 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
965 (void) ibt_close_hca(hca_hdl);
966 return;
967 }
968 hca_list = (ibdm_hca_list_t *)
969 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
970 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
971 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
972 hca_list->hl_hca_guid = hca_attr->hca_node_guid;
973 hca_list->hl_nports = hca_attr->hca_nports;
974 hca_list->hl_attach_time = ddi_get_time();
975 hca_list->hl_hca_hdl = hca_hdl;
976
977 /*
978 * Init a dummy port attribute for the HCA node
979 * This is for Per-HCA Node. Initialize port_attr :
980 * hca_guid & port_guid -> hca_guid
981 * npkeys, pkey_tbl is NULL
982 * port_num, sn_prefix is 0
983 * vendorid, product_id, dev_version from HCA
984 * pa_state is IBT_PORT_ACTIVE
985 */
986 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
987 sizeof (ibdm_port_attr_t), KM_SLEEP);
988 port_attr = hca_list->hl_hca_port_attr;
989 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
990 port_attr->pa_productid = hca_attr->hca_device_id;
991 port_attr->pa_dev_version = hca_attr->hca_version_id;
992 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
993 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
994 port_attr->pa_port_guid = hca_attr->hca_node_guid;
4674 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4675 }
4676
4677
4678 /*
4679 * ibdm_ibnex_unregister_callbacks
4680 */
4681 void
4682 ibdm_ibnex_unregister_callback()
4683 {
4684 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4685 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4686 ibdm.ibdm_ibnex_callback = NULL;
4687 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4688 }
4689
4690 /*
4691 * ibdm_get_waittime()
4692 * Calculates the wait time based on the last HCA attach time
4693 */
4694 static time_t
4695 ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait)
4696 {
4697 int ii;
4698 time_t temp, wait_time = 0;
4699 ibdm_hca_list_t *hca;
4700
4701 IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx"
4702 "\tport settling time %d", hca_guid, dft_wait);
4703
4704 ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex));
4705
4706 hca = ibdm.ibdm_hca_list_head;
4707
4708 if (hca_guid) {
4709 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4710 if ((hca_guid == hca->hl_hca_guid) &&
4711 (hca->hl_nports != hca->hl_nports_active)) {
4712 wait_time =
4713 ddi_get_time() - hca->hl_attach_time;
4714 wait_time = ((wait_time >= dft_wait) ?
4715 0 : (dft_wait - wait_time));
4716 break;
4717 }
4718 hca = hca->hl_next;
4719 }
4720 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld secs",
4721 (long)wait_time);
4722 return (wait_time);
4723 }
4724
4725 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4726 if (hca->hl_nports != hca->hl_nports_active) {
4727 temp = ddi_get_time() - hca->hl_attach_time;
4728 temp = ((temp >= dft_wait) ? 0 : (dft_wait - temp));
4729 wait_time = (temp > wait_time) ? temp : wait_time;
4730 }
4731 hca = hca->hl_next;
4732 }
4733 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld secs",
4734 (long)wait_time);
4735 return (wait_time);
4736 }
4737
4738 void
4739 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait)
4740 {
4741 time_t wait_time;
4742 clock_t delta;
4743
4744 mutex_enter(&ibdm.ibdm_hl_mutex);
4745
4746 while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0) {
4747 if (wait_time > dft_wait) {
4748 IBTF_DPRINTF_L1("ibdm",
4749 "\tibnex_port_settle_wait: wait_time = %ld secs; "
4750 "Resetting to %d secs",
4751 (long)wait_time, dft_wait);
4752 wait_time = dft_wait;
4753 }
4754 delta = drv_usectohz(wait_time * 1000000);
4755 (void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv,
4756 &ibdm.ibdm_hl_mutex, delta, TR_CLOCK_TICK);
4757 }
4758
4759 mutex_exit(&ibdm.ibdm_hl_mutex);
4760 }
4761
4762
4763 /*
4764 * ibdm_ibnex_probe_hcaport
4765 * Probes the presence of HCA port (with HCA dip and port number)
4766 * Returns port attributes structure on SUCCESS
4767 */
4768 ibdm_port_attr_t *
4769 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4770 {
4771 int ii, jj;
4772 ibdm_hca_list_t *hca_list;
4773 ibdm_port_attr_t *port_attr;
4774
4775 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4776
4777 mutex_enter(&ibdm.ibdm_hl_mutex);
|
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
26 */
27
28 /*
29 * ibdm.c
30 *
31 * This file contains the InifiniBand Device Manager (IBDM) support functions.
32 * IB nexus driver will only be the client for the IBDM module.
33 *
34 * IBDM registers with IBTF for HCA arrival/removal notification.
35 * IBDM registers with SA access to send DM MADs to discover the IOC's behind
36 * the IOU's.
37 *
38 * IB nexus driver registers with IBDM to find the information about the
39 * HCA's and IOC's (behind the IOU) present on the IB fabric.
40 */
41
42 #include <sys/sysmacros.h>
43 #include <sys/systm.h>
44 #include <sys/taskq.h>
45 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
46 #include <sys/ib/mgt/ibmf/ibmf_impl.h>
47 #include <sys/ib/ibtl/impl/ibtl_ibnex.h>
48 #include <sys/modctl.h>
49
50 /* Function Prototype declarations */
51 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
52 static int ibdm_fini(void);
53 static int ibdm_init(void);
54 static int ibdm_get_reachable_ports(ibdm_port_attr_t *,
55 ibdm_hca_list_t *);
56 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
57 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
58 static boolean_t ibdm_is_cisco(ib_guid_t);
59 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *);
60 static void ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *);
61 static int ibdm_set_classportinfo(ibdm_dp_gidinfo_t *);
62 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
958 (void) ibt_query_hca(hca_hdl, hca_attr);
959
960 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
961 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
962 hca_attr->hca_version_id, hca_attr->hca_nports);
963
964 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
965 &size)) != IBT_SUCCESS) {
966 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
967 "ibt_query_hca_ports failed, status 0x%x", status);
968 kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
969 (void) ibt_close_hca(hca_hdl);
970 return;
971 }
972 hca_list = (ibdm_hca_list_t *)
973 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
974 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
975 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
976 hca_list->hl_hca_guid = hca_attr->hca_node_guid;
977 hca_list->hl_nports = hca_attr->hca_nports;
978 hca_list->hl_attach_time = gethrtime();
979 hca_list->hl_hca_hdl = hca_hdl;
980
981 /*
982 * Init a dummy port attribute for the HCA node
983 * This is for Per-HCA Node. Initialize port_attr :
984 * hca_guid & port_guid -> hca_guid
985 * npkeys, pkey_tbl is NULL
986 * port_num, sn_prefix is 0
987 * vendorid, product_id, dev_version from HCA
988 * pa_state is IBT_PORT_ACTIVE
989 */
990 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
991 sizeof (ibdm_port_attr_t), KM_SLEEP);
992 port_attr = hca_list->hl_hca_port_attr;
993 port_attr->pa_vendorid = hca_attr->hca_vendor_id;
994 port_attr->pa_productid = hca_attr->hca_device_id;
995 port_attr->pa_dev_version = hca_attr->hca_version_id;
996 port_attr->pa_hca_guid = hca_attr->hca_node_guid;
997 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl;
998 port_attr->pa_port_guid = hca_attr->hca_node_guid;
4678 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4679 }
4680
4681
4682 /*
4683 * ibdm_ibnex_unregister_callbacks
4684 */
4685 void
4686 ibdm_ibnex_unregister_callback()
4687 {
4688 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4689 mutex_enter(&ibdm.ibdm_ibnex_mutex);
4690 ibdm.ibdm_ibnex_callback = NULL;
4691 mutex_exit(&ibdm.ibdm_ibnex_mutex);
4692 }
4693
4694 /*
4695 * ibdm_get_waittime()
4696 * Calculates the wait time based on the last HCA attach time
4697 */
4698 static clock_t
4699 ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait_sec)
4700 {
4701 const hrtime_t dft_wait = dft_wait_sec * NANOSEC;
4702 hrtime_t temp, wait_time = 0;
4703 clock_t usecs;
4704 int i;
4705 ibdm_hca_list_t *hca;
4706
4707 IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx"
4708 "\tport settling time %d", hca_guid, dft_wait);
4709
4710 ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex));
4711
4712 hca = ibdm.ibdm_hca_list_head;
4713
4714 for (i = 0; i < ibdm.ibdm_hca_count; i++, hca = hca->hl_next) {
4715 if (hca->hl_nports == hca->hl_nports_active)
4716 continue;
4717
4718 if (hca_guid && (hca_guid != hca->hl_hca_guid))
4719 continue;
4720
4721 temp = gethrtime() - hca->hl_attach_time;
4722 temp = MAX(0, (dft_wait - temp));
4723
4724 if (hca_guid) {
4725 wait_time = temp;
4726 break;
4727 }
4728
4729 wait_time = MAX(temp, wait_time);
4730 }
4731
4732 /* convert to microseconds */
4733 usecs = MIN(wait_time, dft_wait) / (NANOSEC / MICROSEC);
4734
4735 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld usecs",
4736 (long)usecs);
4737
4738 return (drv_usectohz(usecs));
4739 }
4740
4741 void
4742 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait)
4743 {
4744 clock_t wait_time;
4745
4746 mutex_enter(&ibdm.ibdm_hl_mutex);
4747
4748 while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0)
4749 (void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv,
4750 &ibdm.ibdm_hl_mutex, wait_time, TR_CLOCK_TICK);
4751
4752 mutex_exit(&ibdm.ibdm_hl_mutex);
4753 }
4754
4755
4756 /*
4757 * ibdm_ibnex_probe_hcaport
4758 * Probes the presence of HCA port (with HCA dip and port number)
4759 * Returns port attributes structure on SUCCESS
4760 */
4761 ibdm_port_attr_t *
4762 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4763 {
4764 int ii, jj;
4765 ibdm_hca_list_t *hca_list;
4766 ibdm_port_attr_t *port_attr;
4767
4768 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4769
4770 mutex_enter(&ibdm.ibdm_hl_mutex);
|