1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
  24  */
  25 
  26 /*
  27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  29  * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
  30  * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
  31  * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
  32  */
  33 
  34 #include "ixgbe_sw.h"
  35 
  36 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
  37 static char ixgbe_version[] = "ixgbe 1.1.7";
  38 
  39 /*
  40  * Local function protoypes
  41  */
  42 static int ixgbe_register_mac(ixgbe_t *);
  43 static int ixgbe_identify_hardware(ixgbe_t *);
  44 static int ixgbe_regs_map(ixgbe_t *);
  45 static void ixgbe_init_properties(ixgbe_t *);
  46 static int ixgbe_init_driver_settings(ixgbe_t *);
  47 static void ixgbe_init_locks(ixgbe_t *);
  48 static void ixgbe_destroy_locks(ixgbe_t *);
  49 static int ixgbe_init(ixgbe_t *);
  50 static int ixgbe_chip_start(ixgbe_t *);
  51 static void ixgbe_chip_stop(ixgbe_t *);
  52 static int ixgbe_reset(ixgbe_t *);
  53 static void ixgbe_tx_clean(ixgbe_t *);
  54 static boolean_t ixgbe_tx_drain(ixgbe_t *);
  55 static boolean_t ixgbe_rx_drain(ixgbe_t *);
  56 static int ixgbe_alloc_rings(ixgbe_t *);
  57 static void ixgbe_free_rings(ixgbe_t *);
  58 static int ixgbe_alloc_rx_data(ixgbe_t *);
  59 static void ixgbe_free_rx_data(ixgbe_t *);
  60 static void ixgbe_setup_rings(ixgbe_t *);
  61 static void ixgbe_setup_rx(ixgbe_t *);
  62 static void ixgbe_setup_tx(ixgbe_t *);
  63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
  64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
  65 static void ixgbe_setup_rss(ixgbe_t *);
  66 static void ixgbe_setup_vmdq(ixgbe_t *);
  67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
  68 static void ixgbe_init_unicst(ixgbe_t *);
  69 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
  70 static void ixgbe_setup_multicst(ixgbe_t *);
  71 static void ixgbe_get_hw_state(ixgbe_t *);
  72 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
  73 static void ixgbe_get_conf(ixgbe_t *);
  74 static void ixgbe_init_params(ixgbe_t *);
  75 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
  76 static void ixgbe_driver_link_check(ixgbe_t *);
  77 static void ixgbe_sfp_check(void *);
  78 static void ixgbe_overtemp_check(void *);
  79 static void ixgbe_link_timer(void *);
  80 static void ixgbe_local_timer(void *);
  81 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
  82 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
  83 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
  84 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
  85 static boolean_t is_valid_mac_addr(uint8_t *);
  86 static boolean_t ixgbe_stall_check(ixgbe_t *);
  87 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
  88 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
  89 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
  90 static int ixgbe_alloc_intrs(ixgbe_t *);
  91 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
  92 static int ixgbe_add_intr_handlers(ixgbe_t *);
  93 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
  94 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
  95 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
  96 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
  97 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
  98 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
  99 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
 100 static void ixgbe_setup_adapter_vector(ixgbe_t *);
 101 static void ixgbe_rem_intr_handlers(ixgbe_t *);
 102 static void ixgbe_rem_intrs(ixgbe_t *);
 103 static int ixgbe_enable_intrs(ixgbe_t *);
 104 static int ixgbe_disable_intrs(ixgbe_t *);
 105 static uint_t ixgbe_intr_legacy(void *, void *);
 106 static uint_t ixgbe_intr_msi(void *, void *);
 107 static uint_t ixgbe_intr_msix(void *, void *);
 108 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
 109 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
 110 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
 111 static void ixgbe_get_driver_control(struct ixgbe_hw *);
 112 static int ixgbe_addmac(void *, const uint8_t *);
 113 static int ixgbe_remmac(void *, const uint8_t *);
 114 static void ixgbe_release_driver_control(struct ixgbe_hw *);
 115 
 116 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
 117 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
 118 static int ixgbe_resume(dev_info_t *);
 119 static int ixgbe_suspend(dev_info_t *);
 120 static int ixgbe_quiesce(dev_info_t *);
 121 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
 122 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
 123 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
 124 static int ixgbe_intr_cb_register(ixgbe_t *);
 125 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
 126 
 127 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
 128     const void *impl_data);
 129 static void ixgbe_fm_init(ixgbe_t *);
 130 static void ixgbe_fm_fini(ixgbe_t *);
 131 
 132 char *ixgbe_priv_props[] = {
 133         "_tx_copy_thresh",
 134         "_tx_recycle_thresh",
 135         "_tx_overload_thresh",
 136         "_tx_resched_thresh",
 137         "_rx_copy_thresh",
 138         "_rx_limit_per_intr",
 139         "_intr_throttling",
 140         "_adv_pause_cap",
 141         "_adv_asym_pause_cap",
 142         NULL
 143 };
 144 
 145 #define IXGBE_MAX_PRIV_PROPS \
 146         (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
 147 
 148 static struct cb_ops ixgbe_cb_ops = {
 149         nulldev,                /* cb_open */
 150         nulldev,                /* cb_close */
 151         nodev,                  /* cb_strategy */
 152         nodev,                  /* cb_print */
 153         nodev,                  /* cb_dump */
 154         nodev,                  /* cb_read */
 155         nodev,                  /* cb_write */
 156         nodev,                  /* cb_ioctl */
 157         nodev,                  /* cb_devmap */
 158         nodev,                  /* cb_mmap */
 159         nodev,                  /* cb_segmap */
 160         nochpoll,               /* cb_chpoll */
 161         ddi_prop_op,            /* cb_prop_op */
 162         NULL,                   /* cb_stream */
 163         D_MP | D_HOTPLUG,       /* cb_flag */
 164         CB_REV,                 /* cb_rev */
 165         nodev,                  /* cb_aread */
 166         nodev                   /* cb_awrite */
 167 };
 168 
 169 static struct dev_ops ixgbe_dev_ops = {
 170         DEVO_REV,               /* devo_rev */
 171         0,                      /* devo_refcnt */
 172         NULL,                   /* devo_getinfo */
 173         nulldev,                /* devo_identify */
 174         nulldev,                /* devo_probe */
 175         ixgbe_attach,           /* devo_attach */
 176         ixgbe_detach,           /* devo_detach */
 177         nodev,                  /* devo_reset */
 178         &ixgbe_cb_ops,              /* devo_cb_ops */
 179         NULL,                   /* devo_bus_ops */
 180         ddi_power,              /* devo_power */
 181         ixgbe_quiesce,          /* devo_quiesce */
 182 };
 183 
 184 static struct modldrv ixgbe_modldrv = {
 185         &mod_driverops,             /* Type of module.  This one is a driver */
 186         ixgbe_ident,            /* Discription string */
 187         &ixgbe_dev_ops              /* driver ops */
 188 };
 189 
 190 static struct modlinkage ixgbe_modlinkage = {
 191         MODREV_1, &ixgbe_modldrv, NULL
 192 };
 193 
 194 /*
 195  * Access attributes for register mapping
 196  */
 197 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
 198         DDI_DEVICE_ATTR_V1,
 199         DDI_STRUCTURE_LE_ACC,
 200         DDI_STRICTORDER_ACC,
 201         DDI_FLAGERR_ACC
 202 };
 203 
 204 /*
 205  * Loopback property
 206  */
 207 static lb_property_t lb_normal = {
 208         normal, "normal", IXGBE_LB_NONE
 209 };
 210 
 211 static lb_property_t lb_mac = {
 212         internal, "MAC", IXGBE_LB_INTERNAL_MAC
 213 };
 214 
 215 static lb_property_t lb_external = {
 216         external, "External", IXGBE_LB_EXTERNAL
 217 };
 218 
 219 #define IXGBE_M_CALLBACK_FLAGS \
 220         (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
 221 
 222 static mac_callbacks_t ixgbe_m_callbacks = {
 223         IXGBE_M_CALLBACK_FLAGS,
 224         ixgbe_m_stat,
 225         ixgbe_m_start,
 226         ixgbe_m_stop,
 227         ixgbe_m_promisc,
 228         ixgbe_m_multicst,
 229         NULL,
 230         NULL,
 231         NULL,
 232         ixgbe_m_ioctl,
 233         ixgbe_m_getcapab,
 234         NULL,
 235         NULL,
 236         ixgbe_m_setprop,
 237         ixgbe_m_getprop,
 238         ixgbe_m_propinfo
 239 };
 240 
 241 /*
 242  * Initialize capabilities of each supported adapter type
 243  */
 244 static adapter_info_t ixgbe_82598eb_cap = {
 245         64,             /* maximum number of rx queues */
 246         1,              /* minimum number of rx queues */
 247         64,             /* default number of rx queues */
 248         16,             /* maximum number of rx groups */
 249         1,              /* minimum number of rx groups */
 250         1,              /* default number of rx groups */
 251         32,             /* maximum number of tx queues */
 252         1,              /* minimum number of tx queues */
 253         8,              /* default number of tx queues */
 254         16366,          /* maximum MTU size */
 255         0xFFFF,         /* maximum interrupt throttle rate */
 256         0,              /* minimum interrupt throttle rate */
 257         200,            /* default interrupt throttle rate */
 258         18,             /* maximum total msix vectors */
 259         16,             /* maximum number of ring vectors */
 260         2,              /* maximum number of other vectors */
 261         IXGBE_EICR_LSC, /* "other" interrupt types handled */
 262         0,              /* "other" interrupt types enable mask */
 263         (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
 264         | IXGBE_FLAG_RSS_CAPABLE
 265         | IXGBE_FLAG_VMDQ_CAPABLE)
 266 };
 267 
 268 static adapter_info_t ixgbe_82599eb_cap = {
 269         128,            /* maximum number of rx queues */
 270         1,              /* minimum number of rx queues */
 271         128,            /* default number of rx queues */
 272         64,             /* maximum number of rx groups */
 273         1,              /* minimum number of rx groups */
 274         1,              /* default number of rx groups */
 275         128,            /* maximum number of tx queues */
 276         1,              /* minimum number of tx queues */
 277         8,              /* default number of tx queues */
 278         15500,          /* maximum MTU size */
 279         0xFF8,          /* maximum interrupt throttle rate */
 280         0,              /* minimum interrupt throttle rate */
 281         200,            /* default interrupt throttle rate */
 282         64,             /* maximum total msix vectors */
 283         16,             /* maximum number of ring vectors */
 284         2,              /* maximum number of other vectors */
 285         (IXGBE_EICR_LSC
 286         | IXGBE_EICR_GPI_SDP1
 287         | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
 288 
 289         (IXGBE_SDP1_GPIEN
 290         | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
 291 
 292         (IXGBE_FLAG_DCA_CAPABLE
 293         | IXGBE_FLAG_RSS_CAPABLE
 294         | IXGBE_FLAG_VMDQ_CAPABLE
 295         | IXGBE_FLAG_RSC_CAPABLE
 296         | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
 297 };
 298 
 299 static adapter_info_t ixgbe_X540_cap = {
 300         128,            /* maximum number of rx queues */
 301         1,              /* minimum number of rx queues */
 302         128,            /* default number of rx queues */
 303         64,             /* maximum number of rx groups */
 304         1,              /* minimum number of rx groups */
 305         1,              /* default number of rx groups */
 306         128,            /* maximum number of tx queues */
 307         1,              /* minimum number of tx queues */
 308         8,              /* default number of tx queues */
 309         15500,          /* maximum MTU size */
 310         0xFF8,          /* maximum interrupt throttle rate */
 311         0,              /* minimum interrupt throttle rate */
 312         200,            /* default interrupt throttle rate */
 313         64,             /* maximum total msix vectors */
 314         16,             /* maximum number of ring vectors */
 315         2,              /* maximum number of other vectors */
 316         (IXGBE_EICR_LSC
 317         | IXGBE_EICR_GPI_SDP1
 318         | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
 319 
 320         (IXGBE_SDP1_GPIEN
 321         | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
 322 
 323         (IXGBE_FLAG_DCA_CAPABLE
 324         | IXGBE_FLAG_RSS_CAPABLE
 325         | IXGBE_FLAG_VMDQ_CAPABLE
 326         | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
 327 };
 328 
 329 /*
 330  * Module Initialization Functions.
 331  */
 332 
 333 int
 334 _init(void)
 335 {
 336         int status;
 337 
 338         mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
 339 
 340         status = mod_install(&ixgbe_modlinkage);
 341 
 342         if (status != DDI_SUCCESS) {
 343                 mac_fini_ops(&ixgbe_dev_ops);
 344         }
 345 
 346         return (status);
 347 }
 348 
 349 int
 350 _fini(void)
 351 {
 352         int status;
 353 
 354         status = mod_remove(&ixgbe_modlinkage);
 355 
 356         if (status == DDI_SUCCESS) {
 357                 mac_fini_ops(&ixgbe_dev_ops);
 358         }
 359 
 360         return (status);
 361 }
 362 
 363 int
 364 _info(struct modinfo *modinfop)
 365 {
 366         int status;
 367 
 368         status = mod_info(&ixgbe_modlinkage, modinfop);
 369 
 370         return (status);
 371 }
 372 
 373 /*
 374  * ixgbe_attach - Driver attach.
 375  *
 376  * This function is the device specific initialization entry
 377  * point. This entry point is required and must be written.
 378  * The DDI_ATTACH command must be provided in the attach entry
 379  * point. When attach() is called with cmd set to DDI_ATTACH,
 380  * all normal kernel services (such as kmem_alloc(9F)) are
 381  * available for use by the driver.
 382  *
 383  * The attach() function will be called once for each instance
 384  * of  the  device  on  the  system with cmd set to DDI_ATTACH.
 385  * Until attach() succeeds, the only driver entry points which
 386  * may be called are open(9E) and getinfo(9E).
 387  */
 388 static int
 389 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
 390 {
 391         ixgbe_t *ixgbe;
 392         struct ixgbe_osdep *osdep;
 393         struct ixgbe_hw *hw;
 394         int instance;
 395         char taskqname[32];
 396 
 397         /*
 398          * Check the command and perform corresponding operations
 399          */
 400         switch (cmd) {
 401         default:
 402                 return (DDI_FAILURE);
 403 
 404         case DDI_RESUME:
 405                 return (ixgbe_resume(devinfo));
 406 
 407         case DDI_ATTACH:
 408                 break;
 409         }
 410 
 411         /* Get the device instance */
 412         instance = ddi_get_instance(devinfo);
 413 
 414         /* Allocate memory for the instance data structure */
 415         ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
 416 
 417         ixgbe->dip = devinfo;
 418         ixgbe->instance = instance;
 419 
 420         hw = &ixgbe->hw;
 421         osdep = &ixgbe->osdep;
 422         hw->back = osdep;
 423         osdep->ixgbe = ixgbe;
 424 
 425         /* Attach the instance pointer to the dev_info data structure */
 426         ddi_set_driver_private(devinfo, ixgbe);
 427 
 428         /*
 429          * Initialize for fma support
 430          */
 431         ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
 432             0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
 433             DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
 434         ixgbe_fm_init(ixgbe);
 435         ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
 436 
 437         /*
 438          * Map PCI config space registers
 439          */
 440         if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
 441                 ixgbe_error(ixgbe, "Failed to map PCI configurations");
 442                 goto attach_fail;
 443         }
 444         ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
 445 
 446         /*
 447          * Identify the chipset family
 448          */
 449         if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
 450                 ixgbe_error(ixgbe, "Failed to identify hardware");
 451                 goto attach_fail;
 452         }
 453 
 454         /*
 455          * Map device registers
 456          */
 457         if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
 458                 ixgbe_error(ixgbe, "Failed to map device registers");
 459                 goto attach_fail;
 460         }
 461         ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
 462 
 463         /*
 464          * Initialize driver parameters
 465          */
 466         ixgbe_init_properties(ixgbe);
 467         ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
 468 
 469         /*
 470          * Register interrupt callback
 471          */
 472         if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
 473                 ixgbe_error(ixgbe, "Failed to register interrupt callback");
 474                 goto attach_fail;
 475         }
 476 
 477         /*
 478          * Allocate interrupts
 479          */
 480         if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
 481                 ixgbe_error(ixgbe, "Failed to allocate interrupts");
 482                 goto attach_fail;
 483         }
 484         ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
 485 
 486         /*
 487          * Allocate rx/tx rings based on the ring numbers.
 488          * The actual numbers of rx/tx rings are decided by the number of
 489          * allocated interrupt vectors, so we should allocate the rings after
 490          * interrupts are allocated.
 491          */
 492         if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
 493                 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
 494                 goto attach_fail;
 495         }
 496         ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
 497 
 498         /*
 499          * Map rings to interrupt vectors
 500          */
 501         if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
 502                 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
 503                 goto attach_fail;
 504         }
 505 
 506         /*
 507          * Add interrupt handlers
 508          */
 509         if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
 510                 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
 511                 goto attach_fail;
 512         }
 513         ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
 514 
 515         /*
 516          * Create a taskq for sfp-change
 517          */
 518         (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
 519         if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
 520             1, TASKQ_DEFAULTPRI, 0)) == NULL) {
 521                 ixgbe_error(ixgbe, "sfp_taskq create failed");
 522                 goto attach_fail;
 523         }
 524         ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
 525 
 526         /*
 527          * Create a taskq for over-temp
 528          */
 529         (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
 530         if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
 531             1, TASKQ_DEFAULTPRI, 0)) == NULL) {
 532                 ixgbe_error(ixgbe, "overtemp_taskq create failed");
 533                 goto attach_fail;
 534         }
 535         ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
 536 
 537         /*
 538          * Initialize driver parameters
 539          */
 540         if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
 541                 ixgbe_error(ixgbe, "Failed to initialize driver settings");
 542                 goto attach_fail;
 543         }
 544 
 545         /*
 546          * Initialize mutexes for this device.
 547          * Do this before enabling the interrupt handler and
 548          * register the softint to avoid the condition where
 549          * interrupt handler can try using uninitialized mutex.
 550          */
 551         ixgbe_init_locks(ixgbe);
 552         ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
 553 
 554         /*
 555          * Initialize chipset hardware
 556          */
 557         if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
 558                 ixgbe_error(ixgbe, "Failed to initialize adapter");
 559                 goto attach_fail;
 560         }
 561         ixgbe->link_check_complete = B_FALSE;
 562         ixgbe->link_check_hrtime = gethrtime() +
 563             (IXGBE_LINK_UP_TIME * 100000000ULL);
 564         ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
 565 
 566         if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
 567                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
 568                 goto attach_fail;
 569         }
 570 
 571         /*
 572          * Initialize statistics
 573          */
 574         if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
 575                 ixgbe_error(ixgbe, "Failed to initialize statistics");
 576                 goto attach_fail;
 577         }
 578         ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
 579 
 580         /*
 581          * Register the driver to the MAC
 582          */
 583         if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
 584                 ixgbe_error(ixgbe, "Failed to register MAC");
 585                 goto attach_fail;
 586         }
 587         mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
 588         ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
 589 
 590         ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
 591             IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
 592         if (ixgbe->periodic_id == 0) {
 593                 ixgbe_error(ixgbe, "Failed to add the link check timer");
 594                 goto attach_fail;
 595         }
 596         ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
 597 
 598         /*
 599          * Now that mutex locks are initialized, and the chip is also
 600          * initialized, enable interrupts.
 601          */
 602         if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
 603                 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
 604                 goto attach_fail;
 605         }
 606         ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
 607 
 608         ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version);
 609         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
 610 
 611         return (DDI_SUCCESS);
 612 
 613 attach_fail:
 614         ixgbe_unconfigure(devinfo, ixgbe);
 615         return (DDI_FAILURE);
 616 }
 617 
 618 /*
 619  * ixgbe_detach - Driver detach.
 620  *
 621  * The detach() function is the complement of the attach routine.
 622  * If cmd is set to DDI_DETACH, detach() is used to remove  the
 623  * state  associated  with  a  given  instance of a device node
 624  * prior to the removal of that instance from the system.
 625  *
 626  * The detach() function will be called once for each  instance
 627  * of the device for which there has been a successful attach()
 628  * once there are no longer  any  opens  on  the  device.
 629  *
 630  * Interrupts routine are disabled, All memory allocated by this
 631  * driver are freed.
 632  */
 633 static int
 634 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
 635 {
 636         ixgbe_t *ixgbe;
 637 
 638         /*
 639          * Check detach command
 640          */
 641         switch (cmd) {
 642         default:
 643                 return (DDI_FAILURE);
 644 
 645         case DDI_SUSPEND:
 646                 return (ixgbe_suspend(devinfo));
 647 
 648         case DDI_DETACH:
 649                 break;
 650         }
 651 
 652         /*
 653          * Get the pointer to the driver private data structure
 654          */
 655         ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
 656         if (ixgbe == NULL)
 657                 return (DDI_FAILURE);
 658 
 659         /*
 660          * If the device is still running, it needs to be stopped first.
 661          * This check is necessary because under some specific circumstances,
 662          * the detach routine can be called without stopping the interface
 663          * first.
 664          */
 665         if (ixgbe->ixgbe_state & IXGBE_STARTED) {
 666                 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
 667                 mutex_enter(&ixgbe->gen_lock);
 668                 ixgbe_stop(ixgbe, B_TRUE);
 669                 mutex_exit(&ixgbe->gen_lock);
 670                 /* Disable and stop the watchdog timer */
 671                 ixgbe_disable_watchdog_timer(ixgbe);
 672         }
 673 
 674         /*
 675          * Check if there are still rx buffers held by the upper layer.
 676          * If so, fail the detach.
 677          */
 678         if (!ixgbe_rx_drain(ixgbe))
 679                 return (DDI_FAILURE);
 680 
 681         /*
 682          * Do the remaining unconfigure routines
 683          */
 684         ixgbe_unconfigure(devinfo, ixgbe);
 685 
 686         return (DDI_SUCCESS);
 687 }
 688 
 689 /*
 690  * quiesce(9E) entry point.
 691  *
 692  * This function is called when the system is single-threaded at high
 693  * PIL with preemption disabled. Therefore, this function must not be
 694  * blocked.
 695  *
 696  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
 697  * DDI_FAILURE indicates an error condition and should almost never happen.
 698  */
 699 static int
 700 ixgbe_quiesce(dev_info_t *devinfo)
 701 {
 702         ixgbe_t *ixgbe;
 703         struct ixgbe_hw *hw;
 704 
 705         ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
 706 
 707         if (ixgbe == NULL)
 708                 return (DDI_FAILURE);
 709 
 710         hw = &ixgbe->hw;
 711 
 712         /*
 713          * Disable the adapter interrupts
 714          */
 715         ixgbe_disable_adapter_interrupts(ixgbe);
 716 
 717         /*
 718          * Tell firmware driver is no longer in control
 719          */
 720         ixgbe_release_driver_control(hw);
 721 
 722         /*
 723          * Reset the chipset
 724          */
 725         (void) ixgbe_reset_hw(hw);
 726 
 727         /*
 728          * Reset PHY
 729          */
 730         (void) ixgbe_reset_phy(hw);
 731 
 732         return (DDI_SUCCESS);
 733 }
 734 
 735 static void
 736 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
 737 {
 738         /*
 739          * Disable interrupt
 740          */
 741         if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
 742                 (void) ixgbe_disable_intrs(ixgbe);
 743         }
 744 
 745         /*
 746          * remove the link check timer
 747          */
 748         if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
 749                 if (ixgbe->periodic_id != NULL) {
 750                         ddi_periodic_delete(ixgbe->periodic_id);
 751                         ixgbe->periodic_id = NULL;
 752                 }
 753         }
 754 
 755         /*
 756          * Unregister MAC
 757          */
 758         if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
 759                 (void) mac_unregister(ixgbe->mac_hdl);
 760         }
 761 
 762         /*
 763          * Free statistics
 764          */
 765         if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
 766                 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
 767         }
 768 
 769         /*
 770          * Remove interrupt handlers
 771          */
 772         if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
 773                 ixgbe_rem_intr_handlers(ixgbe);
 774         }
 775 
 776         /*
 777          * Remove taskq for sfp-status-change
 778          */
 779         if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
 780                 ddi_taskq_destroy(ixgbe->sfp_taskq);
 781         }
 782 
 783         /*
 784          * Remove taskq for over-temp
 785          */
 786         if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
 787                 ddi_taskq_destroy(ixgbe->overtemp_taskq);
 788         }
 789 
 790         /*
 791          * Remove interrupts
 792          */
 793         if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
 794                 ixgbe_rem_intrs(ixgbe);
 795         }
 796 
 797         /*
 798          * Unregister interrupt callback handler
 799          */
 800         (void) ddi_cb_unregister(ixgbe->cb_hdl);
 801 
 802         /*
 803          * Remove driver properties
 804          */
 805         if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
 806                 (void) ddi_prop_remove_all(devinfo);
 807         }
 808 
 809         /*
 810          * Stop the chipset
 811          */
 812         if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
 813                 mutex_enter(&ixgbe->gen_lock);
 814                 ixgbe_chip_stop(ixgbe);
 815                 mutex_exit(&ixgbe->gen_lock);
 816         }
 817 
 818         /*
 819          * Free register handle
 820          */
 821         if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
 822                 if (ixgbe->osdep.reg_handle != NULL)
 823                         ddi_regs_map_free(&ixgbe->osdep.reg_handle);
 824         }
 825 
 826         /*
 827          * Free PCI config handle
 828          */
 829         if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
 830                 if (ixgbe->osdep.cfg_handle != NULL)
 831                         pci_config_teardown(&ixgbe->osdep.cfg_handle);
 832         }
 833 
 834         /*
 835          * Free locks
 836          */
 837         if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
 838                 ixgbe_destroy_locks(ixgbe);
 839         }
 840 
 841         /*
 842          * Free the rx/tx rings
 843          */
 844         if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
 845                 ixgbe_free_rings(ixgbe);
 846         }
 847 
 848         /*
 849          * Unregister FMA capabilities
 850          */
 851         if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
 852                 ixgbe_fm_fini(ixgbe);
 853         }
 854 
 855         /*
 856          * Free the driver data structure
 857          */
 858         kmem_free(ixgbe, sizeof (ixgbe_t));
 859 
 860         ddi_set_driver_private(devinfo, NULL);
 861 }
 862 
 863 /*
 864  * ixgbe_register_mac - Register the driver and its function pointers with
 865  * the GLD interface.
 866  */
 867 static int
 868 ixgbe_register_mac(ixgbe_t *ixgbe)
 869 {
 870         struct ixgbe_hw *hw = &ixgbe->hw;
 871         mac_register_t *mac;
 872         int status;
 873 
 874         if ((mac = mac_alloc(MAC_VERSION)) == NULL)
 875                 return (IXGBE_FAILURE);
 876 
 877         mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
 878         mac->m_driver = ixgbe;
 879         mac->m_dip = ixgbe->dip;
 880         mac->m_src_addr = hw->mac.addr;
 881         mac->m_callbacks = &ixgbe_m_callbacks;
 882         mac->m_min_sdu = 0;
 883         mac->m_max_sdu = ixgbe->default_mtu;
 884         mac->m_margin = VLAN_TAGSZ;
 885         mac->m_priv_props = ixgbe_priv_props;
 886         mac->m_v12n = MAC_VIRT_LEVEL1;
 887 
 888         status = mac_register(mac, &ixgbe->mac_hdl);
 889 
 890         mac_free(mac);
 891 
 892         return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
 893 }
 894 
 895 /*
 896  * ixgbe_identify_hardware - Identify the type of the chipset.
 897  */
 898 static int
 899 ixgbe_identify_hardware(ixgbe_t *ixgbe)
 900 {
 901         struct ixgbe_hw *hw = &ixgbe->hw;
 902         struct ixgbe_osdep *osdep = &ixgbe->osdep;
 903 
 904         /*
 905          * Get the device id
 906          */
 907         hw->vendor_id =
 908             pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
 909         hw->device_id =
 910             pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
 911         hw->revision_id =
 912             pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
 913         hw->subsystem_device_id =
 914             pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
 915         hw->subsystem_vendor_id =
 916             pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
 917 
 918         /*
 919          * Set the mac type of the adapter based on the device id
 920          */
 921         if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
 922                 return (IXGBE_FAILURE);
 923         }
 924 
 925         /*
 926          * Install adapter capabilities
 927          */
 928         switch (hw->mac.type) {
 929         case ixgbe_mac_82598EB:
 930                 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
 931                 ixgbe->capab = &ixgbe_82598eb_cap;
 932 
 933                 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
 934                         ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
 935                         ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
 936                         ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
 937                 }
 938                 break;
 939 
 940         case ixgbe_mac_82599EB:
 941                 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
 942                 ixgbe->capab = &ixgbe_82599eb_cap;
 943 
 944                 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
 945                         ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
 946                         ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
 947                         ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
 948                 }
 949                 break;
 950 
 951         case ixgbe_mac_X540:
 952                 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
 953                 ixgbe->capab = &ixgbe_X540_cap;
 954                 /*
 955                  * For now, X540 is all set in its capab structure.
 956                  * As other X540 variants show up, things can change here.
 957                  */
 958                 break;
 959 
 960         default:
 961                 IXGBE_DEBUGLOG_1(ixgbe,
 962                     "adapter not supported in ixgbe_identify_hardware(): %d\n",
 963                     hw->mac.type);
 964                 return (IXGBE_FAILURE);
 965         }
 966 
 967         return (IXGBE_SUCCESS);
 968 }
 969 
 970 /*
 971  * ixgbe_regs_map - Map the device registers.
 972  *
 973  */
 974 static int
 975 ixgbe_regs_map(ixgbe_t *ixgbe)
 976 {
 977         dev_info_t *devinfo = ixgbe->dip;
 978         struct ixgbe_hw *hw = &ixgbe->hw;
 979         struct ixgbe_osdep *osdep = &ixgbe->osdep;
 980         off_t mem_size;
 981 
 982         /*
 983          * First get the size of device registers to be mapped.
 984          */
 985         if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
 986             != DDI_SUCCESS) {
 987                 return (IXGBE_FAILURE);
 988         }
 989 
 990         /*
 991          * Call ddi_regs_map_setup() to map registers
 992          */
 993         if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
 994             (caddr_t *)&hw->hw_addr, 0,
 995             mem_size, &ixgbe_regs_acc_attr,
 996             &osdep->reg_handle)) != DDI_SUCCESS) {
 997                 return (IXGBE_FAILURE);
 998         }
 999 
1000         return (IXGBE_SUCCESS);
1001 }
1002 
1003 /*
1004  * ixgbe_init_properties - Initialize driver properties.
1005  */
1006 static void
1007 ixgbe_init_properties(ixgbe_t *ixgbe)
1008 {
1009         /*
1010          * Get conf file properties, including link settings
1011          * jumbo frames, ring number, descriptor number, etc.
1012          */
1013         ixgbe_get_conf(ixgbe);
1014 
1015         ixgbe_init_params(ixgbe);
1016 }
1017 
1018 /*
1019  * ixgbe_init_driver_settings - Initialize driver settings.
1020  *
1021  * The settings include hardware function pointers, bus information,
1022  * rx/tx rings settings, link state, and any other parameters that
1023  * need to be setup during driver initialization.
1024  */
1025 static int
1026 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1027 {
1028         struct ixgbe_hw *hw = &ixgbe->hw;
1029         dev_info_t *devinfo = ixgbe->dip;
1030         ixgbe_rx_ring_t *rx_ring;
1031         ixgbe_rx_group_t *rx_group;
1032         ixgbe_tx_ring_t *tx_ring;
1033         uint32_t rx_size;
1034         uint32_t tx_size;
1035         uint32_t ring_per_group;
1036         int i;
1037 
1038         /*
1039          * Initialize chipset specific hardware function pointers
1040          */
1041         if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1042                 return (IXGBE_FAILURE);
1043         }
1044 
1045         /*
1046          * Get the system page size
1047          */
1048         ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1049 
1050         /*
1051          * Set rx buffer size
1052          *
1053          * The IP header alignment room is counted in the calculation.
1054          * The rx buffer size is in unit of 1K that is required by the
1055          * chipset hardware.
1056          */
1057         rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1058         ixgbe->rx_buf_size = ((rx_size >> 10) +
1059             ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1060 
1061         /*
1062          * Set tx buffer size
1063          */
1064         tx_size = ixgbe->max_frame_size;
1065         ixgbe->tx_buf_size = ((tx_size >> 10) +
1066             ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1067 
1068         /*
1069          * Initialize rx/tx rings/groups parameters
1070          */
1071         ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1072         for (i = 0; i < ixgbe->num_rx_rings; i++) {
1073                 rx_ring = &ixgbe->rx_rings[i];
1074                 rx_ring->index = i;
1075                 rx_ring->ixgbe = ixgbe;
1076                 rx_ring->group_index = i / ring_per_group;
1077                 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1078         }
1079 
1080         for (i = 0; i < ixgbe->num_rx_groups; i++) {
1081                 rx_group = &ixgbe->rx_groups[i];
1082                 rx_group->index = i;
1083                 rx_group->ixgbe = ixgbe;
1084         }
1085 
1086         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1087                 tx_ring = &ixgbe->tx_rings[i];
1088                 tx_ring->index = i;
1089                 tx_ring->ixgbe = ixgbe;
1090                 if (ixgbe->tx_head_wb_enable)
1091                         tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1092                 else
1093                         tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1094 
1095                 tx_ring->ring_size = ixgbe->tx_ring_size;
1096                 tx_ring->free_list_size = ixgbe->tx_ring_size +
1097                     (ixgbe->tx_ring_size >> 1);
1098         }
1099 
1100         /*
1101          * Initialize values of interrupt throttling rate
1102          */
1103         for (i = 1; i < MAX_INTR_VECTOR; i++)
1104                 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1105 
1106         /*
1107          * The initial link state should be "unknown"
1108          */
1109         ixgbe->link_state = LINK_STATE_UNKNOWN;
1110 
1111         return (IXGBE_SUCCESS);
1112 }
1113 
1114 /*
1115  * ixgbe_init_locks - Initialize locks.
1116  */
1117 static void
1118 ixgbe_init_locks(ixgbe_t *ixgbe)
1119 {
1120         ixgbe_rx_ring_t *rx_ring;
1121         ixgbe_tx_ring_t *tx_ring;
1122         int i;
1123 
1124         for (i = 0; i < ixgbe->num_rx_rings; i++) {
1125                 rx_ring = &ixgbe->rx_rings[i];
1126                 mutex_init(&rx_ring->rx_lock, NULL,
1127                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1128         }
1129 
1130         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1131                 tx_ring = &ixgbe->tx_rings[i];
1132                 mutex_init(&tx_ring->tx_lock, NULL,
1133                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1134                 mutex_init(&tx_ring->recycle_lock, NULL,
1135                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1136                 mutex_init(&tx_ring->tcb_head_lock, NULL,
1137                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1138                 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1139                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1140         }
1141 
1142         mutex_init(&ixgbe->gen_lock, NULL,
1143             MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1144 
1145         mutex_init(&ixgbe->watchdog_lock, NULL,
1146             MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1147 }
1148 
1149 /*
1150  * ixgbe_destroy_locks - Destroy locks.
1151  */
1152 static void
1153 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1154 {
1155         ixgbe_rx_ring_t *rx_ring;
1156         ixgbe_tx_ring_t *tx_ring;
1157         int i;
1158 
1159         for (i = 0; i < ixgbe->num_rx_rings; i++) {
1160                 rx_ring = &ixgbe->rx_rings[i];
1161                 mutex_destroy(&rx_ring->rx_lock);
1162         }
1163 
1164         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1165                 tx_ring = &ixgbe->tx_rings[i];
1166                 mutex_destroy(&tx_ring->tx_lock);
1167                 mutex_destroy(&tx_ring->recycle_lock);
1168                 mutex_destroy(&tx_ring->tcb_head_lock);
1169                 mutex_destroy(&tx_ring->tcb_tail_lock);
1170         }
1171 
1172         mutex_destroy(&ixgbe->gen_lock);
1173         mutex_destroy(&ixgbe->watchdog_lock);
1174 }
1175 
1176 static int
1177 ixgbe_resume(dev_info_t *devinfo)
1178 {
1179         ixgbe_t *ixgbe;
1180         int i;
1181 
1182         ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1183         if (ixgbe == NULL)
1184                 return (DDI_FAILURE);
1185 
1186         mutex_enter(&ixgbe->gen_lock);
1187 
1188         if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1189                 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1190                         mutex_exit(&ixgbe->gen_lock);
1191                         return (DDI_FAILURE);
1192                 }
1193 
1194                 /*
1195                  * Enable and start the watchdog timer
1196                  */
1197                 ixgbe_enable_watchdog_timer(ixgbe);
1198         }
1199 
1200         atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1201 
1202         if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1203                 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1204                         mac_tx_ring_update(ixgbe->mac_hdl,
1205                             ixgbe->tx_rings[i].ring_handle);
1206                 }
1207         }
1208 
1209         mutex_exit(&ixgbe->gen_lock);
1210 
1211         return (DDI_SUCCESS);
1212 }
1213 
1214 static int
1215 ixgbe_suspend(dev_info_t *devinfo)
1216 {
1217         ixgbe_t *ixgbe;
1218 
1219         ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1220         if (ixgbe == NULL)
1221                 return (DDI_FAILURE);
1222 
1223         mutex_enter(&ixgbe->gen_lock);
1224 
1225         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1226         if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1227                 mutex_exit(&ixgbe->gen_lock);
1228                 return (DDI_SUCCESS);
1229         }
1230         ixgbe_stop(ixgbe, B_FALSE);
1231 
1232         mutex_exit(&ixgbe->gen_lock);
1233 
1234         /*
1235          * Disable and stop the watchdog timer
1236          */
1237         ixgbe_disable_watchdog_timer(ixgbe);
1238 
1239         return (DDI_SUCCESS);
1240 }
1241 
1242 /*
1243  * ixgbe_init - Initialize the device.
1244  */
1245 static int
1246 ixgbe_init(ixgbe_t *ixgbe)
1247 {
1248         struct ixgbe_hw *hw = &ixgbe->hw;
1249         u8 pbanum[IXGBE_PBANUM_LENGTH];
1250 
1251         mutex_enter(&ixgbe->gen_lock);
1252 
1253         /*
1254          * Reset chipset to put the hardware in a known state
1255          * before we try to do anything with the eeprom.
1256          */
1257         if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) {
1258                 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1259                 goto init_fail;
1260         }
1261 
1262         /*
1263          * Need to init eeprom before validating the checksum.
1264          */
1265         if (ixgbe_init_eeprom_params(hw) < 0) {
1266                 ixgbe_error(ixgbe,
1267                     "Unable to intitialize the eeprom interface.");
1268                 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1269                 goto init_fail;
1270         }
1271 
1272         /*
1273          * NVM validation
1274          */
1275         if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1276                 /*
1277                  * Some PCI-E parts fail the first check due to
1278                  * the link being in sleep state.  Call it again,
1279                  * if it fails a second time it's a real issue.
1280                  */
1281                 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1282                         ixgbe_error(ixgbe,
1283                             "Invalid NVM checksum. Please contact "
1284                             "the vendor to update the NVM.");
1285                         ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1286                         goto init_fail;
1287                 }
1288         }
1289 
1290         /*
1291          * Setup default flow control thresholds - enable/disable
1292          * & flow control type is controlled by ixgbe.conf
1293          */
1294         hw->fc.high_water[0] = DEFAULT_FCRTH;
1295         hw->fc.low_water[0] = DEFAULT_FCRTL;
1296         hw->fc.pause_time = DEFAULT_FCPAUSE;
1297         hw->fc.send_xon = B_TRUE;
1298 
1299         /*
1300          * Initialize link settings
1301          */
1302         (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1303 
1304         /*
1305          * Initialize the chipset hardware
1306          */
1307         if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1308                 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1309                 goto init_fail;
1310         }
1311 
1312         /*
1313          * Read identifying information and place in devinfo.
1314          */
1315         pbanum[0] = '\0';
1316         (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1317         if (*pbanum != '\0') {
1318                 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1319                     "printed-board-assembly", (char *)pbanum);
1320         }
1321 
1322         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1323                 goto init_fail;
1324         }
1325 
1326         mutex_exit(&ixgbe->gen_lock);
1327         return (IXGBE_SUCCESS);
1328 
1329 init_fail:
1330         /*
1331          * Reset PHY
1332          */
1333         (void) ixgbe_reset_phy(hw);
1334 
1335         mutex_exit(&ixgbe->gen_lock);
1336         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1337         return (IXGBE_FAILURE);
1338 }
1339 
1340 /*
1341  * ixgbe_chip_start - Initialize and start the chipset hardware.
1342  */
1343 static int
1344 ixgbe_chip_start(ixgbe_t *ixgbe)
1345 {
1346         struct ixgbe_hw *hw = &ixgbe->hw;
1347         int ret_val, i;
1348 
1349         ASSERT(mutex_owned(&ixgbe->gen_lock));
1350 
1351         /*
1352          * Get the mac address
1353          * This function should handle SPARC case correctly.
1354          */
1355         if (!ixgbe_find_mac_address(ixgbe)) {
1356                 ixgbe_error(ixgbe, "Failed to get the mac address");
1357                 return (IXGBE_FAILURE);
1358         }
1359 
1360         /*
1361          * Validate the mac address
1362          */
1363         (void) ixgbe_init_rx_addrs(hw);
1364         if (!is_valid_mac_addr(hw->mac.addr)) {
1365                 ixgbe_error(ixgbe, "Invalid mac address");
1366                 return (IXGBE_FAILURE);
1367         }
1368 
1369         /*
1370          * Configure/Initialize hardware
1371          */
1372         ret_val = ixgbe_init_hw(hw);
1373         if (ret_val != IXGBE_SUCCESS) {
1374                 if (ret_val == IXGBE_ERR_EEPROM_VERSION) {
1375                         ixgbe_error(ixgbe,
1376                             "This 82599 device is pre-release and contains"
1377                             " outdated firmware, please contact your hardware"
1378                             " vendor for a replacement.");
1379                 } else {
1380                         ixgbe_error(ixgbe, "Failed to initialize hardware");
1381                         return (IXGBE_FAILURE);
1382                 }
1383         }
1384 
1385         /*
1386          * Re-enable relaxed ordering for performance.  It is disabled
1387          * by default in the hardware init.
1388          */
1389         if (ixgbe->relax_order_enable == B_TRUE)
1390                 ixgbe_enable_relaxed_ordering(hw);
1391 
1392         /*
1393          * Setup adapter interrupt vectors
1394          */
1395         ixgbe_setup_adapter_vector(ixgbe);
1396 
1397         /*
1398          * Initialize unicast addresses.
1399          */
1400         ixgbe_init_unicst(ixgbe);
1401 
1402         /*
1403          * Setup and initialize the mctable structures.
1404          */
1405         ixgbe_setup_multicst(ixgbe);
1406 
1407         /*
1408          * Set interrupt throttling rate
1409          */
1410         for (i = 0; i < ixgbe->intr_cnt; i++) {
1411                 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1412         }
1413 
1414         /*
1415          * Save the state of the phy
1416          */
1417         ixgbe_get_hw_state(ixgbe);
1418 
1419         /*
1420          * Make sure driver has control
1421          */
1422         ixgbe_get_driver_control(hw);
1423 
1424         return (IXGBE_SUCCESS);
1425 }
1426 
1427 /*
1428  * ixgbe_chip_stop - Stop the chipset hardware
1429  */
1430 static void
1431 ixgbe_chip_stop(ixgbe_t *ixgbe)
1432 {
1433         struct ixgbe_hw *hw = &ixgbe->hw;
1434 
1435         ASSERT(mutex_owned(&ixgbe->gen_lock));
1436 
1437         /*
1438          * Tell firmware driver is no longer in control
1439          */
1440         ixgbe_release_driver_control(hw);
1441 
1442         /*
1443          * Reset the chipset
1444          */
1445         (void) ixgbe_reset_hw(hw);
1446 
1447         /*
1448          * Reset PHY
1449          */
1450         (void) ixgbe_reset_phy(hw);
1451 }
1452 
1453 /*
1454  * ixgbe_reset - Reset the chipset and re-start the driver.
1455  *
1456  * It involves stopping and re-starting the chipset,
1457  * and re-configuring the rx/tx rings.
1458  */
1459 static int
1460 ixgbe_reset(ixgbe_t *ixgbe)
1461 {
1462         int i;
1463 
1464         /*
1465          * Disable and stop the watchdog timer
1466          */
1467         ixgbe_disable_watchdog_timer(ixgbe);
1468 
1469         mutex_enter(&ixgbe->gen_lock);
1470 
1471         ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1472         atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1473 
1474         ixgbe_stop(ixgbe, B_FALSE);
1475 
1476         if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1477                 mutex_exit(&ixgbe->gen_lock);
1478                 return (IXGBE_FAILURE);
1479         }
1480 
1481         /*
1482          * After resetting, need to recheck the link status.
1483          */
1484         ixgbe->link_check_complete = B_FALSE;
1485         ixgbe->link_check_hrtime = gethrtime() +
1486             (IXGBE_LINK_UP_TIME * 100000000ULL);
1487 
1488         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1489 
1490         if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1491                 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1492                         mac_tx_ring_update(ixgbe->mac_hdl,
1493                             ixgbe->tx_rings[i].ring_handle);
1494                 }
1495         }
1496 
1497         mutex_exit(&ixgbe->gen_lock);
1498 
1499         /*
1500          * Enable and start the watchdog timer
1501          */
1502         ixgbe_enable_watchdog_timer(ixgbe);
1503 
1504         return (IXGBE_SUCCESS);
1505 }
1506 
1507 /*
1508  * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1509  */
1510 static void
1511 ixgbe_tx_clean(ixgbe_t *ixgbe)
1512 {
1513         ixgbe_tx_ring_t *tx_ring;
1514         tx_control_block_t *tcb;
1515         link_list_t pending_list;
1516         uint32_t desc_num;
1517         int i, j;
1518 
1519         LINK_LIST_INIT(&pending_list);
1520 
1521         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1522                 tx_ring = &ixgbe->tx_rings[i];
1523 
1524                 mutex_enter(&tx_ring->recycle_lock);
1525 
1526                 /*
1527                  * Clean the pending tx data - the pending packets in the
1528                  * work_list that have no chances to be transmitted again.
1529                  *
1530                  * We must ensure the chipset is stopped or the link is down
1531                  * before cleaning the transmit packets.
1532                  */
1533                 desc_num = 0;
1534                 for (j = 0; j < tx_ring->ring_size; j++) {
1535                         tcb = tx_ring->work_list[j];
1536                         if (tcb != NULL) {
1537                                 desc_num += tcb->desc_num;
1538 
1539                                 tx_ring->work_list[j] = NULL;
1540 
1541                                 ixgbe_free_tcb(tcb);
1542 
1543                                 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1544                         }
1545                 }
1546 
1547                 if (desc_num > 0) {
1548                         atomic_add_32(&tx_ring->tbd_free, desc_num);
1549                         ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1550 
1551                         /*
1552                          * Reset the head and tail pointers of the tbd ring;
1553                          * Reset the writeback head if it's enable.
1554                          */
1555                         tx_ring->tbd_head = 0;
1556                         tx_ring->tbd_tail = 0;
1557                         if (ixgbe->tx_head_wb_enable)
1558                                 *tx_ring->tbd_head_wb = 0;
1559 
1560                         IXGBE_WRITE_REG(&ixgbe->hw,
1561                             IXGBE_TDH(tx_ring->index), 0);
1562                         IXGBE_WRITE_REG(&ixgbe->hw,
1563                             IXGBE_TDT(tx_ring->index), 0);
1564                 }
1565 
1566                 mutex_exit(&tx_ring->recycle_lock);
1567 
1568                 /*
1569                  * Add the tx control blocks in the pending list to
1570                  * the free list.
1571                  */
1572                 ixgbe_put_free_list(tx_ring, &pending_list);
1573         }
1574 }
1575 
1576 /*
1577  * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1578  * transmitted.
1579  */
1580 static boolean_t
1581 ixgbe_tx_drain(ixgbe_t *ixgbe)
1582 {
1583         ixgbe_tx_ring_t *tx_ring;
1584         boolean_t done;
1585         int i, j;
1586 
1587         /*
1588          * Wait for a specific time to allow pending tx packets
1589          * to be transmitted.
1590          *
1591          * Check the counter tbd_free to see if transmission is done.
1592          * No lock protection is needed here.
1593          *
1594          * Return B_TRUE if all pending packets have been transmitted;
1595          * Otherwise return B_FALSE;
1596          */
1597         for (i = 0; i < TX_DRAIN_TIME; i++) {
1598 
1599                 done = B_TRUE;
1600                 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1601                         tx_ring = &ixgbe->tx_rings[j];
1602                         done = done &&
1603                             (tx_ring->tbd_free == tx_ring->ring_size);
1604                 }
1605 
1606                 if (done)
1607                         break;
1608 
1609                 msec_delay(1);
1610         }
1611 
1612         return (done);
1613 }
1614 
1615 /*
1616  * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1617  */
1618 static boolean_t
1619 ixgbe_rx_drain(ixgbe_t *ixgbe)
1620 {
1621         boolean_t done = B_TRUE;
1622         int i;
1623 
1624         /*
1625          * Polling the rx free list to check if those rx buffers held by
1626          * the upper layer are released.
1627          *
1628          * Check the counter rcb_free to see if all pending buffers are
1629          * released. No lock protection is needed here.
1630          *
1631          * Return B_TRUE if all pending buffers have been released;
1632          * Otherwise return B_FALSE;
1633          */
1634         for (i = 0; i < RX_DRAIN_TIME; i++) {
1635                 done = (ixgbe->rcb_pending == 0);
1636 
1637                 if (done)
1638                         break;
1639 
1640                 msec_delay(1);
1641         }
1642 
1643         return (done);
1644 }
1645 
1646 /*
1647  * ixgbe_start - Start the driver/chipset.
1648  */
1649 int
1650 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1651 {
1652         int i;
1653 
1654         ASSERT(mutex_owned(&ixgbe->gen_lock));
1655 
1656         if (alloc_buffer) {
1657                 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1658                         ixgbe_error(ixgbe,
1659                             "Failed to allocate software receive rings");
1660                         return (IXGBE_FAILURE);
1661                 }
1662 
1663                 /* Allocate buffers for all the rx/tx rings */
1664                 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1665                         ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1666                         return (IXGBE_FAILURE);
1667                 }
1668 
1669                 ixgbe->tx_ring_init = B_TRUE;
1670         } else {
1671                 ixgbe->tx_ring_init = B_FALSE;
1672         }
1673 
1674         for (i = 0; i < ixgbe->num_rx_rings; i++)
1675                 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1676         for (i = 0; i < ixgbe->num_tx_rings; i++)
1677                 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1678 
1679         /*
1680          * Start the chipset hardware
1681          */
1682         if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1683                 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1684                 goto start_failure;
1685         }
1686 
1687         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1688                 goto start_failure;
1689         }
1690 
1691         /*
1692          * Setup the rx/tx rings
1693          */
1694         ixgbe_setup_rings(ixgbe);
1695 
1696         /*
1697          * ixgbe_start() will be called when resetting, however if reset
1698          * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1699          * before enabling the interrupts.
1700          */
1701         atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1702             | IXGBE_STALL| IXGBE_OVERTEMP));
1703 
1704         /*
1705          * Enable adapter interrupts
1706          * The interrupts must be enabled after the driver state is START
1707          */
1708         ixgbe_enable_adapter_interrupts(ixgbe);
1709 
1710         for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1711                 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1712         for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1713                 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1714 
1715         return (IXGBE_SUCCESS);
1716 
1717 start_failure:
1718         for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1719                 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1720         for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1721                 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1722 
1723         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1724 
1725         return (IXGBE_FAILURE);
1726 }
1727 
1728 /*
1729  * ixgbe_stop - Stop the driver/chipset.
1730  */
1731 void
1732 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1733 {
1734         int i;
1735 
1736         ASSERT(mutex_owned(&ixgbe->gen_lock));
1737 
1738         /*
1739          * Disable the adapter interrupts
1740          */
1741         ixgbe_disable_adapter_interrupts(ixgbe);
1742 
1743         /*
1744          * Drain the pending tx packets
1745          */
1746         (void) ixgbe_tx_drain(ixgbe);
1747 
1748         for (i = 0; i < ixgbe->num_rx_rings; i++)
1749                 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1750         for (i = 0; i < ixgbe->num_tx_rings; i++)
1751                 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1752 
1753         /*
1754          * Stop the chipset hardware
1755          */
1756         ixgbe_chip_stop(ixgbe);
1757 
1758         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1759                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1760         }
1761 
1762         /*
1763          * Clean the pending tx data/resources
1764          */
1765         ixgbe_tx_clean(ixgbe);
1766 
1767         for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1768                 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1769         for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1770                 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1771 
1772         if (ixgbe->link_state == LINK_STATE_UP) {
1773                 ixgbe->link_state = LINK_STATE_UNKNOWN;
1774                 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1775         }
1776 
1777         if (free_buffer) {
1778                 /*
1779                  * Release the DMA/memory resources of rx/tx rings
1780                  */
1781                 ixgbe_free_dma(ixgbe);
1782                 ixgbe_free_rx_data(ixgbe);
1783         }
1784 }
1785 
1786 /*
1787  * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1788  */
1789 /* ARGSUSED */
1790 static int
1791 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1792     void *arg1, void *arg2)
1793 {
1794         ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1795 
1796         switch (cbaction) {
1797         /* IRM callback */
1798         int count;
1799         case DDI_CB_INTR_ADD:
1800         case DDI_CB_INTR_REMOVE:
1801                 count = (int)(uintptr_t)cbarg;
1802                 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1803                 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1804                     int, ixgbe->intr_cnt);
1805                 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1806                     DDI_SUCCESS) {
1807                         ixgbe_error(ixgbe,
1808                             "IRM CB: Failed to adjust interrupts");
1809                         goto cb_fail;
1810                 }
1811                 break;
1812         default:
1813                 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1814                     cbaction);
1815                 return (DDI_ENOTSUP);
1816         }
1817         return (DDI_SUCCESS);
1818 cb_fail:
1819         return (DDI_FAILURE);
1820 }
1821 
1822 /*
1823  * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1824  */
1825 static int
1826 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1827 {
1828         int i, rc, actual;
1829 
1830         if (count == 0)
1831                 return (DDI_SUCCESS);
1832 
1833         if ((cbaction == DDI_CB_INTR_ADD &&
1834             ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
1835             (cbaction == DDI_CB_INTR_REMOVE &&
1836             ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
1837                 return (DDI_FAILURE);
1838 
1839         if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1840                 return (DDI_FAILURE);
1841         }
1842 
1843         for (i = 0; i < ixgbe->num_rx_rings; i++)
1844                 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
1845         for (i = 0; i < ixgbe->num_tx_rings; i++)
1846                 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
1847 
1848         mutex_enter(&ixgbe->gen_lock);
1849         ixgbe->ixgbe_state &= ~IXGBE_STARTED;
1850         ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
1851         ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
1852         mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
1853 
1854         ixgbe_stop(ixgbe, B_FALSE);
1855         /*
1856          * Disable interrupts
1857          */
1858         if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1859                 rc = ixgbe_disable_intrs(ixgbe);
1860                 ASSERT(rc == IXGBE_SUCCESS);
1861         }
1862         ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
1863 
1864         /*
1865          * Remove interrupt handlers
1866          */
1867         if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
1868                 ixgbe_rem_intr_handlers(ixgbe);
1869         }
1870         ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
1871 
1872         /*
1873          * Clear vect_map
1874          */
1875         bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
1876         switch (cbaction) {
1877         case DDI_CB_INTR_ADD:
1878                 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
1879                     DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
1880                     DDI_INTR_ALLOC_NORMAL);
1881                 if (rc != DDI_SUCCESS || actual != count) {
1882                         ixgbe_log(ixgbe, "Adjust interrupts failed."
1883                             "return: %d, irm cb size: %d, actual: %d",
1884                             rc, count, actual);
1885                         goto intr_adjust_fail;
1886                 }
1887                 ixgbe->intr_cnt += count;
1888                 break;
1889 
1890         case DDI_CB_INTR_REMOVE:
1891                 for (i = ixgbe->intr_cnt - count;
1892                     i < ixgbe->intr_cnt; i ++) {
1893                         rc = ddi_intr_free(ixgbe->htable[i]);
1894                         ixgbe->htable[i] = NULL;
1895                         if (rc != DDI_SUCCESS) {
1896                                 ixgbe_log(ixgbe, "Adjust interrupts failed."
1897                                     "return: %d, irm cb size: %d, actual: %d",
1898                                     rc, count, actual);
1899                                 goto intr_adjust_fail;
1900                         }
1901                 }
1902                 ixgbe->intr_cnt -= count;
1903                 break;
1904         }
1905 
1906         /*
1907          * Get priority for first vector, assume remaining are all the same
1908          */
1909         rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
1910         if (rc != DDI_SUCCESS) {
1911                 ixgbe_log(ixgbe,
1912                     "Get interrupt priority failed: %d", rc);
1913                 goto intr_adjust_fail;
1914         }
1915         rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
1916         if (rc != DDI_SUCCESS) {
1917                 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
1918                 goto intr_adjust_fail;
1919         }
1920         ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
1921 
1922         /*
1923          * Map rings to interrupt vectors
1924          */
1925         if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
1926                 ixgbe_error(ixgbe,
1927                     "IRM CB: Failed to map interrupts to vectors");
1928                 goto intr_adjust_fail;
1929         }
1930 
1931         /*
1932          * Add interrupt handlers
1933          */
1934         if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
1935                 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
1936                 goto intr_adjust_fail;
1937         }
1938         ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
1939 
1940         /*
1941          * Now that mutex locks are initialized, and the chip is also
1942          * initialized, enable interrupts.
1943          */
1944         if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
1945                 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
1946                 goto intr_adjust_fail;
1947         }
1948         ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
1949         if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1950                 ixgbe_error(ixgbe, "IRM CB: Failed to start");
1951                 goto intr_adjust_fail;
1952         }
1953         ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
1954         ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
1955         ixgbe->ixgbe_state |= IXGBE_STARTED;
1956         mutex_exit(&ixgbe->gen_lock);
1957 
1958         for (i = 0; i < ixgbe->num_rx_rings; i++) {
1959                 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
1960                     ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
1961         }
1962         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1963                 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
1964                     ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
1965         }
1966 
1967         /* Wakeup all Tx rings */
1968         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1969                 mac_tx_ring_update(ixgbe->mac_hdl,
1970                     ixgbe->tx_rings[i].ring_handle);
1971         }
1972 
1973         IXGBE_DEBUGLOG_3(ixgbe,
1974             "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
1975             ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
1976         return (DDI_SUCCESS);
1977 
1978 intr_adjust_fail:
1979         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1980         mutex_exit(&ixgbe->gen_lock);
1981         return (DDI_FAILURE);
1982 }
1983 
1984 /*
1985  * ixgbe_intr_cb_register - Register interrupt callback function.
1986  */
1987 static int
1988 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
1989 {
1990         if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
1991             ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
1992                 return (IXGBE_FAILURE);
1993         }
1994         IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
1995         return (IXGBE_SUCCESS);
1996 }
1997 
1998 /*
1999  * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
2000  */
2001 static int
2002 ixgbe_alloc_rings(ixgbe_t *ixgbe)
2003 {
2004         /*
2005          * Allocate memory space for rx rings
2006          */
2007         ixgbe->rx_rings = kmem_zalloc(
2008             sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
2009             KM_NOSLEEP);
2010 
2011         if (ixgbe->rx_rings == NULL) {
2012                 return (IXGBE_FAILURE);
2013         }
2014 
2015         /*
2016          * Allocate memory space for tx rings
2017          */
2018         ixgbe->tx_rings = kmem_zalloc(
2019             sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2020             KM_NOSLEEP);
2021 
2022         if (ixgbe->tx_rings == NULL) {
2023                 kmem_free(ixgbe->rx_rings,
2024                     sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2025                 ixgbe->rx_rings = NULL;
2026                 return (IXGBE_FAILURE);
2027         }
2028 
2029         /*
2030          * Allocate memory space for rx ring groups
2031          */
2032         ixgbe->rx_groups = kmem_zalloc(
2033             sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2034             KM_NOSLEEP);
2035 
2036         if (ixgbe->rx_groups == NULL) {
2037                 kmem_free(ixgbe->rx_rings,
2038                     sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2039                 kmem_free(ixgbe->tx_rings,
2040                     sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2041                 ixgbe->rx_rings = NULL;
2042                 ixgbe->tx_rings = NULL;
2043                 return (IXGBE_FAILURE);
2044         }
2045 
2046         return (IXGBE_SUCCESS);
2047 }
2048 
2049 /*
2050  * ixgbe_free_rings - Free the memory space of rx/tx rings.
2051  */
2052 static void
2053 ixgbe_free_rings(ixgbe_t *ixgbe)
2054 {
2055         if (ixgbe->rx_rings != NULL) {
2056                 kmem_free(ixgbe->rx_rings,
2057                     sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2058                 ixgbe->rx_rings = NULL;
2059         }
2060 
2061         if (ixgbe->tx_rings != NULL) {
2062                 kmem_free(ixgbe->tx_rings,
2063                     sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2064                 ixgbe->tx_rings = NULL;
2065         }
2066 
2067         if (ixgbe->rx_groups != NULL) {
2068                 kmem_free(ixgbe->rx_groups,
2069                     sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2070                 ixgbe->rx_groups = NULL;
2071         }
2072 }
2073 
2074 static int
2075 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2076 {
2077         ixgbe_rx_ring_t *rx_ring;
2078         int i;
2079 
2080         for (i = 0; i < ixgbe->num_rx_rings; i++) {
2081                 rx_ring = &ixgbe->rx_rings[i];
2082                 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2083                         goto alloc_rx_rings_failure;
2084         }
2085         return (IXGBE_SUCCESS);
2086 
2087 alloc_rx_rings_failure:
2088         ixgbe_free_rx_data(ixgbe);
2089         return (IXGBE_FAILURE);
2090 }
2091 
2092 static void
2093 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2094 {
2095         ixgbe_rx_ring_t *rx_ring;
2096         ixgbe_rx_data_t *rx_data;
2097         int i;
2098 
2099         for (i = 0; i < ixgbe->num_rx_rings; i++) {
2100                 rx_ring = &ixgbe->rx_rings[i];
2101 
2102                 mutex_enter(&ixgbe->rx_pending_lock);
2103                 rx_data = rx_ring->rx_data;
2104 
2105                 if (rx_data != NULL) {
2106                         rx_data->flag |= IXGBE_RX_STOPPED;
2107 
2108                         if (rx_data->rcb_pending == 0) {
2109                                 ixgbe_free_rx_ring_data(rx_data);
2110                                 rx_ring->rx_data = NULL;
2111                         }
2112                 }
2113 
2114                 mutex_exit(&ixgbe->rx_pending_lock);
2115         }
2116 }
2117 
2118 /*
2119  * ixgbe_setup_rings - Setup rx/tx rings.
2120  */
2121 static void
2122 ixgbe_setup_rings(ixgbe_t *ixgbe)
2123 {
2124         /*
2125          * Setup the rx/tx rings, including the following:
2126          *
2127          * 1. Setup the descriptor ring and the control block buffers;
2128          * 2. Initialize necessary registers for receive/transmit;
2129          * 3. Initialize software pointers/parameters for receive/transmit;
2130          */
2131         ixgbe_setup_rx(ixgbe);
2132 
2133         ixgbe_setup_tx(ixgbe);
2134 }
2135 
2136 static void
2137 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2138 {
2139         ixgbe_t *ixgbe = rx_ring->ixgbe;
2140         ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2141         struct ixgbe_hw *hw = &ixgbe->hw;
2142         rx_control_block_t *rcb;
2143         union ixgbe_adv_rx_desc *rbd;
2144         uint32_t size;
2145         uint32_t buf_low;
2146         uint32_t buf_high;
2147         uint32_t reg_val;
2148         int i;
2149 
2150         ASSERT(mutex_owned(&rx_ring->rx_lock));
2151         ASSERT(mutex_owned(&ixgbe->gen_lock));
2152 
2153         for (i = 0; i < ixgbe->rx_ring_size; i++) {
2154                 rcb = rx_data->work_list[i];
2155                 rbd = &rx_data->rbd_ring[i];
2156 
2157                 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2158                 rbd->read.hdr_addr = NULL;
2159         }
2160 
2161         /*
2162          * Initialize the length register
2163          */
2164         size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2165         IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2166 
2167         /*
2168          * Initialize the base address registers
2169          */
2170         buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2171         buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2172         IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2173         IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2174 
2175         /*
2176          * Setup head & tail pointers
2177          */
2178         IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2179             rx_data->ring_size - 1);
2180         IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2181 
2182         rx_data->rbd_next = 0;
2183         rx_data->lro_first = 0;
2184 
2185         /*
2186          * Setup the Receive Descriptor Control Register (RXDCTL)
2187          * PTHRESH=32 descriptors (half the internal cache)
2188          * HTHRESH=0 descriptors (to minimize latency on fetch)
2189          * WTHRESH defaults to 1 (writeback each descriptor)
2190          */
2191         reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2192         reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2193 
2194         /* Not a valid value for 82599 or X540 */
2195         if (hw->mac.type == ixgbe_mac_82598EB) {
2196                 reg_val |= 0x0020;      /* pthresh */
2197         }
2198         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2199 
2200         if (hw->mac.type == ixgbe_mac_82599EB ||
2201             hw->mac.type == ixgbe_mac_X540) {
2202                 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2203                 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2204                 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2205         }
2206 
2207         /*
2208          * Setup the Split and Replication Receive Control Register.
2209          * Set the rx buffer size and the advanced descriptor type.
2210          */
2211         reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2212             IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2213         reg_val |= IXGBE_SRRCTL_DROP_EN;
2214         IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2215 }
2216 
2217 static void
2218 ixgbe_setup_rx(ixgbe_t *ixgbe)
2219 {
2220         ixgbe_rx_ring_t *rx_ring;
2221         struct ixgbe_hw *hw = &ixgbe->hw;
2222         uint32_t reg_val;
2223         uint32_t ring_mapping;
2224         uint32_t i, index;
2225         uint32_t psrtype_rss_bit;
2226 
2227         /* PSRTYPE must be configured for 82599 */
2228         if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2229             ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2230                 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2231                     IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2232                 reg_val |= IXGBE_PSRTYPE_L2HDR;
2233                 reg_val |= 0x80000000;
2234                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2235         } else {
2236                 if (ixgbe->num_rx_groups > 32) {
2237                         psrtype_rss_bit = 0x20000000;
2238                 } else {
2239                         psrtype_rss_bit = 0x40000000;
2240                 }
2241                 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2242                         reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2243                             IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2244                         reg_val |= IXGBE_PSRTYPE_L2HDR;
2245                         reg_val |= psrtype_rss_bit;
2246                         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2247                 }
2248         }
2249 
2250         /*
2251          * Set filter control in FCTRL to accept broadcast packets and do
2252          * not pass pause frames to host.  Flow control settings are already
2253          * in this register, so preserve them.
2254          */
2255         reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2256         reg_val |= IXGBE_FCTRL_BAM;     /* broadcast accept mode */
2257         reg_val |= IXGBE_FCTRL_DPF;     /* discard pause frames */
2258         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2259 
2260         /*
2261          * Hardware checksum settings
2262          */
2263         if (ixgbe->rx_hcksum_enable) {
2264                 reg_val = IXGBE_RXCSUM_IPPCSE;  /* IP checksum */
2265                 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2266         }
2267 
2268         /*
2269          * Setup VMDq and RSS for multiple receive queues
2270          */
2271         switch (ixgbe->classify_mode) {
2272         case IXGBE_CLASSIFY_RSS:
2273                 /*
2274                  * One group, only RSS is needed when more than
2275                  * one ring enabled.
2276                  */
2277                 ixgbe_setup_rss(ixgbe);
2278                 break;
2279 
2280         case IXGBE_CLASSIFY_VMDQ:
2281                 /*
2282                  * Multiple groups, each group has one ring,
2283                  * only VMDq is needed.
2284                  */
2285                 ixgbe_setup_vmdq(ixgbe);
2286                 break;
2287 
2288         case IXGBE_CLASSIFY_VMDQ_RSS:
2289                 /*
2290                  * Multiple groups and multiple rings, both
2291                  * VMDq and RSS are needed.
2292                  */
2293                 ixgbe_setup_vmdq_rss(ixgbe);
2294                 break;
2295 
2296         default:
2297                 break;
2298         }
2299 
2300         /*
2301          * Enable the receive unit.  This must be done after filter
2302          * control is set in FCTRL.
2303          */
2304         reg_val = (IXGBE_RXCTRL_RXEN    /* Enable Receive Unit */
2305             | IXGBE_RXCTRL_DMBYPS);     /* descriptor monitor bypass */
2306         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val);
2307 
2308         /*
2309          * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2310          */
2311         for (i = 0; i < ixgbe->num_rx_rings; i++) {
2312                 rx_ring = &ixgbe->rx_rings[i];
2313                 ixgbe_setup_rx_ring(rx_ring);
2314         }
2315 
2316         /*
2317          * Setup the per-ring statistics mapping.
2318          */
2319         ring_mapping = 0;
2320         for (i = 0; i < ixgbe->num_rx_rings; i++) {
2321                 index = ixgbe->rx_rings[i].hw_index;
2322                 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2323                 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2324                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2325         }
2326 
2327         /*
2328          * The Max Frame Size in MHADD/MAXFRS will be internally increased
2329          * by four bytes if the packet has a VLAN field, so includes MTU,
2330          * ethernet header and frame check sequence.
2331          * Register is MAXFRS in 82599.
2332          */
2333         reg_val = (ixgbe->default_mtu + sizeof (struct ether_header)
2334             + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2335         IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2336 
2337         /*
2338          * Setup Jumbo Frame enable bit
2339          */
2340         if (ixgbe->default_mtu > ETHERMTU) {
2341                 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2342                 reg_val |= IXGBE_HLREG0_JUMBOEN;
2343                 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2344         }
2345 
2346         /*
2347          * Setup RSC for multiple receive queues.
2348          */
2349         if (ixgbe->lro_enable) {
2350                 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2351                         /*
2352                          * Make sure rx_buf_size * MAXDESC not greater
2353                          * than 65535.
2354                          * Intel recommends 4 for MAXDESC field value.
2355                          */
2356                         reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2357                         reg_val |= IXGBE_RSCCTL_RSCEN;
2358                         if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2359                                 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2360                         else
2361                                 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2362                         IXGBE_WRITE_REG(hw,  IXGBE_RSCCTL(i), reg_val);
2363                 }
2364 
2365                 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2366                 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2367                 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2368 
2369                 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2370                 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2371                 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2372                 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2373 
2374                 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2375         }
2376 }
2377 
2378 static void
2379 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2380 {
2381         ixgbe_t *ixgbe = tx_ring->ixgbe;
2382         struct ixgbe_hw *hw = &ixgbe->hw;
2383         uint32_t size;
2384         uint32_t buf_low;
2385         uint32_t buf_high;
2386         uint32_t reg_val;
2387 
2388         ASSERT(mutex_owned(&tx_ring->tx_lock));
2389         ASSERT(mutex_owned(&ixgbe->gen_lock));
2390 
2391         /*
2392          * Initialize the length register
2393          */
2394         size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2395         IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2396 
2397         /*
2398          * Initialize the base address registers
2399          */
2400         buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2401         buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2402         IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2403         IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2404 
2405         /*
2406          * Setup head & tail pointers
2407          */
2408         IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2409         IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2410 
2411         /*
2412          * Setup head write-back
2413          */
2414         if (ixgbe->tx_head_wb_enable) {
2415                 /*
2416                  * The memory of the head write-back is allocated using
2417                  * the extra tbd beyond the tail of the tbd ring.
2418                  */
2419                 tx_ring->tbd_head_wb = (uint32_t *)
2420                     ((uintptr_t)tx_ring->tbd_area.address + size);
2421                 *tx_ring->tbd_head_wb = 0;
2422 
2423                 buf_low = (uint32_t)
2424                     (tx_ring->tbd_area.dma_address + size);
2425                 buf_high = (uint32_t)
2426                     ((tx_ring->tbd_area.dma_address + size) >> 32);
2427 
2428                 /* Set the head write-back enable bit */
2429                 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2430 
2431                 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2432                 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2433 
2434                 /*
2435                  * Turn off relaxed ordering for head write back or it will
2436                  * cause problems with the tx recycling
2437                  */
2438 
2439                 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2440                     IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2441                     IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2442                 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2443                 if (hw->mac.type == ixgbe_mac_82598EB) {
2444                         IXGBE_WRITE_REG(hw,
2445                             IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2446                 } else {
2447                         IXGBE_WRITE_REG(hw,
2448                             IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2449                 }
2450         } else {
2451                 tx_ring->tbd_head_wb = NULL;
2452         }
2453 
2454         tx_ring->tbd_head = 0;
2455         tx_ring->tbd_tail = 0;
2456         tx_ring->tbd_free = tx_ring->ring_size;
2457 
2458         if (ixgbe->tx_ring_init == B_TRUE) {
2459                 tx_ring->tcb_head = 0;
2460                 tx_ring->tcb_tail = 0;
2461                 tx_ring->tcb_free = tx_ring->free_list_size;
2462         }
2463 
2464         /*
2465          * Initialize the s/w context structure
2466          */
2467         bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2468 }
2469 
2470 static void
2471 ixgbe_setup_tx(ixgbe_t *ixgbe)
2472 {
2473         struct ixgbe_hw *hw = &ixgbe->hw;
2474         ixgbe_tx_ring_t *tx_ring;
2475         uint32_t reg_val;
2476         uint32_t ring_mapping;
2477         int i;
2478 
2479         for (i = 0; i < ixgbe->num_tx_rings; i++) {
2480                 tx_ring = &ixgbe->tx_rings[i];
2481                 ixgbe_setup_tx_ring(tx_ring);
2482         }
2483 
2484         /*
2485          * Setup the per-ring statistics mapping.
2486          */
2487         ring_mapping = 0;
2488         for (i = 0; i < ixgbe->num_tx_rings; i++) {
2489                 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2490                 if ((i & 0x3) == 0x3) {
2491                         switch (hw->mac.type) {
2492                         case ixgbe_mac_82598EB:
2493                                 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2494                                     ring_mapping);
2495                                 break;
2496 
2497                         case ixgbe_mac_82599EB:
2498                         case ixgbe_mac_X540:
2499                                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2500                                     ring_mapping);
2501                                 break;
2502 
2503                         default:
2504                                 break;
2505                         }
2506 
2507                         ring_mapping = 0;
2508                 }
2509         }
2510         if (i & 0x3) {
2511                 switch (hw->mac.type) {
2512                 case ixgbe_mac_82598EB:
2513                         IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2514                         break;
2515 
2516                 case ixgbe_mac_82599EB:
2517                 case ixgbe_mac_X540:
2518                         IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2519                         break;
2520 
2521                 default:
2522                         break;
2523                 }
2524         }
2525 
2526         /*
2527          * Enable CRC appending and TX padding (for short tx frames)
2528          */
2529         reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2530         reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2531         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2532 
2533         /*
2534          * enable DMA for 82599 and X540 parts
2535          */
2536         if (hw->mac.type == ixgbe_mac_82599EB ||
2537             hw->mac.type == ixgbe_mac_X540) {
2538                 /* DMATXCTL.TE must be set after all Tx config is complete */
2539                 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2540                 reg_val |= IXGBE_DMATXCTL_TE;
2541                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2542 
2543                 /* Disable arbiter to set MTQC */
2544                 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2545                 reg_val |= IXGBE_RTTDCS_ARBDIS;
2546                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2547                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2548                 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2549                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2550         }
2551 
2552         /*
2553          * Enabling tx queues ..
2554          * For 82599 must be done after DMATXCTL.TE is set
2555          */
2556         for (i = 0; i < ixgbe->num_tx_rings; i++) {
2557                 tx_ring = &ixgbe->tx_rings[i];
2558                 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2559                 reg_val |= IXGBE_TXDCTL_ENABLE;
2560                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2561         }
2562 }
2563 
2564 /*
2565  * ixgbe_setup_rss - Setup receive-side scaling feature.
2566  */
2567 static void
2568 ixgbe_setup_rss(ixgbe_t *ixgbe)
2569 {
2570         struct ixgbe_hw *hw = &ixgbe->hw;
2571         uint32_t i, mrqc, rxcsum;
2572         uint32_t random;
2573         uint32_t reta;
2574         uint32_t ring_per_group;
2575 
2576         /*
2577          * Fill out redirection table
2578          */
2579         reta = 0;
2580         ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2581 
2582         for (i = 0; i < 128; i++) {
2583                 reta = (reta << 8) | (i % ring_per_group) |
2584                     ((i % ring_per_group) << 4);
2585                 if ((i & 3) == 3)
2586                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2587         }
2588 
2589         /*
2590          * Fill out hash function seeds with a random constant
2591          */
2592         for (i = 0; i < 10; i++) {
2593                 (void) random_get_pseudo_bytes((uint8_t *)&random,
2594                     sizeof (uint32_t));
2595                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2596         }
2597 
2598         /*
2599          * Enable RSS & perform hash on these packet types
2600          */
2601         mrqc = IXGBE_MRQC_RSSEN |
2602             IXGBE_MRQC_RSS_FIELD_IPV4 |
2603             IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2604             IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2605             IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2606             IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2607             IXGBE_MRQC_RSS_FIELD_IPV6 |
2608             IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2609             IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2610             IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2611         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2612 
2613         /*
2614          * Disable Packet Checksum to enable RSS for multiple receive queues.
2615          * It is an adapter hardware limitation that Packet Checksum is
2616          * mutually exclusive with RSS.
2617          */
2618         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2619         rxcsum |= IXGBE_RXCSUM_PCSD;
2620         rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2621         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2622 }
2623 
2624 /*
2625  * ixgbe_setup_vmdq - Setup MAC classification feature
2626  */
2627 static void
2628 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2629 {
2630         struct ixgbe_hw *hw = &ixgbe->hw;
2631         uint32_t vmdctl, i, vtctl;
2632 
2633         /*
2634          * Setup the VMDq Control register, enable VMDq based on
2635          * packet destination MAC address:
2636          */
2637         switch (hw->mac.type) {
2638         case ixgbe_mac_82598EB:
2639                 /*
2640                  * VMDq Enable = 1;
2641                  * VMDq Filter = 0; MAC filtering
2642                  * Default VMDq output index = 0;
2643                  */
2644                 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2645                 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2646                 break;
2647 
2648         case ixgbe_mac_82599EB:
2649         case ixgbe_mac_X540:
2650                 /*
2651                  * Enable VMDq-only.
2652                  */
2653                 vmdctl = IXGBE_MRQC_VMDQEN;
2654                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2655 
2656                 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2657                         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2658                         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2659                 }
2660 
2661                 /*
2662                  * Enable Virtualization and Replication.
2663                  */
2664                 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2665                 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2666 
2667                 /*
2668                  * Enable receiving packets to all VFs
2669                  */
2670                 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2671                 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2672                 break;
2673 
2674         default:
2675                 break;
2676         }
2677 }
2678 
2679 /*
2680  * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2681  */
2682 static void
2683 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2684 {
2685         struct ixgbe_hw *hw = &ixgbe->hw;
2686         uint32_t i, mrqc, rxcsum;
2687         uint32_t random;
2688         uint32_t reta;
2689         uint32_t ring_per_group;
2690         uint32_t vmdctl, vtctl;
2691 
2692         /*
2693          * Fill out redirection table
2694          */
2695         reta = 0;
2696         ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2697         for (i = 0; i < 128; i++) {
2698                 reta = (reta << 8) | (i % ring_per_group) |
2699                     ((i % ring_per_group) << 4);
2700                 if ((i & 3) == 3)
2701                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2702         }
2703 
2704         /*
2705          * Fill out hash function seeds with a random constant
2706          */
2707         for (i = 0; i < 10; i++) {
2708                 (void) random_get_pseudo_bytes((uint8_t *)&random,
2709                     sizeof (uint32_t));
2710                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2711         }
2712 
2713         /*
2714          * Enable and setup RSS and VMDq
2715          */
2716         switch (hw->mac.type) {
2717         case ixgbe_mac_82598EB:
2718                 /*
2719                  * Enable RSS & Setup RSS Hash functions
2720                  */
2721                 mrqc = IXGBE_MRQC_RSSEN |
2722                     IXGBE_MRQC_RSS_FIELD_IPV4 |
2723                     IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2724                     IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2725                     IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2726                     IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2727                     IXGBE_MRQC_RSS_FIELD_IPV6 |
2728                     IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2729                     IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2730                     IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2731                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2732 
2733                 /*
2734                  * Enable and Setup VMDq
2735                  * VMDq Filter = 0; MAC filtering
2736                  * Default VMDq output index = 0;
2737                  */
2738                 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2739                 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2740                 break;
2741 
2742         case ixgbe_mac_82599EB:
2743         case ixgbe_mac_X540:
2744                 /*
2745                  * Enable RSS & Setup RSS Hash functions
2746                  */
2747                 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2748                     IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2749                     IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2750                     IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2751                     IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2752                     IXGBE_MRQC_RSS_FIELD_IPV6 |
2753                     IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2754                     IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2755                     IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2756 
2757                 /*
2758                  * Enable VMDq+RSS.
2759                  */
2760                 if (ixgbe->num_rx_groups > 32)  {
2761                         mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2762                 } else {
2763                         mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2764                 }
2765 
2766                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2767 
2768                 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2769                         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2770                         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2771                 }
2772                 break;
2773 
2774         default:
2775                 break;
2776 
2777         }
2778 
2779         /*
2780          * Disable Packet Checksum to enable RSS for multiple receive queues.
2781          * It is an adapter hardware limitation that Packet Checksum is
2782          * mutually exclusive with RSS.
2783          */
2784         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2785         rxcsum |= IXGBE_RXCSUM_PCSD;
2786         rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2787         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2788 
2789         if (hw->mac.type == ixgbe_mac_82599EB ||
2790             hw->mac.type == ixgbe_mac_X540) {
2791                 /*
2792                  * Enable Virtualization and Replication.
2793                  */
2794                 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2795                 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2796 
2797                 /*
2798                  * Enable receiving packets to all VFs
2799                  */
2800                 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2801                 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2802         }
2803 }
2804 
2805 /*
2806  * ixgbe_init_unicst - Initialize the unicast addresses.
2807  */
2808 static void
2809 ixgbe_init_unicst(ixgbe_t *ixgbe)
2810 {
2811         struct ixgbe_hw *hw = &ixgbe->hw;
2812         uint8_t *mac_addr;
2813         int slot;
2814         /*
2815          * Here we should consider two situations:
2816          *
2817          * 1. Chipset is initialized at the first time,
2818          *    Clear all the multiple unicast addresses.
2819          *
2820          * 2. Chipset is reset
2821          *    Recover the multiple unicast addresses from the
2822          *    software data structure to the RAR registers.
2823          */
2824         if (!ixgbe->unicst_init) {
2825                 /*
2826                  * Initialize the multiple unicast addresses
2827                  */
2828                 ixgbe->unicst_total = hw->mac.num_rar_entries;
2829                 ixgbe->unicst_avail = ixgbe->unicst_total;
2830                 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2831                         mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2832                         bzero(mac_addr, ETHERADDRL);
2833                         (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
2834                         ixgbe->unicst_addr[slot].mac.set = 0;
2835                 }
2836                 ixgbe->unicst_init = B_TRUE;
2837         } else {
2838                 /* Re-configure the RAR registers */
2839                 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2840                         mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2841                         if (ixgbe->unicst_addr[slot].mac.set == 1) {
2842                                 (void) ixgbe_set_rar(hw, slot, mac_addr,
2843                                     ixgbe->unicst_addr[slot].mac.group_index,
2844                                     IXGBE_RAH_AV);
2845                         } else {
2846                                 bzero(mac_addr, ETHERADDRL);
2847                                 (void) ixgbe_set_rar(hw, slot, mac_addr,
2848                                     NULL, NULL);
2849                         }
2850                 }
2851         }
2852 }
2853 
2854 /*
2855  * ixgbe_unicst_find - Find the slot for the specified unicast address
2856  */
2857 int
2858 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
2859 {
2860         int slot;
2861 
2862         ASSERT(mutex_owned(&ixgbe->gen_lock));
2863 
2864         for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2865                 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
2866                     mac_addr, ETHERADDRL) == 0)
2867                         return (slot);
2868         }
2869 
2870         return (-1);
2871 }
2872 
2873 /*
2874  * ixgbe_multicst_add - Add a multicst address.
2875  */
2876 int
2877 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2878 {
2879         ASSERT(mutex_owned(&ixgbe->gen_lock));
2880 
2881         if ((multiaddr[0] & 01) == 0) {
2882                 return (EINVAL);
2883         }
2884 
2885         if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
2886                 return (ENOENT);
2887         }
2888 
2889         bcopy(multiaddr,
2890             &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
2891         ixgbe->mcast_count++;
2892 
2893         /*
2894          * Update the multicast table in the hardware
2895          */
2896         ixgbe_setup_multicst(ixgbe);
2897 
2898         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2899                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2900                 return (EIO);
2901         }
2902 
2903         return (0);
2904 }
2905 
2906 /*
2907  * ixgbe_multicst_remove - Remove a multicst address.
2908  */
2909 int
2910 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2911 {
2912         int i;
2913 
2914         ASSERT(mutex_owned(&ixgbe->gen_lock));
2915 
2916         for (i = 0; i < ixgbe->mcast_count; i++) {
2917                 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
2918                     ETHERADDRL) == 0) {
2919                         for (i++; i < ixgbe->mcast_count; i++) {
2920                                 ixgbe->mcast_table[i - 1] =
2921                                     ixgbe->mcast_table[i];
2922                         }
2923                         ixgbe->mcast_count--;
2924                         break;
2925                 }
2926         }
2927 
2928         /*
2929          * Update the multicast table in the hardware
2930          */
2931         ixgbe_setup_multicst(ixgbe);
2932 
2933         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2934                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2935                 return (EIO);
2936         }
2937 
2938         return (0);
2939 }
2940 
2941 /*
2942  * ixgbe_setup_multicast - Setup multicast data structures.
2943  *
2944  * This routine initializes all of the multicast related structures
2945  * and save them in the hardware registers.
2946  */
2947 static void
2948 ixgbe_setup_multicst(ixgbe_t *ixgbe)
2949 {
2950         uint8_t *mc_addr_list;
2951         uint32_t mc_addr_count;
2952         struct ixgbe_hw *hw = &ixgbe->hw;
2953 
2954         ASSERT(mutex_owned(&ixgbe->gen_lock));
2955 
2956         ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
2957 
2958         mc_addr_list = (uint8_t *)ixgbe->mcast_table;
2959         mc_addr_count = ixgbe->mcast_count;
2960 
2961         /*
2962          * Update the multicast addresses to the MTA registers
2963          */
2964         (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
2965             ixgbe_mc_table_itr, TRUE);
2966 }
2967 
2968 /*
2969  * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
2970  *
2971  * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
2972  * Different chipsets may have different allowed configuration of vmdq and rss.
2973  */
2974 static void
2975 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
2976 {
2977         struct ixgbe_hw *hw = &ixgbe->hw;
2978         uint32_t ring_per_group;
2979 
2980         switch (hw->mac.type) {
2981         case ixgbe_mac_82598EB:
2982                 /*
2983                  * 82598 supports the following combination:
2984                  * vmdq no. x rss no.
2985                  * [5..16]  x 1
2986                  * [1..4]   x [1..16]
2987                  * However 8 rss queue per pool (vmdq) is sufficient for
2988                  * most cases.
2989                  */
2990                 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2991                 if (ixgbe->num_rx_groups > 4) {
2992                         ixgbe->num_rx_rings = ixgbe->num_rx_groups;
2993                 } else {
2994                         ixgbe->num_rx_rings = ixgbe->num_rx_groups *
2995                             min(8, ring_per_group);
2996                 }
2997 
2998                 break;
2999 
3000         case ixgbe_mac_82599EB:
3001         case ixgbe_mac_X540:
3002                 /*
3003                  * 82599 supports the following combination:
3004                  * vmdq no. x rss no.
3005                  * [33..64] x [1..2]
3006                  * [2..32]  x [1..4]
3007                  * 1 x [1..16]
3008                  * However 8 rss queue per pool (vmdq) is sufficient for
3009                  * most cases.
3010                  *
3011                  * For now, treat X540 like the 82599.
3012                  */
3013                 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3014                 if (ixgbe->num_rx_groups == 1) {
3015                         ixgbe->num_rx_rings = min(8, ring_per_group);
3016                 } else if (ixgbe->num_rx_groups <= 32) {
3017                         ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3018                             min(4, ring_per_group);
3019                 } else if (ixgbe->num_rx_groups <= 64) {
3020                         ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3021                             min(2, ring_per_group);
3022                 }
3023                 break;
3024 
3025         default:
3026                 break;
3027         }
3028 
3029         ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3030 
3031         if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3032                 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3033         } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3034                 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3035         } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3036                 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3037         } else {
3038                 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3039         }
3040 
3041         IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3042             ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3043 }
3044 
3045 /*
3046  * ixgbe_get_conf - Get driver configurations set in driver.conf.
3047  *
3048  * This routine gets user-configured values out of the configuration
3049  * file ixgbe.conf.
3050  *
3051  * For each configurable value, there is a minimum, a maximum, and a
3052  * default.
3053  * If user does not configure a value, use the default.
3054  * If user configures below the minimum, use the minumum.
3055  * If user configures above the maximum, use the maxumum.
3056  */
3057 static void
3058 ixgbe_get_conf(ixgbe_t *ixgbe)
3059 {
3060         struct ixgbe_hw *hw = &ixgbe->hw;
3061         uint32_t flow_control;
3062 
3063         /*
3064          * ixgbe driver supports the following user configurations:
3065          *
3066          * Jumbo frame configuration:
3067          *    default_mtu
3068          *
3069          * Ethernet flow control configuration:
3070          *    flow_control
3071          *
3072          * Multiple rings configurations:
3073          *    tx_queue_number
3074          *    tx_ring_size
3075          *    rx_queue_number
3076          *    rx_ring_size
3077          *
3078          * Call ixgbe_get_prop() to get the value for a specific
3079          * configuration parameter.
3080          */
3081 
3082         /*
3083          * Jumbo frame configuration - max_frame_size controls host buffer
3084          * allocation, so includes MTU, ethernet header, vlan tag and
3085          * frame check sequence.
3086          */
3087         ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3088             MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3089 
3090         ixgbe->max_frame_size = ixgbe->default_mtu +
3091             sizeof (struct ether_vlan_header) + ETHERFCSL;
3092 
3093         /*
3094          * Ethernet flow control configuration
3095          */
3096         flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3097             ixgbe_fc_none, 3, ixgbe_fc_none);
3098         if (flow_control == 3)
3099                 flow_control = ixgbe_fc_default;
3100 
3101         /*
3102          * fc.requested mode is what the user requests.  After autoneg,
3103          * fc.current_mode will be the flow_control mode that was negotiated.
3104          */
3105         hw->fc.requested_mode = flow_control;
3106 
3107         /*
3108          * Multiple rings configurations
3109          */
3110         ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3111             ixgbe->capab->min_tx_que_num,
3112             ixgbe->capab->max_tx_que_num,
3113             ixgbe->capab->def_tx_que_num);
3114         ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3115             MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3116 
3117         ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3118             ixgbe->capab->min_rx_que_num,
3119             ixgbe->capab->max_rx_que_num,
3120             ixgbe->capab->def_rx_que_num);
3121         ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3122             MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3123 
3124         /*
3125          * Multiple groups configuration
3126          */
3127         ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3128             ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3129             ixgbe->capab->def_rx_grp_num);
3130 
3131         ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3132             0, 1, DEFAULT_MR_ENABLE);
3133 
3134         if (ixgbe->mr_enable == B_FALSE) {
3135                 ixgbe->num_tx_rings = 1;
3136                 ixgbe->num_rx_rings = 1;
3137                 ixgbe->num_rx_groups = 1;
3138                 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3139         } else {
3140                 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3141                     max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3142                 /*
3143                  * The combination of num_rx_rings and num_rx_groups
3144                  * may be not supported by h/w. We need to adjust
3145                  * them to appropriate values.
3146                  */
3147                 ixgbe_setup_vmdq_rss_conf(ixgbe);
3148         }
3149 
3150         /*
3151          * Tunable used to force an interrupt type. The only use is
3152          * for testing of the lesser interrupt types.
3153          * 0 = don't force interrupt type
3154          * 1 = force interrupt type MSI-X
3155          * 2 = force interrupt type MSI
3156          * 3 = force interrupt type Legacy
3157          */
3158         ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3159             IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3160 
3161         ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3162             0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3163         ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3164             0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3165         ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3166             0, 1, DEFAULT_LSO_ENABLE);
3167         ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3168             0, 1, DEFAULT_LRO_ENABLE);
3169         ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3170             0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3171         ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3172             PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3173 
3174         /* Head Write Back not recommended for 82599 and X540 */
3175         if (hw->mac.type == ixgbe_mac_82599EB ||
3176             hw->mac.type == ixgbe_mac_X540) {
3177                 ixgbe->tx_head_wb_enable = B_FALSE;
3178         }
3179 
3180         /*
3181          * ixgbe LSO needs the tx h/w checksum support.
3182          * LSO will be disabled if tx h/w checksum is not
3183          * enabled.
3184          */
3185         if (ixgbe->tx_hcksum_enable == B_FALSE) {
3186                 ixgbe->lso_enable = B_FALSE;
3187         }
3188 
3189         /*
3190          * ixgbe LRO needs the rx h/w checksum support.
3191          * LRO will be disabled if rx h/w checksum is not
3192          * enabled.
3193          */
3194         if (ixgbe->rx_hcksum_enable == B_FALSE) {
3195                 ixgbe->lro_enable = B_FALSE;
3196         }
3197 
3198         /*
3199          * ixgbe LRO only been supported by 82599 and X540 now
3200          */
3201         if (hw->mac.type == ixgbe_mac_82598EB) {
3202                 ixgbe->lro_enable = B_FALSE;
3203         }
3204         ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3205             MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3206             DEFAULT_TX_COPY_THRESHOLD);
3207         ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3208             PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3209             MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3210         ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3211             PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3212             MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3213         ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3214             PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3215             MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3216 
3217         ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3218             MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3219             DEFAULT_RX_COPY_THRESHOLD);
3220         ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3221             MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3222             DEFAULT_RX_LIMIT_PER_INTR);
3223 
3224         ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3225             ixgbe->capab->min_intr_throttle,
3226             ixgbe->capab->max_intr_throttle,
3227             ixgbe->capab->def_intr_throttle);
3228         /*
3229          * 82599 and X540 require the interrupt throttling rate is
3230          * a multiple of 8. This is enforced by the register
3231          * definiton.
3232          */
3233         if (hw->mac.type == ixgbe_mac_82599EB || hw->mac.type == ixgbe_mac_X540)
3234                 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3235 
3236         hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3237             PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3238 }
3239 
3240 static void
3241 ixgbe_init_params(ixgbe_t *ixgbe)
3242 {
3243         ixgbe->param_en_10000fdx_cap = 1;
3244         ixgbe->param_en_1000fdx_cap = 1;
3245         ixgbe->param_en_100fdx_cap = 1;
3246         ixgbe->param_adv_10000fdx_cap = 1;
3247         ixgbe->param_adv_1000fdx_cap = 1;
3248         ixgbe->param_adv_100fdx_cap = 1;
3249 
3250         ixgbe->param_pause_cap = 1;
3251         ixgbe->param_asym_pause_cap = 1;
3252         ixgbe->param_rem_fault = 0;
3253 
3254         ixgbe->param_adv_autoneg_cap = 1;
3255         ixgbe->param_adv_pause_cap = 1;
3256         ixgbe->param_adv_asym_pause_cap = 1;
3257         ixgbe->param_adv_rem_fault = 0;
3258 
3259         ixgbe->param_lp_10000fdx_cap = 0;
3260         ixgbe->param_lp_1000fdx_cap = 0;
3261         ixgbe->param_lp_100fdx_cap = 0;
3262         ixgbe->param_lp_autoneg_cap = 0;
3263         ixgbe->param_lp_pause_cap = 0;
3264         ixgbe->param_lp_asym_pause_cap = 0;
3265         ixgbe->param_lp_rem_fault = 0;
3266 }
3267 
3268 /*
3269  * ixgbe_get_prop - Get a property value out of the configuration file
3270  * ixgbe.conf.
3271  *
3272  * Caller provides the name of the property, a default value, a minimum
3273  * value, and a maximum value.
3274  *
3275  * Return configured value of the property, with default, minimum and
3276  * maximum properly applied.
3277  */
3278 static int
3279 ixgbe_get_prop(ixgbe_t *ixgbe,
3280     char *propname,     /* name of the property */
3281     int minval,         /* minimum acceptable value */
3282     int maxval,         /* maximim acceptable value */
3283     int defval)         /* default value */
3284 {
3285         int value;
3286 
3287         /*
3288          * Call ddi_prop_get_int() to read the conf settings
3289          */
3290         value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3291             DDI_PROP_DONTPASS, propname, defval);
3292         if (value > maxval)
3293                 value = maxval;
3294 
3295         if (value < minval)
3296                 value = minval;
3297 
3298         return (value);
3299 }
3300 
3301 /*
3302  * ixgbe_driver_setup_link - Using the link properties to setup the link.
3303  */
3304 int
3305 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3306 {
3307         u32 autoneg_advertised = 0;
3308 
3309         /*
3310          * No half duplex support with 10Gb parts
3311          */
3312         if (ixgbe->param_adv_10000fdx_cap == 1)
3313                 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3314 
3315         if (ixgbe->param_adv_1000fdx_cap == 1)
3316                 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3317 
3318         if (ixgbe->param_adv_100fdx_cap == 1)
3319                 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
3320 
3321         if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) {
3322                 ixgbe_notice(ixgbe, "Invalid link settings. Setup link "
3323                     "to autonegotiation with full link capabilities.");
3324 
3325                 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL |
3326                     IXGBE_LINK_SPEED_1GB_FULL |
3327                     IXGBE_LINK_SPEED_100_FULL;
3328         }
3329 
3330         if (setup_hw) {
3331                 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised,
3332                     ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) {
3333                         ixgbe_notice(ixgbe, "Setup link failed on this "
3334                             "device.");
3335                         return (IXGBE_FAILURE);
3336                 }
3337         }
3338 
3339         return (IXGBE_SUCCESS);
3340 }
3341 
3342 /*
3343  * ixgbe_driver_link_check - Link status processing.
3344  *
3345  * This function can be called in both kernel context and interrupt context
3346  */
3347 static void
3348 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3349 {
3350         struct ixgbe_hw *hw = &ixgbe->hw;
3351         ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3352         boolean_t link_up = B_FALSE;
3353         boolean_t link_changed = B_FALSE;
3354 
3355         ASSERT(mutex_owned(&ixgbe->gen_lock));
3356 
3357         (void) ixgbe_check_link(hw, &speed, &link_up, false);
3358         if (link_up) {
3359                 ixgbe->link_check_complete = B_TRUE;
3360 
3361                 /* Link is up, enable flow control settings */
3362                 (void) ixgbe_fc_enable(hw);
3363 
3364                 /*
3365                  * The Link is up, check whether it was marked as down earlier
3366                  */
3367                 if (ixgbe->link_state != LINK_STATE_UP) {
3368                         switch (speed) {
3369                         case IXGBE_LINK_SPEED_10GB_FULL:
3370                                 ixgbe->link_speed = SPEED_10GB;
3371                                 break;
3372                         case IXGBE_LINK_SPEED_1GB_FULL:
3373                                 ixgbe->link_speed = SPEED_1GB;
3374                                 break;
3375                         case IXGBE_LINK_SPEED_100_FULL:
3376                                 ixgbe->link_speed = SPEED_100;
3377                         }
3378                         ixgbe->link_duplex = LINK_DUPLEX_FULL;
3379                         ixgbe->link_state = LINK_STATE_UP;
3380                         link_changed = B_TRUE;
3381                 }
3382         } else {
3383                 if (ixgbe->link_check_complete == B_TRUE ||
3384                     (ixgbe->link_check_complete == B_FALSE &&
3385                     gethrtime() >= ixgbe->link_check_hrtime)) {
3386                         /*
3387                          * The link is really down
3388                          */
3389                         ixgbe->link_check_complete = B_TRUE;
3390 
3391                         if (ixgbe->link_state != LINK_STATE_DOWN) {
3392                                 ixgbe->link_speed = 0;
3393                                 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3394                                 ixgbe->link_state = LINK_STATE_DOWN;
3395                                 link_changed = B_TRUE;
3396                         }
3397                 }
3398         }
3399 
3400         /*
3401          * If we are in an interrupt context, need to re-enable the
3402          * interrupt, which was automasked
3403          */
3404         if (servicing_interrupt() != 0) {
3405                 ixgbe->eims |= IXGBE_EICR_LSC;
3406                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3407         }
3408 
3409         if (link_changed) {
3410                 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3411         }
3412 }
3413 
3414 /*
3415  * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3416  */
3417 static void
3418 ixgbe_sfp_check(void *arg)
3419 {
3420         ixgbe_t *ixgbe = (ixgbe_t *)arg;
3421         uint32_t eicr = ixgbe->eicr;
3422         struct ixgbe_hw *hw = &ixgbe->hw;
3423 
3424         mutex_enter(&ixgbe->gen_lock);
3425         if (eicr & IXGBE_EICR_GPI_SDP1) {
3426                 /* clear the interrupt */
3427                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
3428 
3429                 /* if link up, do multispeed fiber setup */
3430                 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3431                     B_TRUE, B_TRUE);
3432                 ixgbe_driver_link_check(ixgbe);
3433                 ixgbe_get_hw_state(ixgbe);
3434         } else if (eicr & IXGBE_EICR_GPI_SDP2) {
3435                 /* clear the interrupt */
3436                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
3437 
3438                 /* if link up, do sfp module setup */
3439                 (void) hw->mac.ops.setup_sfp(hw);
3440 
3441                 /* do multispeed fiber setup */
3442                 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3443                     B_TRUE, B_TRUE);
3444                 ixgbe_driver_link_check(ixgbe);
3445                 ixgbe_get_hw_state(ixgbe);
3446         }
3447         mutex_exit(&ixgbe->gen_lock);
3448 
3449         /*
3450          * We need to fully re-check the link later.
3451          */
3452         ixgbe->link_check_complete = B_FALSE;
3453         ixgbe->link_check_hrtime = gethrtime() +
3454             (IXGBE_LINK_UP_TIME * 100000000ULL);
3455 }
3456 
3457 /*
3458  * ixgbe_overtemp_check - overtemp module processing done in taskq
3459  *
3460  * This routine will only be called on adapters with temperature sensor.
3461  * The indication of over-temperature can be either SDP0 interrupt or the link
3462  * status change interrupt.
3463  */
3464 static void
3465 ixgbe_overtemp_check(void *arg)
3466 {
3467         ixgbe_t *ixgbe = (ixgbe_t *)arg;
3468         struct ixgbe_hw *hw = &ixgbe->hw;
3469         uint32_t eicr = ixgbe->eicr;
3470         ixgbe_link_speed speed;
3471         boolean_t link_up;
3472 
3473         mutex_enter(&ixgbe->gen_lock);
3474 
3475         /* make sure we know current state of link */
3476         (void) ixgbe_check_link(hw, &speed, &link_up, false);
3477 
3478         /* check over-temp condition */
3479         if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) ||
3480             (eicr & IXGBE_EICR_LSC)) {
3481                 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3482                         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3483 
3484                         /*
3485                          * Disable the adapter interrupts
3486                          */
3487                         ixgbe_disable_adapter_interrupts(ixgbe);
3488 
3489                         /*
3490                          * Disable Rx/Tx units
3491                          */
3492                         (void) ixgbe_stop_adapter(hw);
3493 
3494                         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3495                         ixgbe_error(ixgbe,
3496                             "Problem: Network adapter has been stopped "
3497                             "because it has overheated");
3498                         ixgbe_error(ixgbe,
3499                             "Action: Restart the computer. "
3500                             "If the problem persists, power off the system "
3501                             "and replace the adapter");
3502                 }
3503         }
3504 
3505         /* write to clear the interrupt */
3506         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3507 
3508         mutex_exit(&ixgbe->gen_lock);
3509 }
3510 
3511 /*
3512  * ixgbe_link_timer - timer for link status detection
3513  */
3514 static void
3515 ixgbe_link_timer(void *arg)
3516 {
3517         ixgbe_t *ixgbe = (ixgbe_t *)arg;
3518 
3519         mutex_enter(&ixgbe->gen_lock);
3520         ixgbe_driver_link_check(ixgbe);
3521         mutex_exit(&ixgbe->gen_lock);
3522 }
3523 
3524 /*
3525  * ixgbe_local_timer - Driver watchdog function.
3526  *
3527  * This function will handle the transmit stall check and other routines.
3528  */
3529 static void
3530 ixgbe_local_timer(void *arg)
3531 {
3532         ixgbe_t *ixgbe = (ixgbe_t *)arg;
3533 
3534         if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3535                 goto out;
3536 
3537         if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3538                 ixgbe->reset_count++;
3539                 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3540                         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3541                 goto out;
3542         }
3543 
3544         if (ixgbe_stall_check(ixgbe)) {
3545                 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3546                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3547 
3548                 ixgbe->reset_count++;
3549                 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3550                         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3551         }
3552 
3553 out:
3554         ixgbe_restart_watchdog_timer(ixgbe);
3555 }
3556 
3557 /*
3558  * ixgbe_stall_check - Check for transmit stall.
3559  *
3560  * This function checks if the adapter is stalled (in transmit).
3561  *
3562  * It is called each time the watchdog timeout is invoked.
3563  * If the transmit descriptor reclaim continuously fails,
3564  * the watchdog value will increment by 1. If the watchdog
3565  * value exceeds the threshold, the ixgbe is assumed to
3566  * have stalled and need to be reset.
3567  */
3568 static boolean_t
3569 ixgbe_stall_check(ixgbe_t *ixgbe)
3570 {
3571         ixgbe_tx_ring_t *tx_ring;
3572         boolean_t result;
3573         int i;
3574 
3575         if (ixgbe->link_state != LINK_STATE_UP)
3576                 return (B_FALSE);
3577 
3578         /*
3579          * If any tx ring is stalled, we'll reset the chipset
3580          */
3581         result = B_FALSE;
3582         for (i = 0; i < ixgbe->num_tx_rings; i++) {
3583                 tx_ring = &ixgbe->tx_rings[i];
3584                 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3585                         tx_ring->tx_recycle(tx_ring);
3586                 }
3587 
3588                 if (tx_ring->recycle_fail > 0)
3589                         tx_ring->stall_watchdog++;
3590                 else
3591                         tx_ring->stall_watchdog = 0;
3592 
3593                 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3594                         result = B_TRUE;
3595                         break;
3596                 }
3597         }
3598 
3599         if (result) {
3600                 tx_ring->stall_watchdog = 0;
3601                 tx_ring->recycle_fail = 0;
3602         }
3603 
3604         return (result);
3605 }
3606 
3607 
3608 /*
3609  * is_valid_mac_addr - Check if the mac address is valid.
3610  */
3611 static boolean_t
3612 is_valid_mac_addr(uint8_t *mac_addr)
3613 {
3614         const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3615         const uint8_t addr_test2[6] =
3616             { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3617 
3618         if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3619             !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3620                 return (B_FALSE);
3621 
3622         return (B_TRUE);
3623 }
3624 
3625 static boolean_t
3626 ixgbe_find_mac_address(ixgbe_t *ixgbe)
3627 {
3628 #ifdef __sparc
3629         struct ixgbe_hw *hw = &ixgbe->hw;
3630         uchar_t *bytes;
3631         struct ether_addr sysaddr;
3632         uint_t nelts;
3633         int err;
3634         boolean_t found = B_FALSE;
3635 
3636         /*
3637          * The "vendor's factory-set address" may already have
3638          * been extracted from the chip, but if the property
3639          * "local-mac-address" is set we use that instead.
3640          *
3641          * We check whether it looks like an array of 6
3642          * bytes (which it should, if OBP set it).  If we can't
3643          * make sense of it this way, we'll ignore it.
3644          */
3645         err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3646             DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3647         if (err == DDI_PROP_SUCCESS) {
3648                 if (nelts == ETHERADDRL) {
3649                         while (nelts--)
3650                                 hw->mac.addr[nelts] = bytes[nelts];
3651                         found = B_TRUE;
3652                 }
3653                 ddi_prop_free(bytes);
3654         }
3655 
3656         /*
3657          * Look up the OBP property "local-mac-address?". If the user has set
3658          * 'local-mac-address? = false', use "the system address" instead.
3659          */
3660         if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
3661             "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3662                 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3663                         if (localetheraddr(NULL, &sysaddr) != 0) {
3664                                 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3665                                 found = B_TRUE;
3666                         }
3667                 }
3668                 ddi_prop_free(bytes);
3669         }
3670 
3671         /*
3672          * Finally(!), if there's a valid "mac-address" property (created
3673          * if we netbooted from this interface), we must use this instead
3674          * of any of the above to ensure that the NFS/install server doesn't
3675          * get confused by the address changing as Solaris takes over!
3676          */
3677         err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3678             DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3679         if (err == DDI_PROP_SUCCESS) {
3680                 if (nelts == ETHERADDRL) {
3681                         while (nelts--)
3682                                 hw->mac.addr[nelts] = bytes[nelts];
3683                         found = B_TRUE;
3684                 }
3685                 ddi_prop_free(bytes);
3686         }
3687 
3688         if (found) {
3689                 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3690                 return (B_TRUE);
3691         }
3692 #else
3693         _NOTE(ARGUNUSED(ixgbe));
3694 #endif
3695 
3696         return (B_TRUE);
3697 }
3698 
3699 #pragma inline(ixgbe_arm_watchdog_timer)
3700 static void
3701 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
3702 {
3703         /*
3704          * Fire a watchdog timer
3705          */
3706         ixgbe->watchdog_tid =
3707             timeout(ixgbe_local_timer,
3708             (void *)ixgbe, drv_sectohz(1));
3709 
3710 }
3711 
3712 /*
3713  * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
3714  */
3715 void
3716 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
3717 {
3718         mutex_enter(&ixgbe->watchdog_lock);
3719 
3720         if (!ixgbe->watchdog_enable) {
3721                 ixgbe->watchdog_enable = B_TRUE;
3722                 ixgbe->watchdog_start = B_TRUE;
3723                 ixgbe_arm_watchdog_timer(ixgbe);
3724         }
3725 
3726         mutex_exit(&ixgbe->watchdog_lock);
3727 }
3728 
3729 /*
3730  * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
3731  */
3732 void
3733 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
3734 {
3735         timeout_id_t tid;
3736 
3737         mutex_enter(&ixgbe->watchdog_lock);
3738 
3739         ixgbe->watchdog_enable = B_FALSE;
3740         ixgbe->watchdog_start = B_FALSE;
3741         tid = ixgbe->watchdog_tid;
3742         ixgbe->watchdog_tid = 0;
3743 
3744         mutex_exit(&ixgbe->watchdog_lock);
3745 
3746         if (tid != 0)
3747                 (void) untimeout(tid);
3748 }
3749 
3750 /*
3751  * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
3752  */
3753 void
3754 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
3755 {
3756         mutex_enter(&ixgbe->watchdog_lock);
3757 
3758         if (ixgbe->watchdog_enable) {
3759                 if (!ixgbe->watchdog_start) {
3760                         ixgbe->watchdog_start = B_TRUE;
3761                         ixgbe_arm_watchdog_timer(ixgbe);
3762                 }
3763         }
3764 
3765         mutex_exit(&ixgbe->watchdog_lock);
3766 }
3767 
3768 /*
3769  * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
3770  */
3771 static void
3772 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
3773 {
3774         mutex_enter(&ixgbe->watchdog_lock);
3775 
3776         if (ixgbe->watchdog_start)
3777                 ixgbe_arm_watchdog_timer(ixgbe);
3778 
3779         mutex_exit(&ixgbe->watchdog_lock);
3780 }
3781 
3782 /*
3783  * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
3784  */
3785 void
3786 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
3787 {
3788         timeout_id_t tid;
3789 
3790         mutex_enter(&ixgbe->watchdog_lock);
3791 
3792         ixgbe->watchdog_start = B_FALSE;
3793         tid = ixgbe->watchdog_tid;
3794         ixgbe->watchdog_tid = 0;
3795 
3796         mutex_exit(&ixgbe->watchdog_lock);
3797 
3798         if (tid != 0)
3799                 (void) untimeout(tid);
3800 }
3801 
3802 /*
3803  * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
3804  */
3805 static void
3806 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
3807 {
3808         struct ixgbe_hw *hw = &ixgbe->hw;
3809 
3810         /*
3811          * mask all interrupts off
3812          */
3813         IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
3814 
3815         /*
3816          * for MSI-X, also disable autoclear
3817          */
3818         if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3819                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
3820         }
3821 
3822         IXGBE_WRITE_FLUSH(hw);
3823 }
3824 
3825 /*
3826  * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
3827  */
3828 static void
3829 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
3830 {
3831         struct ixgbe_hw *hw = &ixgbe->hw;
3832         uint32_t eiac, eiam;
3833         uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
3834 
3835         /* interrupt types to enable */
3836         ixgbe->eims = IXGBE_EIMS_ENABLE_MASK;        /* shared code default */
3837         ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER;    /* minus tcp timer */
3838         ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
3839 
3840         /* enable automask on "other" causes that this adapter can generate */
3841         eiam = ixgbe->capab->other_intr;
3842 
3843         /*
3844          * msi-x mode
3845          */
3846         if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3847                 /* enable autoclear but not on bits 29:20 */
3848                 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
3849 
3850                 /* general purpose interrupt enable */
3851                 gpie |= (IXGBE_GPIE_MSIX_MODE
3852                     | IXGBE_GPIE_PBA_SUPPORT
3853                     | IXGBE_GPIE_OCD
3854                     | IXGBE_GPIE_EIAME);
3855         /*
3856          * non-msi-x mode
3857          */
3858         } else {
3859 
3860                 /* disable autoclear, leave gpie at default */
3861                 eiac = 0;
3862 
3863                 /*
3864                  * General purpose interrupt enable.
3865                  * For 82599 or X540, extended interrupt automask enable
3866                  * only in MSI or MSI-X mode
3867                  */
3868                 if ((hw->mac.type == ixgbe_mac_82598EB) ||
3869                     (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
3870                         gpie |= IXGBE_GPIE_EIAME;
3871                 }
3872         }
3873 
3874         /* Enable specific "other" interrupt types */
3875         switch (hw->mac.type) {
3876         case ixgbe_mac_82598EB:
3877                 gpie |= ixgbe->capab->other_gpie;
3878                 break;
3879 
3880         case ixgbe_mac_82599EB:
3881         case ixgbe_mac_X540:
3882                 gpie |= ixgbe->capab->other_gpie;
3883 
3884                 /* Enable RSC Delay 8us when LRO enabled  */
3885                 if (ixgbe->lro_enable) {
3886                         gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
3887                 }
3888                 break;
3889 
3890         default:
3891                 break;
3892         }
3893 
3894         /* write to interrupt control registers */
3895         IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3896         IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
3897         IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
3898         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3899         IXGBE_WRITE_FLUSH(hw);
3900 }
3901 
3902 /*
3903  * ixgbe_loopback_ioctl - Loopback support.
3904  */
3905 enum ioc_reply
3906 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
3907 {
3908         lb_info_sz_t *lbsp;
3909         lb_property_t *lbpp;
3910         uint32_t *lbmp;
3911         uint32_t size;
3912         uint32_t value;
3913 
3914         if (mp->b_cont == NULL)
3915                 return (IOC_INVAL);
3916 
3917         switch (iocp->ioc_cmd) {
3918         default:
3919                 return (IOC_INVAL);
3920 
3921         case LB_GET_INFO_SIZE:
3922                 size = sizeof (lb_info_sz_t);
3923                 if (iocp->ioc_count != size)
3924                         return (IOC_INVAL);
3925 
3926                 value = sizeof (lb_normal);
3927                 value += sizeof (lb_mac);
3928                 value += sizeof (lb_external);
3929 
3930                 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3931                 *lbsp = value;
3932                 break;
3933 
3934         case LB_GET_INFO:
3935                 value = sizeof (lb_normal);
3936                 value += sizeof (lb_mac);
3937                 value += sizeof (lb_external);
3938 
3939                 size = value;
3940                 if (iocp->ioc_count != size)
3941                         return (IOC_INVAL);
3942 
3943                 value = 0;
3944                 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3945 
3946                 lbpp[value++] = lb_normal;
3947                 lbpp[value++] = lb_mac;
3948                 lbpp[value++] = lb_external;
3949                 break;
3950 
3951         case LB_GET_MODE:
3952                 size = sizeof (uint32_t);
3953                 if (iocp->ioc_count != size)
3954                         return (IOC_INVAL);
3955 
3956                 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3957                 *lbmp = ixgbe->loopback_mode;
3958                 break;
3959 
3960         case LB_SET_MODE:
3961                 size = 0;
3962                 if (iocp->ioc_count != sizeof (uint32_t))
3963                         return (IOC_INVAL);
3964 
3965                 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3966                 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
3967                         return (IOC_INVAL);
3968                 break;
3969         }
3970 
3971         iocp->ioc_count = size;
3972         iocp->ioc_error = 0;
3973 
3974         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3975                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3976                 return (IOC_INVAL);
3977         }
3978 
3979         return (IOC_REPLY);
3980 }
3981 
3982 /*
3983  * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
3984  */
3985 static boolean_t
3986 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
3987 {
3988         if (mode == ixgbe->loopback_mode)
3989                 return (B_TRUE);
3990 
3991         ixgbe->loopback_mode = mode;
3992 
3993         if (mode == IXGBE_LB_NONE) {
3994                 /*
3995                  * Reset the chip
3996                  */
3997                 (void) ixgbe_reset(ixgbe);
3998                 return (B_TRUE);
3999         }
4000 
4001         mutex_enter(&ixgbe->gen_lock);
4002 
4003         switch (mode) {
4004         default:
4005                 mutex_exit(&ixgbe->gen_lock);
4006                 return (B_FALSE);
4007 
4008         case IXGBE_LB_EXTERNAL:
4009                 break;
4010 
4011         case IXGBE_LB_INTERNAL_MAC:
4012                 ixgbe_set_internal_mac_loopback(ixgbe);
4013                 break;
4014         }
4015 
4016         mutex_exit(&ixgbe->gen_lock);
4017 
4018         return (B_TRUE);
4019 }
4020 
4021 /*
4022  * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4023  */
4024 static void
4025 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4026 {
4027         struct ixgbe_hw *hw;
4028         uint32_t reg;
4029         uint8_t atlas;
4030 
4031         hw = &ixgbe->hw;
4032 
4033         /*
4034          * Setup MAC loopback
4035          */
4036         reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4037         reg |= IXGBE_HLREG0_LPBK;
4038         IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4039 
4040         reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4041         reg &= ~IXGBE_AUTOC_LMS_MASK;
4042         IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4043 
4044         /*
4045          * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4046          */
4047         switch (hw->mac.type) {
4048         case ixgbe_mac_82598EB:
4049                 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4050                     &atlas);
4051                 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4052                 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4053                     atlas);
4054 
4055                 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4056                     &atlas);
4057                 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4058                 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4059                     atlas);
4060 
4061                 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4062                     &atlas);
4063                 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4064                 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4065                     atlas);
4066 
4067                 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4068                     &atlas);
4069                 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4070                 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4071                     atlas);
4072                 break;
4073 
4074         case ixgbe_mac_82599EB:
4075         case ixgbe_mac_X540:
4076                 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4077                 reg |= (IXGBE_AUTOC_FLU |
4078                     IXGBE_AUTOC_10G_KX4);
4079                 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4080 
4081                 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4082                     B_FALSE, B_TRUE);
4083                 break;
4084 
4085         default:
4086                 break;
4087         }
4088 }
4089 
4090 #pragma inline(ixgbe_intr_rx_work)
4091 /*
4092  * ixgbe_intr_rx_work - RX processing of ISR.
4093  */
4094 static void
4095 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4096 {
4097         mblk_t *mp;
4098 
4099         mutex_enter(&rx_ring->rx_lock);
4100 
4101         mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4102         mutex_exit(&rx_ring->rx_lock);
4103 
4104         if (mp != NULL)
4105                 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4106                     rx_ring->ring_gen_num);
4107 }
4108 
4109 #pragma inline(ixgbe_intr_tx_work)
4110 /*
4111  * ixgbe_intr_tx_work - TX processing of ISR.
4112  */
4113 static void
4114 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4115 {
4116         ixgbe_t *ixgbe = tx_ring->ixgbe;
4117 
4118         /*
4119          * Recycle the tx descriptors
4120          */
4121         tx_ring->tx_recycle(tx_ring);
4122 
4123         /*
4124          * Schedule the re-transmit
4125          */
4126         if (tx_ring->reschedule &&
4127             (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4128                 tx_ring->reschedule = B_FALSE;
4129                 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4130                     tx_ring->ring_handle);
4131                 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4132         }
4133 }
4134 
4135 #pragma inline(ixgbe_intr_other_work)
4136 /*
4137  * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4138  */
4139 static void
4140 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4141 {
4142         ASSERT(mutex_owned(&ixgbe->gen_lock));
4143 
4144         /*
4145          * handle link status change
4146          */
4147         if (eicr & IXGBE_EICR_LSC) {
4148                 ixgbe_driver_link_check(ixgbe);
4149                 ixgbe_get_hw_state(ixgbe);
4150         }
4151 
4152         /*
4153          * check for fan failure on adapters with fans
4154          */
4155         if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4156             (eicr & IXGBE_EICR_GPI_SDP1)) {
4157                 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4158 
4159                 /*
4160                  * Disable the adapter interrupts
4161                  */
4162                 ixgbe_disable_adapter_interrupts(ixgbe);
4163 
4164                 /*
4165                  * Disable Rx/Tx units
4166                  */
4167                 (void) ixgbe_stop_adapter(&ixgbe->hw);
4168 
4169                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4170                 ixgbe_error(ixgbe,
4171                     "Problem: Network adapter has been stopped "
4172                     "because the fan has stopped.\n");
4173                 ixgbe_error(ixgbe,
4174                     "Action: Replace the adapter.\n");
4175 
4176                 /* re-enable the interrupt, which was automasked */
4177                 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4178         }
4179 
4180         /*
4181          * Do SFP check for adapters with hot-plug capability
4182          */
4183         if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4184             ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) {
4185                 ixgbe->eicr = eicr;
4186                 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4187                     ixgbe_sfp_check, (void *)ixgbe,
4188                     DDI_NOSLEEP)) != DDI_SUCCESS) {
4189                         ixgbe_log(ixgbe, "No memory available to dispatch "
4190                             "taskq for SFP check");
4191                 }
4192         }
4193 
4194         /*
4195          * Do over-temperature check for adapters with temp sensor
4196          */
4197         if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4198             ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
4199                 ixgbe->eicr = eicr;
4200                 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4201                     ixgbe_overtemp_check, (void *)ixgbe,
4202                     DDI_NOSLEEP)) != DDI_SUCCESS) {
4203                         ixgbe_log(ixgbe, "No memory available to dispatch "
4204                             "taskq for overtemp check");
4205                 }
4206         }
4207 }
4208 
4209 /*
4210  * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4211  */
4212 static uint_t
4213 ixgbe_intr_legacy(void *arg1, void *arg2)
4214 {
4215         ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4216         struct ixgbe_hw *hw = &ixgbe->hw;
4217         ixgbe_tx_ring_t *tx_ring;
4218         ixgbe_rx_ring_t *rx_ring;
4219         uint32_t eicr;
4220         mblk_t *mp;
4221         boolean_t tx_reschedule;
4222         uint_t result;
4223 
4224         _NOTE(ARGUNUSED(arg2));
4225 
4226         mutex_enter(&ixgbe->gen_lock);
4227         if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4228                 mutex_exit(&ixgbe->gen_lock);
4229                 return (DDI_INTR_UNCLAIMED);
4230         }
4231 
4232         mp = NULL;
4233         tx_reschedule = B_FALSE;
4234 
4235         /*
4236          * Any bit set in eicr: claim this interrupt
4237          */
4238         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4239 
4240         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4241                 mutex_exit(&ixgbe->gen_lock);
4242                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4243                 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4244                 return (DDI_INTR_CLAIMED);
4245         }
4246 
4247         if (eicr) {
4248                 /*
4249                  * For legacy interrupt, we have only one interrupt,
4250                  * so we have only one rx ring and one tx ring enabled.
4251                  */
4252                 ASSERT(ixgbe->num_rx_rings == 1);
4253                 ASSERT(ixgbe->num_tx_rings == 1);
4254 
4255                 /*
4256                  * For legacy interrupt, rx rings[0] will use RTxQ[0].
4257                  */
4258                 if (eicr & 0x1) {
4259                         ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4260                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4261                         ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4262                         /*
4263                          * Clean the rx descriptors
4264                          */
4265                         rx_ring = &ixgbe->rx_rings[0];
4266                         mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4267                 }
4268 
4269                 /*
4270                  * For legacy interrupt, tx rings[0] will use RTxQ[1].
4271                  */
4272                 if (eicr & 0x2) {
4273                         /*
4274                          * Recycle the tx descriptors
4275                          */
4276                         tx_ring = &ixgbe->tx_rings[0];
4277                         tx_ring->tx_recycle(tx_ring);
4278 
4279                         /*
4280                          * Schedule the re-transmit
4281                          */
4282                         tx_reschedule = (tx_ring->reschedule &&
4283                             (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4284                 }
4285 
4286                 /* any interrupt type other than tx/rx */
4287                 if (eicr & ixgbe->capab->other_intr) {
4288                         switch (hw->mac.type) {
4289                         case ixgbe_mac_82598EB:
4290                                 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4291                                 break;
4292 
4293                         case ixgbe_mac_82599EB:
4294                         case ixgbe_mac_X540:
4295                                 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4296                                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4297                                 break;
4298 
4299                         default:
4300                                 break;
4301                         }
4302                         ixgbe_intr_other_work(ixgbe, eicr);
4303                         ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4304                 }
4305 
4306                 mutex_exit(&ixgbe->gen_lock);
4307 
4308                 result = DDI_INTR_CLAIMED;
4309         } else {
4310                 mutex_exit(&ixgbe->gen_lock);
4311 
4312                 /*
4313                  * No interrupt cause bits set: don't claim this interrupt.
4314                  */
4315                 result = DDI_INTR_UNCLAIMED;
4316         }
4317 
4318         /* re-enable the interrupts which were automasked */
4319         IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4320 
4321         /*
4322          * Do the following work outside of the gen_lock
4323          */
4324         if (mp != NULL) {
4325                 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4326                     rx_ring->ring_gen_num);
4327         }
4328 
4329         if (tx_reschedule)  {
4330                 tx_ring->reschedule = B_FALSE;
4331                 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4332                 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4333         }
4334 
4335         return (result);
4336 }
4337 
4338 /*
4339  * ixgbe_intr_msi - Interrupt handler for MSI.
4340  */
4341 static uint_t
4342 ixgbe_intr_msi(void *arg1, void *arg2)
4343 {
4344         ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4345         struct ixgbe_hw *hw = &ixgbe->hw;
4346         uint32_t eicr;
4347 
4348         _NOTE(ARGUNUSED(arg2));
4349 
4350         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4351 
4352         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4353                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4354                 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4355                 return (DDI_INTR_CLAIMED);
4356         }
4357 
4358         /*
4359          * For MSI interrupt, we have only one vector,
4360          * so we have only one rx ring and one tx ring enabled.
4361          */
4362         ASSERT(ixgbe->num_rx_rings == 1);
4363         ASSERT(ixgbe->num_tx_rings == 1);
4364 
4365         /*
4366          * For MSI interrupt, rx rings[0] will use RTxQ[0].
4367          */
4368         if (eicr & 0x1) {
4369                 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4370         }
4371 
4372         /*
4373          * For MSI interrupt, tx rings[0] will use RTxQ[1].
4374          */
4375         if (eicr & 0x2) {
4376                 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4377         }
4378 
4379         /* any interrupt type other than tx/rx */
4380         if (eicr & ixgbe->capab->other_intr) {
4381                 mutex_enter(&ixgbe->gen_lock);
4382                 switch (hw->mac.type) {
4383                 case ixgbe_mac_82598EB:
4384                         ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4385                         break;
4386 
4387                 case ixgbe_mac_82599EB:
4388                 case ixgbe_mac_X540:
4389                         ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4390                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4391                         break;
4392 
4393                 default:
4394                         break;
4395                 }
4396                 ixgbe_intr_other_work(ixgbe, eicr);
4397                 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4398                 mutex_exit(&ixgbe->gen_lock);
4399         }
4400 
4401         /* re-enable the interrupts which were automasked */
4402         IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4403 
4404         return (DDI_INTR_CLAIMED);
4405 }
4406 
4407 /*
4408  * ixgbe_intr_msix - Interrupt handler for MSI-X.
4409  */
4410 static uint_t
4411 ixgbe_intr_msix(void *arg1, void *arg2)
4412 {
4413         ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4414         ixgbe_t *ixgbe = vect->ixgbe;
4415         struct ixgbe_hw *hw = &ixgbe->hw;
4416         uint32_t eicr;
4417         int r_idx = 0;
4418 
4419         _NOTE(ARGUNUSED(arg2));
4420 
4421         /*
4422          * Clean each rx ring that has its bit set in the map
4423          */
4424         r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4425         while (r_idx >= 0) {
4426                 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4427                 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4428                     (ixgbe->num_rx_rings - 1));
4429         }
4430 
4431         /*
4432          * Clean each tx ring that has its bit set in the map
4433          */
4434         r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4435         while (r_idx >= 0) {
4436                 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4437                 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4438                     (ixgbe->num_tx_rings - 1));
4439         }
4440 
4441 
4442         /*
4443          * Clean other interrupt (link change) that has its bit set in the map
4444          */
4445         if (BT_TEST(vect->other_map, 0) == 1) {
4446                 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4447 
4448                 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4449                     DDI_FM_OK) {
4450                         ddi_fm_service_impact(ixgbe->dip,
4451                             DDI_SERVICE_DEGRADED);
4452                         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4453                         return (DDI_INTR_CLAIMED);
4454                 }
4455 
4456                 /*
4457                  * Check "other" cause bits: any interrupt type other than tx/rx
4458                  */
4459                 if (eicr & ixgbe->capab->other_intr) {
4460                         mutex_enter(&ixgbe->gen_lock);
4461                         switch (hw->mac.type) {
4462                         case ixgbe_mac_82598EB:
4463                                 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4464                                 ixgbe_intr_other_work(ixgbe, eicr);
4465                                 break;
4466 
4467                         case ixgbe_mac_82599EB:
4468                         case ixgbe_mac_X540:
4469                                 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4470                                 ixgbe_intr_other_work(ixgbe, eicr);
4471                                 break;
4472 
4473                         default:
4474                                 break;
4475                         }
4476                         mutex_exit(&ixgbe->gen_lock);
4477                 }
4478 
4479                 /* re-enable the interrupts which were automasked */
4480                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4481         }
4482 
4483         return (DDI_INTR_CLAIMED);
4484 }
4485 
4486 /*
4487  * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4488  *
4489  * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4490  * if not successful, try Legacy.
4491  * ixgbe->intr_force can be used to force sequence to start with
4492  * any of the 3 types.
4493  * If MSI-X is not used, number of tx/rx rings is forced to 1.
4494  */
4495 static int
4496 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4497 {
4498         dev_info_t *devinfo;
4499         int intr_types;
4500         int rc;
4501 
4502         devinfo = ixgbe->dip;
4503 
4504         /*
4505          * Get supported interrupt types
4506          */
4507         rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4508 
4509         if (rc != DDI_SUCCESS) {
4510                 ixgbe_log(ixgbe,
4511                     "Get supported interrupt types failed: %d", rc);
4512                 return (IXGBE_FAILURE);
4513         }
4514         IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4515 
4516         ixgbe->intr_type = 0;
4517 
4518         /*
4519          * Install MSI-X interrupts
4520          */
4521         if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4522             (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4523                 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4524                 if (rc == IXGBE_SUCCESS)
4525                         return (IXGBE_SUCCESS);
4526 
4527                 ixgbe_log(ixgbe,
4528                     "Allocate MSI-X failed, trying MSI interrupts...");
4529         }
4530 
4531         /*
4532          * MSI-X not used, force rings and groups to 1
4533          */
4534         ixgbe->num_rx_rings = 1;
4535         ixgbe->num_rx_groups = 1;
4536         ixgbe->num_tx_rings = 1;
4537         ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4538         ixgbe_log(ixgbe,
4539             "MSI-X not used, force rings and groups number to 1");
4540 
4541         /*
4542          * Install MSI interrupts
4543          */
4544         if ((intr_types & DDI_INTR_TYPE_MSI) &&
4545             (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4546                 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4547                 if (rc == IXGBE_SUCCESS)
4548                         return (IXGBE_SUCCESS);
4549 
4550                 ixgbe_log(ixgbe,
4551                     "Allocate MSI failed, trying Legacy interrupts...");
4552         }
4553 
4554         /*
4555          * Install legacy interrupts
4556          */
4557         if (intr_types & DDI_INTR_TYPE_FIXED) {
4558                 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4559                 if (rc == IXGBE_SUCCESS)
4560                         return (IXGBE_SUCCESS);
4561 
4562                 ixgbe_log(ixgbe,
4563                     "Allocate Legacy interrupts failed");
4564         }
4565 
4566         /*
4567          * If none of the 3 types succeeded, return failure
4568          */
4569         return (IXGBE_FAILURE);
4570 }
4571 
4572 /*
4573  * ixgbe_alloc_intr_handles - Allocate interrupt handles.
4574  *
4575  * For legacy and MSI, only 1 handle is needed.  For MSI-X,
4576  * if fewer than 2 handles are available, return failure.
4577  * Upon success, this maps the vectors to rx and tx rings for
4578  * interrupts.
4579  */
4580 static int
4581 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
4582 {
4583         dev_info_t *devinfo;
4584         int request, count, actual;
4585         int minimum;
4586         int rc;
4587         uint32_t ring_per_group;
4588 
4589         devinfo = ixgbe->dip;
4590 
4591         switch (intr_type) {
4592         case DDI_INTR_TYPE_FIXED:
4593                 request = 1;    /* Request 1 legacy interrupt handle */
4594                 minimum = 1;
4595                 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
4596                 break;
4597 
4598         case DDI_INTR_TYPE_MSI:
4599                 request = 1;    /* Request 1 MSI interrupt handle */
4600                 minimum = 1;
4601                 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
4602                 break;
4603 
4604         case DDI_INTR_TYPE_MSIX:
4605                 /*
4606                  * Best number of vectors for the adapter is
4607                  * (# rx rings + # tx rings), however we will
4608                  * limit the request number.
4609                  */
4610                 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
4611                 if (request > ixgbe->capab->max_ring_vect)
4612                         request = ixgbe->capab->max_ring_vect;
4613                 minimum = 1;
4614                 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
4615                 break;
4616 
4617         default:
4618                 ixgbe_log(ixgbe,
4619                     "invalid call to ixgbe_alloc_intr_handles(): %d\n",
4620                     intr_type);
4621                 return (IXGBE_FAILURE);
4622         }
4623         IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d  minimum: %d",
4624             request, minimum);
4625 
4626         /*
4627          * Get number of supported interrupts
4628          */
4629         rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4630         if ((rc != DDI_SUCCESS) || (count < minimum)) {
4631                 ixgbe_log(ixgbe,
4632                     "Get interrupt number failed. Return: %d, count: %d",
4633                     rc, count);
4634                 return (IXGBE_FAILURE);
4635         }
4636         IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
4637 
4638         actual = 0;
4639         ixgbe->intr_cnt = 0;
4640         ixgbe->intr_cnt_max = 0;
4641         ixgbe->intr_cnt_min = 0;
4642 
4643         /*
4644          * Allocate an array of interrupt handles
4645          */
4646         ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
4647         ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
4648 
4649         rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
4650             request, &actual, DDI_INTR_ALLOC_NORMAL);
4651         if (rc != DDI_SUCCESS) {
4652                 ixgbe_log(ixgbe, "Allocate interrupts failed. "
4653                     "return: %d, request: %d, actual: %d",
4654                     rc, request, actual);
4655                 goto alloc_handle_fail;
4656         }
4657         IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
4658 
4659         /*
4660          * upper/lower limit of interrupts
4661          */
4662         ixgbe->intr_cnt = actual;
4663         ixgbe->intr_cnt_max = request;
4664         ixgbe->intr_cnt_min = minimum;
4665 
4666         /*
4667          * rss number per group should not exceed the rx interrupt number,
4668          * else need to adjust rx ring number.
4669          */
4670         ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4671         ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
4672         if (actual < ring_per_group) {
4673                 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
4674                 ixgbe_setup_vmdq_rss_conf(ixgbe);
4675         }
4676 
4677         /*
4678          * Now we know the actual number of vectors.  Here we map the vector
4679          * to other, rx rings and tx ring.
4680          */
4681         if (actual < minimum) {
4682                 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
4683                     actual);
4684                 goto alloc_handle_fail;
4685         }
4686 
4687         /*
4688          * Get priority for first vector, assume remaining are all the same
4689          */
4690         rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
4691         if (rc != DDI_SUCCESS) {
4692                 ixgbe_log(ixgbe,
4693                     "Get interrupt priority failed: %d", rc);
4694                 goto alloc_handle_fail;
4695         }
4696 
4697         rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
4698         if (rc != DDI_SUCCESS) {
4699                 ixgbe_log(ixgbe,
4700                     "Get interrupt cap failed: %d", rc);
4701                 goto alloc_handle_fail;
4702         }
4703 
4704         ixgbe->intr_type = intr_type;
4705 
4706         return (IXGBE_SUCCESS);
4707 
4708 alloc_handle_fail:
4709         ixgbe_rem_intrs(ixgbe);
4710 
4711         return (IXGBE_FAILURE);
4712 }
4713 
4714 /*
4715  * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
4716  *
4717  * Before adding the interrupt handlers, the interrupt vectors have
4718  * been allocated, and the rx/tx rings have also been allocated.
4719  */
4720 static int
4721 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
4722 {
4723         int vector = 0;
4724         int rc;
4725 
4726         switch (ixgbe->intr_type) {
4727         case DDI_INTR_TYPE_MSIX:
4728                 /*
4729                  * Add interrupt handler for all vectors
4730                  */
4731                 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
4732                         /*
4733                          * install pointer to vect_map[vector]
4734                          */
4735                         rc = ddi_intr_add_handler(ixgbe->htable[vector],
4736                             (ddi_intr_handler_t *)ixgbe_intr_msix,
4737                             (void *)&ixgbe->vect_map[vector], NULL);
4738 
4739                         if (rc != DDI_SUCCESS) {
4740                                 ixgbe_log(ixgbe,
4741                                     "Add interrupt handler failed. "
4742                                     "return: %d, vector: %d", rc, vector);
4743                                 for (vector--; vector >= 0; vector--) {
4744                                         (void) ddi_intr_remove_handler(
4745                                             ixgbe->htable[vector]);
4746                                 }
4747                                 return (IXGBE_FAILURE);
4748                         }
4749                 }
4750 
4751                 break;
4752 
4753         case DDI_INTR_TYPE_MSI:
4754                 /*
4755                  * Add interrupt handlers for the only vector
4756                  */
4757                 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4758                     (ddi_intr_handler_t *)ixgbe_intr_msi,
4759                     (void *)ixgbe, NULL);
4760 
4761                 if (rc != DDI_SUCCESS) {
4762                         ixgbe_log(ixgbe,
4763                             "Add MSI interrupt handler failed: %d", rc);
4764                         return (IXGBE_FAILURE);
4765                 }
4766 
4767                 break;
4768 
4769         case DDI_INTR_TYPE_FIXED:
4770                 /*
4771                  * Add interrupt handlers for the only vector
4772                  */
4773                 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4774                     (ddi_intr_handler_t *)ixgbe_intr_legacy,
4775                     (void *)ixgbe, NULL);
4776 
4777                 if (rc != DDI_SUCCESS) {
4778                         ixgbe_log(ixgbe,
4779                             "Add legacy interrupt handler failed: %d", rc);
4780                         return (IXGBE_FAILURE);
4781                 }
4782 
4783                 break;
4784 
4785         default:
4786                 return (IXGBE_FAILURE);
4787         }
4788 
4789         return (IXGBE_SUCCESS);
4790 }
4791 
4792 #pragma inline(ixgbe_map_rxring_to_vector)
4793 /*
4794  * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
4795  */
4796 static void
4797 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
4798 {
4799         /*
4800          * Set bit in map
4801          */
4802         BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
4803 
4804         /*
4805          * Count bits set
4806          */
4807         ixgbe->vect_map[v_idx].rxr_cnt++;
4808 
4809         /*
4810          * Remember bit position
4811          */
4812         ixgbe->rx_rings[r_idx].intr_vector = v_idx;
4813         ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
4814 }
4815 
4816 #pragma inline(ixgbe_map_txring_to_vector)
4817 /*
4818  * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
4819  */
4820 static void
4821 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
4822 {
4823         /*
4824          * Set bit in map
4825          */
4826         BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
4827 
4828         /*
4829          * Count bits set
4830          */
4831         ixgbe->vect_map[v_idx].txr_cnt++;
4832 
4833         /*
4834          * Remember bit position
4835          */
4836         ixgbe->tx_rings[t_idx].intr_vector = v_idx;
4837         ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
4838 }
4839 
4840 /*
4841  * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
4842  * allocation register (IVAR).
4843  * cause:
4844  *   -1 : other cause
4845  *    0 : rx
4846  *    1 : tx
4847  */
4848 static void
4849 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
4850     int8_t cause)
4851 {
4852         struct ixgbe_hw *hw = &ixgbe->hw;
4853         u32 ivar, index;
4854 
4855         switch (hw->mac.type) {
4856         case ixgbe_mac_82598EB:
4857                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4858                 if (cause == -1) {
4859                         cause = 0;
4860                 }
4861                 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4862                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4863                 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
4864                 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
4865                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4866                 break;
4867 
4868         case ixgbe_mac_82599EB:
4869         case ixgbe_mac_X540:
4870                 if (cause == -1) {
4871                         /* other causes */
4872                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4873                         index = (intr_alloc_entry & 1) * 8;
4874                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4875                         ivar &= ~(0xFF << index);
4876                         ivar |= (msix_vector << index);
4877                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4878                 } else {
4879                         /* tx or rx causes */
4880                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4881                         index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4882                         ivar = IXGBE_READ_REG(hw,
4883                             IXGBE_IVAR(intr_alloc_entry >> 1));
4884                         ivar &= ~(0xFF << index);
4885                         ivar |= (msix_vector << index);
4886                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4887                             ivar);
4888                 }
4889                 break;
4890 
4891         default:
4892                 break;
4893         }
4894 }
4895 
4896 /*
4897  * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
4898  * given interrupt vector allocation register (IVAR).
4899  * cause:
4900  *   -1 : other cause
4901  *    0 : rx
4902  *    1 : tx
4903  */
4904 static void
4905 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4906 {
4907         struct ixgbe_hw *hw = &ixgbe->hw;
4908         u32 ivar, index;
4909 
4910         switch (hw->mac.type) {
4911         case ixgbe_mac_82598EB:
4912                 if (cause == -1) {
4913                         cause = 0;
4914                 }
4915                 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4916                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4917                 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
4918                     (intr_alloc_entry & 0x3)));
4919                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4920                 break;
4921 
4922         case ixgbe_mac_82599EB:
4923         case ixgbe_mac_X540:
4924                 if (cause == -1) {
4925                         /* other causes */
4926                         index = (intr_alloc_entry & 1) * 8;
4927                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4928                         ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4929                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4930                 } else {
4931                         /* tx or rx causes */
4932                         index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4933                         ivar = IXGBE_READ_REG(hw,
4934                             IXGBE_IVAR(intr_alloc_entry >> 1));
4935                         ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
4936                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4937                             ivar);
4938                 }
4939                 break;
4940 
4941         default:
4942                 break;
4943         }
4944 }
4945 
4946 /*
4947  * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
4948  * given interrupt vector allocation register (IVAR).
4949  * cause:
4950  *   -1 : other cause
4951  *    0 : rx
4952  *    1 : tx
4953  */
4954 static void
4955 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4956 {
4957         struct ixgbe_hw *hw = &ixgbe->hw;
4958         u32 ivar, index;
4959 
4960         switch (hw->mac.type) {
4961         case ixgbe_mac_82598EB:
4962                 if (cause == -1) {
4963                         cause = 0;
4964                 }
4965                 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4966                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4967                 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
4968                     (intr_alloc_entry & 0x3)));
4969                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4970                 break;
4971 
4972         case ixgbe_mac_82599EB:
4973         case ixgbe_mac_X540:
4974                 if (cause == -1) {
4975                         /* other causes */
4976                         index = (intr_alloc_entry & 1) * 8;
4977                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4978                         ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4979                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4980                 } else {
4981                         /* tx or rx causes */
4982                         index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4983                         ivar = IXGBE_READ_REG(hw,
4984                             IXGBE_IVAR(intr_alloc_entry >> 1));
4985                         ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
4986                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4987                             ivar);
4988                 }
4989                 break;
4990 
4991         default:
4992                 break;
4993         }
4994 }
4995 
4996 /*
4997  * Convert the rx ring index driver maintained to the rx ring index
4998  * in h/w.
4999  */
5000 static uint32_t
5001 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5002 {
5003 
5004         struct ixgbe_hw *hw = &ixgbe->hw;
5005         uint32_t rx_ring_per_group, hw_rx_index;
5006 
5007         if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5008             ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5009                 return (sw_rx_index);
5010         } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5011                 switch (hw->mac.type) {
5012                 case ixgbe_mac_82598EB:
5013                         return (sw_rx_index);
5014 
5015                 case ixgbe_mac_82599EB:
5016                 case ixgbe_mac_X540:
5017                         return (sw_rx_index * 2);
5018 
5019                 default:
5020                         break;
5021                 }
5022         } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5023                 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5024 
5025                 switch (hw->mac.type) {
5026                 case ixgbe_mac_82598EB:
5027                         hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5028                             16 + (sw_rx_index % rx_ring_per_group);
5029                         return (hw_rx_index);
5030 
5031                 case ixgbe_mac_82599EB:
5032                 case ixgbe_mac_X540:
5033                         if (ixgbe->num_rx_groups > 32) {
5034                                 hw_rx_index = (sw_rx_index /
5035                                     rx_ring_per_group) * 2 +
5036                                     (sw_rx_index % rx_ring_per_group);
5037                         } else {
5038                                 hw_rx_index = (sw_rx_index /
5039                                     rx_ring_per_group) * 4 +
5040                                     (sw_rx_index % rx_ring_per_group);
5041                         }
5042                         return (hw_rx_index);
5043 
5044                 default:
5045                         break;
5046                 }
5047         }
5048 
5049         /*
5050          * Should never reach. Just to make compiler happy.
5051          */
5052         return (sw_rx_index);
5053 }
5054 
5055 /*
5056  * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5057  *
5058  * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5059  * to vector[0 - (intr_cnt -1)].
5060  */
5061 static int
5062 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5063 {
5064         int i, vector = 0;
5065 
5066         /* initialize vector map */
5067         bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5068         for (i = 0; i < ixgbe->intr_cnt; i++) {
5069                 ixgbe->vect_map[i].ixgbe = ixgbe;
5070         }
5071 
5072         /*
5073          * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5074          * tx rings[0] on RTxQ[1].
5075          */
5076         if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5077                 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5078                 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5079                 return (IXGBE_SUCCESS);
5080         }
5081 
5082         /*
5083          * Interrupts/vectors mapping for MSI-X
5084          */
5085 
5086         /*
5087          * Map other interrupt to vector 0,
5088          * Set bit in map and count the bits set.
5089          */
5090         BT_SET(ixgbe->vect_map[vector].other_map, 0);
5091         ixgbe->vect_map[vector].other_cnt++;
5092 
5093         /*
5094          * Map rx ring interrupts to vectors
5095          */
5096         for (i = 0; i < ixgbe->num_rx_rings; i++) {
5097                 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5098                 vector = (vector +1) % ixgbe->intr_cnt;
5099         }
5100 
5101         /*
5102          * Map tx ring interrupts to vectors
5103          */
5104         for (i = 0; i < ixgbe->num_tx_rings; i++) {
5105                 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5106                 vector = (vector +1) % ixgbe->intr_cnt;
5107         }
5108 
5109         return (IXGBE_SUCCESS);
5110 }
5111 
5112 /*
5113  * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5114  *
5115  * This relies on ring/vector mapping already set up in the
5116  * vect_map[] structures
5117  */
5118 static void
5119 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5120 {
5121         struct ixgbe_hw *hw = &ixgbe->hw;
5122         ixgbe_intr_vector_t *vect;      /* vector bitmap */
5123         int r_idx;      /* ring index */
5124         int v_idx;      /* vector index */
5125         uint32_t hw_index;
5126 
5127         /*
5128          * Clear any previous entries
5129          */
5130         switch (hw->mac.type) {
5131         case ixgbe_mac_82598EB:
5132                 for (v_idx = 0; v_idx < 25; v_idx++)
5133                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5134                 break;
5135 
5136         case ixgbe_mac_82599EB:
5137         case ixgbe_mac_X540:
5138                 for (v_idx = 0; v_idx < 64; v_idx++)
5139                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5140                 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5141                 break;
5142 
5143         default:
5144                 break;
5145         }
5146 
5147         /*
5148          * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5149          * tx rings[0] will use RTxQ[1].
5150          */
5151         if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5152                 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5153                 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5154                 return;
5155         }
5156 
5157         /*
5158          * For MSI-X interrupt, "Other" is always on vector[0].
5159          */
5160         ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5161 
5162         /*
5163          * For each interrupt vector, populate the IVAR table
5164          */
5165         for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5166                 vect = &ixgbe->vect_map[v_idx];
5167 
5168                 /*
5169                  * For each rx ring bit set
5170                  */
5171                 r_idx = bt_getlowbit(vect->rx_map, 0,
5172                     (ixgbe->num_rx_rings - 1));
5173 
5174                 while (r_idx >= 0) {
5175                         hw_index = ixgbe->rx_rings[r_idx].hw_index;
5176                         ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5177                         r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5178                             (ixgbe->num_rx_rings - 1));
5179                 }
5180 
5181                 /*
5182                  * For each tx ring bit set
5183                  */
5184                 r_idx = bt_getlowbit(vect->tx_map, 0,
5185                     (ixgbe->num_tx_rings - 1));
5186 
5187                 while (r_idx >= 0) {
5188                         ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5189                         r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5190                             (ixgbe->num_tx_rings - 1));
5191                 }
5192         }
5193 }
5194 
5195 /*
5196  * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5197  */
5198 static void
5199 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5200 {
5201         int i;
5202         int rc;
5203 
5204         for (i = 0; i < ixgbe->intr_cnt; i++) {
5205                 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5206                 if (rc != DDI_SUCCESS) {
5207                         IXGBE_DEBUGLOG_1(ixgbe,
5208                             "Remove intr handler failed: %d", rc);
5209                 }
5210         }
5211 }
5212 
5213 /*
5214  * ixgbe_rem_intrs - Remove the allocated interrupts.
5215  */
5216 static void
5217 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5218 {
5219         int i;
5220         int rc;
5221 
5222         for (i = 0; i < ixgbe->intr_cnt; i++) {
5223                 rc = ddi_intr_free(ixgbe->htable[i]);
5224                 if (rc != DDI_SUCCESS) {
5225                         IXGBE_DEBUGLOG_1(ixgbe,
5226                             "Free intr failed: %d", rc);
5227                 }
5228         }
5229 
5230         kmem_free(ixgbe->htable, ixgbe->intr_size);
5231         ixgbe->htable = NULL;
5232 }
5233 
5234 /*
5235  * ixgbe_enable_intrs - Enable all the ddi interrupts.
5236  */
5237 static int
5238 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5239 {
5240         int i;
5241         int rc;
5242 
5243         /*
5244          * Enable interrupts
5245          */
5246         if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5247                 /*
5248                  * Call ddi_intr_block_enable() for MSI
5249                  */
5250                 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5251                 if (rc != DDI_SUCCESS) {
5252                         ixgbe_log(ixgbe,
5253                             "Enable block intr failed: %d", rc);
5254                         return (IXGBE_FAILURE);
5255                 }
5256         } else {
5257                 /*
5258                  * Call ddi_intr_enable() for Legacy/MSI non block enable
5259                  */
5260                 for (i = 0; i < ixgbe->intr_cnt; i++) {
5261                         rc = ddi_intr_enable(ixgbe->htable[i]);
5262                         if (rc != DDI_SUCCESS) {
5263                                 ixgbe_log(ixgbe,
5264                                     "Enable intr failed: %d", rc);
5265                                 return (IXGBE_FAILURE);
5266                         }
5267                 }
5268         }
5269 
5270         return (IXGBE_SUCCESS);
5271 }
5272 
5273 /*
5274  * ixgbe_disable_intrs - Disable all the interrupts.
5275  */
5276 static int
5277 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5278 {
5279         int i;
5280         int rc;
5281 
5282         /*
5283          * Disable all interrupts
5284          */
5285         if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5286                 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5287                 if (rc != DDI_SUCCESS) {
5288                         ixgbe_log(ixgbe,
5289                             "Disable block intr failed: %d", rc);
5290                         return (IXGBE_FAILURE);
5291                 }
5292         } else {
5293                 for (i = 0; i < ixgbe->intr_cnt; i++) {
5294                         rc = ddi_intr_disable(ixgbe->htable[i]);
5295                         if (rc != DDI_SUCCESS) {
5296                                 ixgbe_log(ixgbe,
5297                                     "Disable intr failed: %d", rc);
5298                                 return (IXGBE_FAILURE);
5299                         }
5300                 }
5301         }
5302 
5303         return (IXGBE_SUCCESS);
5304 }
5305 
5306 /*
5307  * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5308  */
5309 static void
5310 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5311 {
5312         struct ixgbe_hw *hw = &ixgbe->hw;
5313         ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
5314         boolean_t link_up = B_FALSE;
5315         uint32_t pcs1g_anlp = 0;
5316         uint32_t pcs1g_ana = 0;
5317         boolean_t autoneg = B_FALSE;
5318 
5319         ASSERT(mutex_owned(&ixgbe->gen_lock));
5320         ixgbe->param_lp_1000fdx_cap = 0;
5321         ixgbe->param_lp_100fdx_cap  = 0;
5322 
5323         /* check for link, don't wait */
5324         (void) ixgbe_check_link(hw, &speed, &link_up, false);
5325         pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
5326 
5327         if (link_up) {
5328                 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5329 
5330                 ixgbe->param_lp_1000fdx_cap =
5331                     (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5332                 ixgbe->param_lp_100fdx_cap =
5333                     (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5334         }
5335 
5336         (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg);
5337 
5338         ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5339             (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0;
5340         ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5341             (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0;
5342 }
5343 
5344 /*
5345  * ixgbe_get_driver_control - Notify that driver is in control of device.
5346  */
5347 static void
5348 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5349 {
5350         uint32_t ctrl_ext;
5351 
5352         /*
5353          * Notify firmware that driver is in control of device
5354          */
5355         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5356         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5357         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5358 }
5359 
5360 /*
5361  * ixgbe_release_driver_control - Notify that driver is no longer in control
5362  * of device.
5363  */
5364 static void
5365 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5366 {
5367         uint32_t ctrl_ext;
5368 
5369         /*
5370          * Notify firmware that driver is no longer in control of device
5371          */
5372         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5373         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5374         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5375 }
5376 
5377 /*
5378  * ixgbe_atomic_reserve - Atomic decrease operation.
5379  */
5380 int
5381 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5382 {
5383         uint32_t oldval;
5384         uint32_t newval;
5385 
5386         /*
5387          * ATOMICALLY
5388          */
5389         do {
5390                 oldval = *count_p;
5391                 if (oldval < n)
5392                         return (-1);
5393                 newval = oldval - n;
5394         } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5395 
5396         return (newval);
5397 }
5398 
5399 /*
5400  * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5401  */
5402 static uint8_t *
5403 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5404 {
5405         uint8_t *addr = *upd_ptr;
5406         uint8_t *new_ptr;
5407 
5408         _NOTE(ARGUNUSED(hw));
5409         _NOTE(ARGUNUSED(vmdq));
5410 
5411         new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5412         *upd_ptr = new_ptr;
5413         return (addr);
5414 }
5415 
5416 /*
5417  * FMA support
5418  */
5419 int
5420 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5421 {
5422         ddi_fm_error_t de;
5423 
5424         ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5425         ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5426         return (de.fme_status);
5427 }
5428 
5429 int
5430 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5431 {
5432         ddi_fm_error_t de;
5433 
5434         ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5435         return (de.fme_status);
5436 }
5437 
5438 /*
5439  * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5440  */
5441 static int
5442 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5443 {
5444         _NOTE(ARGUNUSED(impl_data));
5445         /*
5446          * as the driver can always deal with an error in any dma or
5447          * access handle, we can just return the fme_status value.
5448          */
5449         pci_ereport_post(dip, err, NULL);
5450         return (err->fme_status);
5451 }
5452 
5453 static void
5454 ixgbe_fm_init(ixgbe_t *ixgbe)
5455 {
5456         ddi_iblock_cookie_t iblk;
5457         int fma_dma_flag;
5458 
5459         /*
5460          * Only register with IO Fault Services if we have some capability
5461          */
5462         if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5463                 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5464         } else {
5465                 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5466         }
5467 
5468         if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5469                 fma_dma_flag = 1;
5470         } else {
5471                 fma_dma_flag = 0;
5472         }
5473 
5474         ixgbe_set_fma_flags(fma_dma_flag);
5475 
5476         if (ixgbe->fm_capabilities) {
5477 
5478                 /*
5479                  * Register capabilities with IO Fault Services
5480                  */
5481                 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5482 
5483                 /*
5484                  * Initialize pci ereport capabilities if ereport capable
5485                  */
5486                 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5487                     DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5488                         pci_ereport_setup(ixgbe->dip);
5489 
5490                 /*
5491                  * Register error callback if error callback capable
5492                  */
5493                 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5494                         ddi_fm_handler_register(ixgbe->dip,
5495                             ixgbe_fm_error_cb, (void*) ixgbe);
5496         }
5497 }
5498 
5499 static void
5500 ixgbe_fm_fini(ixgbe_t *ixgbe)
5501 {
5502         /*
5503          * Only unregister FMA capabilities if they are registered
5504          */
5505         if (ixgbe->fm_capabilities) {
5506 
5507                 /*
5508                  * Release any resources allocated by pci_ereport_setup()
5509                  */
5510                 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5511                     DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5512                         pci_ereport_teardown(ixgbe->dip);
5513 
5514                 /*
5515                  * Un-register error callback if error callback capable
5516                  */
5517                 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5518                         ddi_fm_handler_unregister(ixgbe->dip);
5519 
5520                 /*
5521                  * Unregister from IO Fault Service
5522                  */
5523                 ddi_fm_fini(ixgbe->dip);
5524         }
5525 }
5526 
5527 void
5528 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5529 {
5530         uint64_t ena;
5531         char buf[FM_MAX_CLASS];
5532 
5533         (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5534         ena = fm_ena_generate(0, FM_ENA_FMT1);
5535         if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
5536                 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
5537                     FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5538         }
5539 }
5540 
5541 static int
5542 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
5543 {
5544         ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
5545 
5546         mutex_enter(&rx_ring->rx_lock);
5547         rx_ring->ring_gen_num = mr_gen_num;
5548         mutex_exit(&rx_ring->rx_lock);
5549         return (0);
5550 }
5551 
5552 /*
5553  * Get the global ring index by a ring index within a group.
5554  */
5555 static int
5556 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
5557 {
5558         ixgbe_rx_ring_t *rx_ring;
5559         int i;
5560 
5561         for (i = 0; i < ixgbe->num_rx_rings; i++) {
5562                 rx_ring = &ixgbe->rx_rings[i];
5563                 if (rx_ring->group_index == gindex)
5564                         rindex--;
5565                 if (rindex < 0)
5566                         return (i);
5567         }
5568 
5569         return (-1);
5570 }
5571 
5572 /*
5573  * Callback funtion for MAC layer to register all rings.
5574  */
5575 /* ARGSUSED */
5576 void
5577 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
5578     const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
5579 {
5580         ixgbe_t *ixgbe = (ixgbe_t *)arg;
5581         mac_intr_t *mintr = &infop->mri_intr;
5582 
5583         switch (rtype) {
5584         case MAC_RING_TYPE_RX: {
5585                 /*
5586                  * 'index' is the ring index within the group.
5587                  * Need to get the global ring index by searching in groups.
5588                  */
5589                 int global_ring_index = ixgbe_get_rx_ring_index(
5590                     ixgbe, group_index, ring_index);
5591 
5592                 ASSERT(global_ring_index >= 0);
5593 
5594                 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
5595                 rx_ring->ring_handle = rh;
5596 
5597                 infop->mri_driver = (mac_ring_driver_t)rx_ring;
5598                 infop->mri_start = ixgbe_ring_start;
5599                 infop->mri_stop = NULL;
5600                 infop->mri_poll = ixgbe_ring_rx_poll;
5601                 infop->mri_stat = ixgbe_rx_ring_stat;
5602 
5603                 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
5604                 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
5605                 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
5606                 if (ixgbe->intr_type &
5607                     (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5608                         mintr->mi_ddi_handle =
5609                             ixgbe->htable[rx_ring->intr_vector];
5610                 }
5611 
5612                 break;
5613         }
5614         case MAC_RING_TYPE_TX: {
5615                 ASSERT(group_index == -1);
5616                 ASSERT(ring_index < ixgbe->num_tx_rings);
5617 
5618                 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
5619                 tx_ring->ring_handle = rh;
5620 
5621                 infop->mri_driver = (mac_ring_driver_t)tx_ring;
5622                 infop->mri_start = NULL;
5623                 infop->mri_stop = NULL;
5624                 infop->mri_tx = ixgbe_ring_tx;
5625                 infop->mri_stat = ixgbe_tx_ring_stat;
5626                 if (ixgbe->intr_type &
5627                     (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5628                         mintr->mi_ddi_handle =
5629                             ixgbe->htable[tx_ring->intr_vector];
5630                 }
5631                 break;
5632         }
5633         default:
5634                 break;
5635         }
5636 }
5637 
5638 /*
5639  * Callback funtion for MAC layer to register all groups.
5640  */
5641 void
5642 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
5643     mac_group_info_t *infop, mac_group_handle_t gh)
5644 {
5645         ixgbe_t *ixgbe = (ixgbe_t *)arg;
5646 
5647         switch (rtype) {
5648         case MAC_RING_TYPE_RX: {
5649                 ixgbe_rx_group_t *rx_group;
5650 
5651                 rx_group = &ixgbe->rx_groups[index];
5652                 rx_group->group_handle = gh;
5653 
5654                 infop->mgi_driver = (mac_group_driver_t)rx_group;
5655                 infop->mgi_start = NULL;
5656                 infop->mgi_stop = NULL;
5657                 infop->mgi_addmac = ixgbe_addmac;
5658                 infop->mgi_remmac = ixgbe_remmac;
5659                 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
5660 
5661                 break;
5662         }
5663         case MAC_RING_TYPE_TX:
5664                 break;
5665         default:
5666                 break;
5667         }
5668 }
5669 
5670 /*
5671  * Enable interrupt on the specificed rx ring.
5672  */
5673 int
5674 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
5675 {
5676         ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5677         ixgbe_t *ixgbe = rx_ring->ixgbe;
5678         int r_idx = rx_ring->index;
5679         int hw_r_idx = rx_ring->hw_index;
5680         int v_idx = rx_ring->intr_vector;
5681 
5682         mutex_enter(&ixgbe->gen_lock);
5683         if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5684                 mutex_exit(&ixgbe->gen_lock);
5685                 /*
5686                  * Simply return 0.
5687                  * Interrupts are being adjusted. ixgbe_intr_adjust()
5688                  * will eventually re-enable the interrupt when it's
5689                  * done with the adjustment.
5690                  */
5691                 return (0);
5692         }
5693 
5694         /*
5695          * To enable interrupt by setting the VAL bit of given interrupt
5696          * vector allocation register (IVAR).
5697          */
5698         ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
5699 
5700         BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5701 
5702         /*
5703          * Trigger a Rx interrupt on this ring
5704          */
5705         IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
5706         IXGBE_WRITE_FLUSH(&ixgbe->hw);
5707 
5708         mutex_exit(&ixgbe->gen_lock);
5709 
5710         return (0);
5711 }
5712 
5713 /*
5714  * Disable interrupt on the specificed rx ring.
5715  */
5716 int
5717 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
5718 {
5719         ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5720         ixgbe_t *ixgbe = rx_ring->ixgbe;
5721         int r_idx = rx_ring->index;
5722         int hw_r_idx = rx_ring->hw_index;
5723         int v_idx = rx_ring->intr_vector;
5724 
5725         mutex_enter(&ixgbe->gen_lock);
5726         if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5727                 mutex_exit(&ixgbe->gen_lock);
5728                 /*
5729                  * Simply return 0.
5730                  * In the rare case where an interrupt is being
5731                  * disabled while interrupts are being adjusted,
5732                  * we don't fail the operation. No interrupts will
5733                  * be generated while they are adjusted, and
5734                  * ixgbe_intr_adjust() will cause the interrupts
5735                  * to be re-enabled once it completes. Note that
5736                  * in this case, packets may be delivered to the
5737                  * stack via interrupts before xgbe_rx_ring_intr_enable()
5738                  * is called again. This is acceptable since interrupt
5739                  * adjustment is infrequent, and the stack will be
5740                  * able to handle these packets.
5741                  */
5742                 return (0);
5743         }
5744 
5745         /*
5746          * To disable interrupt by clearing the VAL bit of given interrupt
5747          * vector allocation register (IVAR).
5748          */
5749         ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
5750 
5751         BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
5752 
5753         mutex_exit(&ixgbe->gen_lock);
5754 
5755         return (0);
5756 }
5757 
5758 /*
5759  * Add a mac address.
5760  */
5761 static int
5762 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
5763 {
5764         ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5765         ixgbe_t *ixgbe = rx_group->ixgbe;
5766         struct ixgbe_hw *hw = &ixgbe->hw;
5767         int slot, i;
5768 
5769         mutex_enter(&ixgbe->gen_lock);
5770 
5771         if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5772                 mutex_exit(&ixgbe->gen_lock);
5773                 return (ECANCELED);
5774         }
5775 
5776         if (ixgbe->unicst_avail == 0) {
5777                 /* no slots available */
5778                 mutex_exit(&ixgbe->gen_lock);
5779                 return (ENOSPC);
5780         }
5781 
5782         /*
5783          * The first ixgbe->num_rx_groups slots are reserved for each respective
5784          * group. The rest slots are shared by all groups. While adding a
5785          * MAC address, reserved slots are firstly checked then the shared
5786          * slots are searched.
5787          */
5788         slot = -1;
5789         if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
5790                 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
5791                         if (ixgbe->unicst_addr[i].mac.set == 0) {
5792                                 slot = i;
5793                                 break;
5794                         }
5795                 }
5796         } else {
5797                 slot = rx_group->index;
5798         }
5799 
5800         if (slot == -1) {
5801                 /* no slots available */
5802                 mutex_exit(&ixgbe->gen_lock);
5803                 return (ENOSPC);
5804         }
5805 
5806         bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5807         (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
5808             rx_group->index, IXGBE_RAH_AV);
5809         ixgbe->unicst_addr[slot].mac.set = 1;
5810         ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
5811         ixgbe->unicst_avail--;
5812 
5813         mutex_exit(&ixgbe->gen_lock);
5814 
5815         return (0);
5816 }
5817 
5818 /*
5819  * Remove a mac address.
5820  */
5821 static int
5822 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
5823 {
5824         ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5825         ixgbe_t *ixgbe = rx_group->ixgbe;
5826         struct ixgbe_hw *hw = &ixgbe->hw;
5827         int slot;
5828 
5829         mutex_enter(&ixgbe->gen_lock);
5830 
5831         if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5832                 mutex_exit(&ixgbe->gen_lock);
5833                 return (ECANCELED);
5834         }
5835 
5836         slot = ixgbe_unicst_find(ixgbe, mac_addr);
5837         if (slot == -1) {
5838                 mutex_exit(&ixgbe->gen_lock);
5839                 return (EINVAL);
5840         }
5841 
5842         if (ixgbe->unicst_addr[slot].mac.set == 0) {
5843                 mutex_exit(&ixgbe->gen_lock);
5844                 return (EINVAL);
5845         }
5846 
5847         bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5848         (void) ixgbe_clear_rar(hw, slot);
5849         ixgbe->unicst_addr[slot].mac.set = 0;
5850         ixgbe->unicst_avail++;
5851 
5852         mutex_exit(&ixgbe->gen_lock);
5853 
5854         return (0);
5855 }