1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 #include <sys/scsi/adapters/pmcs/pmcs.h>
  25 
  26 #define PMCS_DRIVER_VERSION     "pmcs HBA device driver"
  27 
  28 static  char    *pmcs_driver_rev = PMCS_DRIVER_VERSION;
  29 
  30 /*
  31  * Non-DDI Compliant stuff
  32  */
  33 extern char hw_serial[];
  34 
  35 /*
  36  * Global driver data
  37  */
  38 void *pmcs_softc_state = NULL;
  39 void *pmcs_iport_softstate = NULL;
  40 
  41 /*
  42  * Tracing and Logging info
  43  */
  44 pmcs_tbuf_t *pmcs_tbuf = NULL;
  45 uint32_t pmcs_tbuf_num_elems = 0;
  46 pmcs_tbuf_t *pmcs_tbuf_ptr;
  47 uint32_t pmcs_tbuf_idx = 0;
  48 boolean_t pmcs_tbuf_wrap = B_FALSE;
  49 kmutex_t pmcs_trace_lock;
  50 
  51 /*
  52  * If pmcs_force_syslog value is non-zero, all messages put in the trace log
  53  * will also be sent to system log.
  54  */
  55 int pmcs_force_syslog = 0;
  56 int pmcs_console = 0;
  57 
  58 /*
  59  * External References
  60  */
  61 extern int ncpus_online;
  62 
  63 /*
  64  * Local static data
  65  */
  66 static int fwlog_level = 3;
  67 static int physpeed = PHY_LINK_ALL;
  68 static int phymode = PHY_LM_AUTO;
  69 static int block_mask = 0;
  70 static int phymap_stable_usec = 3 * MICROSEC;
  71 static int iportmap_stable_usec = 2 * MICROSEC;
  72 static int iportmap_csync_usec = 20 * MICROSEC;
  73 
  74 #ifdef DEBUG
  75 static int debug_mask = 1;
  76 #else
  77 static int debug_mask = 0;
  78 #endif
  79 
  80 #ifdef DISABLE_MSIX
  81 static int disable_msix = 1;
  82 #else
  83 static int disable_msix = 0;
  84 #endif
  85 
  86 #ifdef DISABLE_MSI
  87 static int disable_msi = 1;
  88 #else
  89 static int disable_msi = 0;
  90 #endif
  91 
  92 /*
  93  * DEBUG: testing: allow detach with an active port:
  94  *
  95  * # echo 'detach_driver_unconfig/W 10'         | mdb -kw
  96  * # echo 'scsi_hba_bus_unconfig_remove/W 1'    | mdb -kw
  97  * # echo 'pmcs`detach_with_active_port/W 1'    | mdb -kw
  98  * # modunload -i <pmcs_driver_index>
  99  */
 100 static int detach_with_active_port = 0;
 101 
 102 static uint16_t maxqdepth = 0xfffe;
 103 
 104 /*
 105  * Local prototypes
 106  */
 107 static int pmcs_attach(dev_info_t *, ddi_attach_cmd_t);
 108 static int pmcs_detach(dev_info_t *, ddi_detach_cmd_t);
 109 static int pmcs_unattach(pmcs_hw_t *);
 110 static int pmcs_iport_unattach(pmcs_iport_t *);
 111 static int pmcs_add_more_chunks(pmcs_hw_t *, unsigned long);
 112 static void pmcs_watchdog(void *);
 113 static int pmcs_setup_intr(pmcs_hw_t *);
 114 static int pmcs_teardown_intr(pmcs_hw_t *);
 115 
 116 static uint_t pmcs_nonio_ix(caddr_t, caddr_t);
 117 static uint_t pmcs_general_ix(caddr_t, caddr_t);
 118 static uint_t pmcs_event_ix(caddr_t, caddr_t);
 119 static uint_t pmcs_iodone_ix(caddr_t, caddr_t);
 120 static uint_t pmcs_fatal_ix(caddr_t, caddr_t);
 121 static uint_t pmcs_all_intr(caddr_t, caddr_t);
 122 static int pmcs_quiesce(dev_info_t *dip);
 123 static boolean_t pmcs_fabricate_wwid(pmcs_hw_t *);
 124 
 125 static void pmcs_create_all_phy_stats(pmcs_iport_t *);
 126 int pmcs_update_phy_stats(kstat_t *, int);
 127 
 128 static void pmcs_fm_fini(pmcs_hw_t *pwp);
 129 static void pmcs_fm_init(pmcs_hw_t *pwp);
 130 static int pmcs_fm_error_cb(dev_info_t *dip,
 131     ddi_fm_error_t *err, const void *impl_data);
 132 
 133 /*
 134  * Local configuration data
 135  */
 136 static struct dev_ops pmcs_ops = {
 137         DEVO_REV,               /* devo_rev, */
 138         0,                      /* refcnt */
 139         ddi_no_info,            /* info */
 140         nulldev,                /* identify */
 141         nulldev,                /* probe */
 142         pmcs_attach,            /* attach */
 143         pmcs_detach,            /* detach */
 144         nodev,                  /* reset */
 145         NULL,                   /* driver operations */
 146         NULL,                   /* bus operations */
 147         ddi_power,              /* power management */
 148         pmcs_quiesce            /* quiesce */
 149 };
 150 
 151 static struct modldrv modldrv = {
 152         &mod_driverops,
 153         PMCS_DRIVER_VERSION,
 154         &pmcs_ops,  /* driver ops */
 155 };
 156 static struct modlinkage modlinkage = {
 157         MODREV_1, &modldrv, NULL
 158 };
 159 
 160 const ddi_dma_attr_t pmcs_dattr = {
 161         DMA_ATTR_V0,                    /* dma_attr version     */
 162         0x0000000000000000ull,          /* dma_attr_addr_lo     */
 163         0xFFFFFFFFFFFFFFFFull,          /* dma_attr_addr_hi     */
 164         0x00000000FFFFFFFFull,          /* dma_attr_count_max   */
 165         0x0000000000000001ull,          /* dma_attr_align       */
 166         0x00000078,                     /* dma_attr_burstsizes  */
 167         0x00000001,                     /* dma_attr_minxfer     */
 168         0x00000000FFFFFFFFull,          /* dma_attr_maxxfer     */
 169         0x00000000FFFFFFFFull,          /* dma_attr_seg         */
 170         1,                              /* dma_attr_sgllen      */
 171         512,                            /* dma_attr_granular    */
 172         0                               /* dma_attr_flags       */
 173 };
 174 
 175 static ddi_device_acc_attr_t rattr = {
 176         DDI_DEVICE_ATTR_V1,
 177         DDI_STRUCTURE_LE_ACC,
 178         DDI_STRICTORDER_ACC,
 179         DDI_DEFAULT_ACC
 180 };
 181 
 182 
 183 /*
 184  * Attach/Detach functions
 185  */
 186 
 187 int
 188 _init(void)
 189 {
 190         int ret;
 191 
 192         ret = ddi_soft_state_init(&pmcs_softc_state, sizeof (pmcs_hw_t), 1);
 193         if (ret != 0) {
 194                 cmn_err(CE_WARN, "?soft state init failed for pmcs");
 195                 return (ret);
 196         }
 197 
 198         if ((ret = scsi_hba_init(&modlinkage)) != 0) {
 199                 cmn_err(CE_WARN, "?scsi_hba_init failed for pmcs");
 200                 ddi_soft_state_fini(&pmcs_softc_state);
 201                 return (ret);
 202         }
 203 
 204         /*
 205          * Allocate soft state for iports
 206          */
 207         ret = ddi_soft_state_init(&pmcs_iport_softstate,
 208             sizeof (pmcs_iport_t), 2);
 209         if (ret != 0) {
 210                 cmn_err(CE_WARN, "?iport soft state init failed for pmcs");
 211                 ddi_soft_state_fini(&pmcs_softc_state);
 212                 return (ret);
 213         }
 214 
 215         ret = mod_install(&modlinkage);
 216         if (ret != 0) {
 217                 cmn_err(CE_WARN, "?mod_install failed for pmcs (%d)", ret);
 218                 scsi_hba_fini(&modlinkage);
 219                 ddi_soft_state_fini(&pmcs_iport_softstate);
 220                 ddi_soft_state_fini(&pmcs_softc_state);
 221                 return (ret);
 222         }
 223 
 224         /* Initialize the global trace lock */
 225         mutex_init(&pmcs_trace_lock, NULL, MUTEX_DRIVER, NULL);
 226 
 227         return (0);
 228 }
 229 
 230 int
 231 _fini(void)
 232 {
 233         int ret;
 234         if ((ret = mod_remove(&modlinkage)) != 0) {
 235                 return (ret);
 236         }
 237         scsi_hba_fini(&modlinkage);
 238 
 239         /* Free pmcs log buffer and destroy the global lock */
 240         if (pmcs_tbuf) {
 241                 kmem_free(pmcs_tbuf,
 242                     pmcs_tbuf_num_elems * sizeof (pmcs_tbuf_t));
 243                 pmcs_tbuf = NULL;
 244         }
 245         mutex_destroy(&pmcs_trace_lock);
 246 
 247         ddi_soft_state_fini(&pmcs_iport_softstate);
 248         ddi_soft_state_fini(&pmcs_softc_state);
 249         return (0);
 250 }
 251 
 252 int
 253 _info(struct modinfo *modinfop)
 254 {
 255         return (mod_info(&modlinkage, modinfop));
 256 }
 257 
 258 static int
 259 pmcs_iport_attach(dev_info_t *dip)
 260 {
 261         pmcs_iport_t            *iport;
 262         pmcs_hw_t               *pwp;
 263         scsi_hba_tran_t         *tran;
 264         void                    *ua_priv = NULL;
 265         char                    *iport_ua;
 266         char                    *init_port;
 267         int                     hba_inst;
 268         int                     inst;
 269 
 270         hba_inst = ddi_get_instance(ddi_get_parent(dip));
 271         inst = ddi_get_instance(dip);
 272 
 273         pwp = ddi_get_soft_state(pmcs_softc_state, hba_inst);
 274         if (pwp == NULL) {
 275                 cmn_err(CE_WARN, "%s: No HBA softstate for instance %d",
 276                     __func__, inst);
 277                 return (DDI_FAILURE);
 278         }
 279 
 280         if ((pwp->state == STATE_UNPROBING) || (pwp->state == STATE_DEAD)) {
 281                 return (DDI_FAILURE);
 282         }
 283 
 284         if ((iport_ua = scsi_hba_iport_unit_address(dip)) == NULL) {
 285                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 286                     "%s: invoked with NULL unit address, inst (%d)",
 287                     __func__, inst);
 288                 return (DDI_FAILURE);
 289         }
 290 
 291         if (ddi_soft_state_zalloc(pmcs_iport_softstate, inst) != DDI_SUCCESS) {
 292                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 293                     "Failed to alloc soft state for iport %d", inst);
 294                 return (DDI_FAILURE);
 295         }
 296 
 297         iport = ddi_get_soft_state(pmcs_iport_softstate, inst);
 298         if (iport == NULL) {
 299                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 300                     "cannot get iport soft state");
 301                 goto iport_attach_fail1;
 302         }
 303 
 304         mutex_init(&iport->lock, NULL, MUTEX_DRIVER,
 305             DDI_INTR_PRI(pwp->intr_pri));
 306         cv_init(&iport->refcnt_cv, NULL, CV_DEFAULT, NULL);
 307         cv_init(&iport->smp_cv, NULL, CV_DEFAULT, NULL);
 308         mutex_init(&iport->refcnt_lock, NULL, MUTEX_DRIVER,
 309             DDI_INTR_PRI(pwp->intr_pri));
 310         mutex_init(&iport->smp_lock, NULL, MUTEX_DRIVER,
 311             DDI_INTR_PRI(pwp->intr_pri));
 312 
 313         /* Set some data on the iport handle */
 314         iport->dip = dip;
 315         iport->pwp = pwp;
 316 
 317         /* Dup the UA into the iport handle */
 318         iport->ua = strdup(iport_ua);
 319 
 320         tran = (scsi_hba_tran_t *)ddi_get_driver_private(dip);
 321         tran->tran_hba_private = iport;
 322 
 323         list_create(&iport->phys, sizeof (pmcs_phy_t),
 324             offsetof(pmcs_phy_t, list_node));
 325 
 326         /*
 327          * If our unit address is active in the phymap, configure our
 328          * iport's phylist.
 329          */
 330         mutex_enter(&iport->lock);
 331         ua_priv = sas_phymap_lookup_uapriv(pwp->hss_phymap, iport->ua);
 332         if (ua_priv) {
 333                 /* Non-NULL private data indicates the unit address is active */
 334                 iport->ua_state = UA_ACTIVE;
 335                 if (pmcs_iport_configure_phys(iport) != DDI_SUCCESS) {
 336                         pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
 337                             "%s: failed to "
 338                             "configure phys on iport handle (0x%p), "
 339                             " unit address [%s]", __func__,
 340                             (void *)iport, iport_ua);
 341                         mutex_exit(&iport->lock);
 342                         goto iport_attach_fail2;
 343                 }
 344         } else {
 345                 iport->ua_state = UA_INACTIVE;
 346         }
 347         mutex_exit(&iport->lock);
 348 
 349         /* Allocate string-based soft state pool for targets */
 350         iport->tgt_sstate = NULL;
 351         if (ddi_soft_state_bystr_init(&iport->tgt_sstate,
 352             sizeof (pmcs_xscsi_t), PMCS_TGT_SSTATE_SZ) != 0) {
 353                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 354                     "cannot get iport tgt soft state");
 355                 goto iport_attach_fail2;
 356         }
 357 
 358         /* Create this iport's target map */
 359         if (pmcs_iport_tgtmap_create(iport) == B_FALSE) {
 360                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 361                     "Failed to create tgtmap on iport %d", inst);
 362                 goto iport_attach_fail3;
 363         }
 364 
 365         /* Set up the 'initiator-port' DDI property on this iport */
 366         init_port = kmem_zalloc(PMCS_MAX_UA_SIZE, KM_SLEEP);
 367         if (pwp->separate_ports) {
 368                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 369                     "%s: separate ports not supported", __func__);
 370         } else {
 371                 /* Set initiator-port value to the HBA's base WWN */
 372                 (void) scsi_wwn_to_wwnstr(pwp->sas_wwns[0], 1,
 373                     init_port);
 374         }
 375 
 376         mutex_enter(&iport->lock);
 377         pmcs_smhba_add_iport_prop(iport, DATA_TYPE_STRING,
 378             SCSI_ADDR_PROP_INITIATOR_PORT, init_port);
 379         kmem_free(init_port, PMCS_MAX_UA_SIZE);
 380 
 381         /* Set up a 'num-phys' DDI property for the iport node */
 382         pmcs_smhba_add_iport_prop(iport, DATA_TYPE_INT32, PMCS_NUM_PHYS,
 383             &iport->nphy);
 384         mutex_exit(&iport->lock);
 385 
 386         /* Create kstats for each of the phys in this port */
 387         pmcs_create_all_phy_stats(iport);
 388 
 389         /*
 390          * Insert this iport handle into our list and set
 391          * iports_attached on the HBA node.
 392          */
 393         rw_enter(&pwp->iports_lock, RW_WRITER);
 394         ASSERT(!list_link_active(&iport->list_node));
 395         list_insert_tail(&pwp->iports, iport);
 396         pwp->iports_attached = 1;
 397         pwp->num_iports++;
 398         rw_exit(&pwp->iports_lock);
 399 
 400         pmcs_prt(pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
 401             "iport%d attached", inst);
 402         ddi_report_dev(dip);
 403         return (DDI_SUCCESS);
 404 
 405         /* teardown and fail */
 406 iport_attach_fail3:
 407         ddi_soft_state_bystr_fini(&iport->tgt_sstate);
 408 iport_attach_fail2:
 409         list_destroy(&iport->phys);
 410         strfree(iport->ua);
 411         mutex_destroy(&iport->refcnt_lock);
 412         mutex_destroy(&iport->smp_lock);
 413         cv_destroy(&iport->refcnt_cv);
 414         cv_destroy(&iport->smp_cv);
 415         mutex_destroy(&iport->lock);
 416 iport_attach_fail1:
 417         ddi_soft_state_free(pmcs_iport_softstate, inst);
 418         return (DDI_FAILURE);
 419 }
 420 
 421 static int
 422 pmcs_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 423 {
 424         scsi_hba_tran_t *tran;
 425         char chiprev, *fwsupport, hw_rev[24], fw_rev[24];
 426         off_t set3size;
 427         int inst, i;
 428         int sm_hba = 1;
 429         int protocol = 0;
 430         int num_phys = 0;
 431         pmcs_hw_t *pwp;
 432         pmcs_phy_t *phyp;
 433         uint32_t num_threads;
 434         char buf[64];
 435         char *fwl_file;
 436 
 437         switch (cmd) {
 438         case DDI_ATTACH:
 439                 break;
 440 
 441         case DDI_PM_RESUME:
 442         case DDI_RESUME:
 443                 tran = (scsi_hba_tran_t *)ddi_get_driver_private(dip);
 444                 if (!tran) {
 445                         return (DDI_FAILURE);
 446                 }
 447                 /* No DDI_?_RESUME on iport nodes */
 448                 if (scsi_hba_iport_unit_address(dip) != NULL) {
 449                         return (DDI_SUCCESS);
 450                 }
 451                 pwp = TRAN2PMC(tran);
 452                 if (pwp == NULL) {
 453                         return (DDI_FAILURE);
 454                 }
 455 
 456                 mutex_enter(&pwp->lock);
 457                 pwp->suspended = 0;
 458                 if (pwp->tq) {
 459                         ddi_taskq_resume(pwp->tq);
 460                 }
 461                 mutex_exit(&pwp->lock);
 462                 return (DDI_SUCCESS);
 463 
 464         default:
 465                 return (DDI_FAILURE);
 466         }
 467 
 468         /*
 469          * If this is an iport node, invoke iport attach.
 470          */
 471         if (scsi_hba_iport_unit_address(dip) != NULL) {
 472                 return (pmcs_iport_attach(dip));
 473         }
 474 
 475         /*
 476          * From here on is attach for the HBA node
 477          */
 478 
 479 #ifdef  DEBUG
 480         /*
 481          * Check to see if this unit is to be disabled.  We can't disable
 482          * on a per-iport node.  It's either the entire HBA or nothing.
 483          */
 484         (void) snprintf(buf, sizeof (buf),
 485             "disable-instance-%d", ddi_get_instance(dip));
 486         if (ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 487             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, buf, 0)) {
 488                 cmn_err(CE_NOTE, "pmcs%d: disabled by configuration",
 489                     ddi_get_instance(dip));
 490                 return (DDI_FAILURE);
 491         }
 492 #endif
 493 
 494         /*
 495          * Allocate softstate
 496          */
 497         inst = ddi_get_instance(dip);
 498         if (ddi_soft_state_zalloc(pmcs_softc_state, inst) != DDI_SUCCESS) {
 499                 cmn_err(CE_WARN, "pmcs%d: Failed to alloc soft state", inst);
 500                 return (DDI_FAILURE);
 501         }
 502 
 503         pwp = ddi_get_soft_state(pmcs_softc_state, inst);
 504         if (pwp == NULL) {
 505                 cmn_err(CE_WARN, "pmcs%d: cannot get soft state", inst);
 506                 ddi_soft_state_free(pmcs_softc_state, inst);
 507                 return (DDI_FAILURE);
 508         }
 509         pwp->dip = dip;
 510         STAILQ_INIT(&pwp->dq);
 511         STAILQ_INIT(&pwp->cq);
 512         STAILQ_INIT(&pwp->wf);
 513         STAILQ_INIT(&pwp->pf);
 514 
 515         /*
 516          * Create the list for iports and init its lock.
 517          */
 518         list_create(&pwp->iports, sizeof (pmcs_iport_t),
 519             offsetof(pmcs_iport_t, list_node));
 520         rw_init(&pwp->iports_lock, NULL, RW_DRIVER, NULL);
 521 
 522         pwp->state = STATE_PROBING;
 523 
 524         /*
 525          * Get driver.conf properties
 526          */
 527         pwp->debug_mask = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 528             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-debug-mask",
 529             debug_mask);
 530         pwp->phyid_block_mask = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 531             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-phyid-block-mask",
 532             block_mask);
 533         pwp->physpeed = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 534             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-physpeed", physpeed);
 535         pwp->phymode = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 536             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-phymode", phymode);
 537         pwp->fwlog = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 538             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-fwlog", fwlog_level);
 539         if (pwp->fwlog > PMCS_FWLOG_MAX) {
 540                 pwp->fwlog = PMCS_FWLOG_MAX;
 541         }
 542         if ((ddi_prop_lookup_string(DDI_DEV_T_ANY, dip, 0, "pmcs-fwlogfile",
 543             &fwl_file) == DDI_SUCCESS)) {
 544                 if (snprintf(pwp->fwlogfile_aap1, MAXPATHLEN, "%s%d-aap1.0",
 545                     fwl_file, ddi_get_instance(dip)) > MAXPATHLEN) {
 546                         pwp->fwlogfile_aap1[0] = '\0';
 547                         pwp->fwlogfile_iop[0] = '\0';
 548                 } else if (snprintf(pwp->fwlogfile_iop, MAXPATHLEN,
 549                     "%s%d-iop.0", fwl_file,
 550                     ddi_get_instance(dip)) > MAXPATHLEN) {
 551                         pwp->fwlogfile_aap1[0] = '\0';
 552                         pwp->fwlogfile_iop[0] = '\0';
 553                 }
 554                 ddi_prop_free(fwl_file);
 555         } else {
 556                 pwp->fwlogfile_aap1[0] = '\0';
 557                 pwp->fwlogfile_iop[0] = '\0';
 558         }
 559 
 560         pwp->open_retry_interval = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 561             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-open-retry-interval",
 562             OPEN_RETRY_INTERVAL_DEF);
 563         if (pwp->open_retry_interval > OPEN_RETRY_INTERVAL_MAX) {
 564                 pwp->open_retry_interval = OPEN_RETRY_INTERVAL_MAX;
 565         }
 566 
 567         mutex_enter(&pmcs_trace_lock);
 568         if (pmcs_tbuf == NULL) {
 569                 /* Allocate trace buffer */
 570                 pmcs_tbuf_num_elems = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 571                     DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-tbuf-num-elems",
 572                     PMCS_TBUF_NUM_ELEMS_DEF);
 573                 if ((pmcs_tbuf_num_elems == DDI_PROP_NOT_FOUND) ||
 574                     (pmcs_tbuf_num_elems == 0)) {
 575                         pmcs_tbuf_num_elems = PMCS_TBUF_NUM_ELEMS_DEF;
 576                 }
 577 
 578                 pmcs_tbuf = kmem_zalloc(pmcs_tbuf_num_elems *
 579                     sizeof (pmcs_tbuf_t), KM_SLEEP);
 580                 pmcs_tbuf_ptr = pmcs_tbuf;
 581                 pmcs_tbuf_idx = 0;
 582         }
 583         mutex_exit(&pmcs_trace_lock);
 584 
 585         if (pwp->fwlog && strlen(pwp->fwlogfile_aap1) > 0) {
 586                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 587                     "%s: firmware event log files: %s, %s", __func__,
 588                     pwp->fwlogfile_aap1, pwp->fwlogfile_iop);
 589                 pwp->fwlog_file = 1;
 590         } else {
 591                 if (pwp->fwlog == 0) {
 592                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 593                             "%s: No firmware event log will be written "
 594                             "(event log disabled)", __func__);
 595                 } else {
 596                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 597                             "%s: No firmware event log will be written "
 598                             "(no filename configured - too long?)", __func__);
 599                 }
 600                 pwp->fwlog_file = 0;
 601         }
 602 
 603         disable_msix = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 604             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-disable-msix",
 605             disable_msix);
 606         disable_msi = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 607             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-disable-msi",
 608             disable_msi);
 609         maxqdepth = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 610             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-maxqdepth", maxqdepth);
 611         pwp->fw_force_update = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 612             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-fw-force-update", 0);
 613         if (pwp->fw_force_update == 0) {
 614                 pwp->fw_disable_update = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 615                     DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
 616                     "pmcs-fw-disable-update", 0);
 617         }
 618         pwp->ioq_depth = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 619             DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "pmcs-num-io-qentries",
 620             PMCS_NQENTRY);
 621 
 622         /*
 623          * Initialize FMA
 624          */
 625         pwp->dev_acc_attr = pwp->reg_acc_attr = rattr;
 626         pwp->iqp_dma_attr = pwp->oqp_dma_attr =
 627             pwp->regdump_dma_attr = pwp->cip_dma_attr =
 628             pwp->fwlog_dma_attr = pmcs_dattr;
 629         pwp->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, pwp->dip,
 630             DDI_PROP_NOTPROM | DDI_PROP_DONTPASS, "fm-capable",
 631             DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
 632             DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
 633         pmcs_fm_init(pwp);
 634 
 635         /*
 636          * Map registers
 637          */
 638         if (pci_config_setup(dip, &pwp->pci_acc_handle)) {
 639                 pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
 640                     "pci config setup failed");
 641                 ddi_soft_state_free(pmcs_softc_state, inst);
 642                 return (DDI_FAILURE);
 643         }
 644 
 645         /*
 646          * Get the size of register set 3.
 647          */
 648         if (ddi_dev_regsize(dip, PMCS_REGSET_3, &set3size) != DDI_SUCCESS) {
 649                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 650                     "unable to get size of register set %d", PMCS_REGSET_3);
 651                 pci_config_teardown(&pwp->pci_acc_handle);
 652                 ddi_soft_state_free(pmcs_softc_state, inst);
 653                 return (DDI_FAILURE);
 654         }
 655 
 656         /*
 657          * Map registers
 658          */
 659         pwp->reg_acc_attr.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC;
 660 
 661         if (ddi_regs_map_setup(dip, PMCS_REGSET_0, (caddr_t *)&pwp->msg_regs,
 662             0, 0, &pwp->reg_acc_attr, &pwp->msg_acc_handle)) {
 663                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 664                     "failed to map Message Unit registers");
 665                 pci_config_teardown(&pwp->pci_acc_handle);
 666                 ddi_soft_state_free(pmcs_softc_state, inst);
 667                 return (DDI_FAILURE);
 668         }
 669 
 670         if (ddi_regs_map_setup(dip, PMCS_REGSET_1, (caddr_t *)&pwp->top_regs,
 671             0, 0, &pwp->reg_acc_attr, &pwp->top_acc_handle)) {
 672                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 673                     "failed to map TOP registers");
 674                 ddi_regs_map_free(&pwp->msg_acc_handle);
 675                 pci_config_teardown(&pwp->pci_acc_handle);
 676                 ddi_soft_state_free(pmcs_softc_state, inst);
 677                 return (DDI_FAILURE);
 678         }
 679 
 680         if (ddi_regs_map_setup(dip, PMCS_REGSET_2, (caddr_t *)&pwp->gsm_regs,
 681             0, 0, &pwp->reg_acc_attr, &pwp->gsm_acc_handle)) {
 682                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 683                     "failed to map GSM registers");
 684                 ddi_regs_map_free(&pwp->top_acc_handle);
 685                 ddi_regs_map_free(&pwp->msg_acc_handle);
 686                 pci_config_teardown(&pwp->pci_acc_handle);
 687                 ddi_soft_state_free(pmcs_softc_state, inst);
 688                 return (DDI_FAILURE);
 689         }
 690 
 691         if (ddi_regs_map_setup(dip, PMCS_REGSET_3, (caddr_t *)&pwp->mpi_regs,
 692             0, 0, &pwp->reg_acc_attr, &pwp->mpi_acc_handle)) {
 693                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 694                     "failed to map MPI registers");
 695                 ddi_regs_map_free(&pwp->top_acc_handle);
 696                 ddi_regs_map_free(&pwp->gsm_acc_handle);
 697                 ddi_regs_map_free(&pwp->msg_acc_handle);
 698                 pci_config_teardown(&pwp->pci_acc_handle);
 699                 ddi_soft_state_free(pmcs_softc_state, inst);
 700                 return (DDI_FAILURE);
 701         }
 702         pwp->mpibar =
 703             (((5U << 2) + 0x10) << PMCS_MSGU_MPI_BAR_SHIFT) | set3size;
 704 
 705         /*
 706          * Make sure we can support this card.
 707          */
 708         pwp->chiprev = pmcs_rd_topunit(pwp, PMCS_DEVICE_REVISION);
 709 
 710         switch (pwp->chiprev) {
 711         case PMCS_PM8001_REV_A:
 712         case PMCS_PM8001_REV_B:
 713                 pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
 714                     "Rev A/B Card no longer supported");
 715                 goto failure;
 716         case PMCS_PM8001_REV_C:
 717                 break;
 718         default:
 719                 pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
 720                     "Unknown chip revision (%d)", pwp->chiprev);
 721                 goto failure;
 722         }
 723 
 724         /*
 725          * Allocate DMA addressable area for Inbound and Outbound Queue indices
 726          * that the chip needs to access plus a space for scratch usage
 727          */
 728         pwp->cip_dma_attr.dma_attr_align = sizeof (uint32_t);
 729         if (pmcs_dma_setup(pwp, &pwp->cip_dma_attr, &pwp->cip_acchdls,
 730             &pwp->cip_handles, ptob(1), (caddr_t *)&pwp->cip,
 731             &pwp->ciaddr) == B_FALSE) {
 732                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 733                     "Failed to setup DMA for index/scratch");
 734                 goto failure;
 735         }
 736 
 737         bzero(pwp->cip, ptob(1));
 738         pwp->scratch = &pwp->cip[PMCS_INDICES_SIZE];
 739         pwp->scratch_dma = pwp->ciaddr + PMCS_INDICES_SIZE;
 740 
 741         /*
 742          * Allocate DMA S/G list chunks
 743          */
 744         (void) pmcs_add_more_chunks(pwp, ptob(1) * PMCS_MIN_CHUNK_PAGES);
 745 
 746         /*
 747          * Allocate a DMA addressable area for the firmware log (if needed)
 748          */
 749         if (pwp->fwlog) {
 750                 /*
 751                  * Align to event log header and entry size
 752                  */
 753                 pwp->fwlog_dma_attr.dma_attr_align = 32;
 754                 if (pmcs_dma_setup(pwp, &pwp->fwlog_dma_attr,
 755                     &pwp->fwlog_acchdl,
 756                     &pwp->fwlog_hndl, PMCS_FWLOG_SIZE,
 757                     (caddr_t *)&pwp->fwlogp,
 758                     &pwp->fwaddr) == B_FALSE) {
 759                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 760                             "Failed to setup DMA for fwlog area");
 761                         pwp->fwlog = 0;
 762                 } else {
 763                         bzero(pwp->fwlogp, PMCS_FWLOG_SIZE);
 764                         pwp->fwlogp_aap1 = (pmcs_fw_event_hdr_t *)pwp->fwlogp;
 765                         pwp->fwlogp_iop = (pmcs_fw_event_hdr_t *)((void *)
 766                             ((caddr_t)pwp->fwlogp + (PMCS_FWLOG_SIZE / 2)));
 767                 }
 768         }
 769 
 770         if (pwp->flash_chunk_addr == NULL) {
 771                 pwp->regdump_dma_attr.dma_attr_align = PMCS_FLASH_CHUNK_SIZE;
 772                 if (pmcs_dma_setup(pwp, &pwp->regdump_dma_attr,
 773                     &pwp->regdump_acchdl,
 774                     &pwp->regdump_hndl, PMCS_FLASH_CHUNK_SIZE,
 775                     (caddr_t *)&pwp->flash_chunkp, &pwp->flash_chunk_addr) ==
 776                     B_FALSE) {
 777                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 778                             "Failed to setup DMA for register dump area");
 779                         goto failure;
 780                 }
 781                 bzero(pwp->flash_chunkp, PMCS_FLASH_CHUNK_SIZE);
 782         }
 783 
 784         /*
 785          * More bits of local initialization...
 786          */
 787         pwp->tq = ddi_taskq_create(dip, "_tq", 4, TASKQ_DEFAULTPRI, 0);
 788         if (pwp->tq == NULL) {
 789                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 790                     "unable to create worker taskq");
 791                 goto failure;
 792         }
 793 
 794         /*
 795          * Cache of structures for dealing with I/O completion callbacks.
 796          */
 797         (void) snprintf(buf, sizeof (buf), "pmcs_iocomp_cb_cache%d", inst);
 798         pwp->iocomp_cb_cache = kmem_cache_create(buf,
 799             sizeof (pmcs_iocomp_cb_t), 16, NULL, NULL, NULL, NULL, NULL, 0);
 800 
 801         /*
 802          * Cache of PHY structures
 803          */
 804         (void) snprintf(buf, sizeof (buf), "pmcs_phy_cache%d", inst);
 805         pwp->phy_cache = kmem_cache_create(buf, sizeof (pmcs_phy_t), 8,
 806             pmcs_phy_constructor, pmcs_phy_destructor, NULL, (void *)pwp,
 807             NULL, 0);
 808 
 809         /*
 810          * Allocate space for the I/O completion threads
 811          */
 812         num_threads = ncpus_online;
 813         if (num_threads > PMCS_MAX_CQ_THREADS) {
 814                 num_threads = PMCS_MAX_CQ_THREADS;
 815         }
 816 
 817         pwp->cq_info.cq_threads = num_threads;
 818         pwp->cq_info.cq_thr_info = kmem_zalloc(
 819             sizeof (pmcs_cq_thr_info_t) * pwp->cq_info.cq_threads, KM_SLEEP);
 820         pwp->cq_info.cq_next_disp_thr = 0;
 821         pwp->cq_info.cq_stop = B_FALSE;
 822 
 823         /*
 824          * Set the quantum value in clock ticks for the I/O interrupt
 825          * coalescing timer.
 826          */
 827         pwp->io_intr_coal.quantum = drv_usectohz(PMCS_QUANTUM_TIME_USECS);
 828 
 829         /*
 830          * We have a delicate dance here. We need to set up
 831          * interrupts so we know how to set up some OQC
 832          * tables. However, while we're setting up table
 833          * access, we may need to flash new firmware and
 834          * reset the card, which will take some finessing.
 835          */
 836 
 837         /*
 838          * Set up interrupts here.
 839          */
 840         switch (pmcs_setup_intr(pwp)) {
 841         case 0:
 842                 break;
 843         case EIO:
 844                 pwp->stuck = 1;
 845                 /* FALLTHROUGH */
 846         default:
 847                 goto failure;
 848         }
 849 
 850         /*
 851          * Set these up now becuase they are used to initialize the OQC tables.
 852          *
 853          * If we have MSI or MSI-X interrupts set up and we have enough
 854          * vectors for each OQ, the Outbound Queue vectors can all be the
 855          * same as the appropriate interrupt routine will have been called
 856          * and the doorbell register automatically cleared.
 857          * This keeps us from having to check the Outbound Doorbell register
 858          * when the routines for these interrupts are called.
 859          *
 860          * If we have Legacy INT-X interrupts set up or we didn't have enough
 861          * MSI/MSI-X vectors to uniquely identify each OQ, we point these
 862          * vectors to the bits we would like to have set in the Outbound
 863          * Doorbell register because pmcs_all_intr will read the doorbell
 864          * register to find out why we have an interrupt and write the
 865          * corresponding 'clear' bit for that interrupt.
 866          */
 867 
 868         switch (pwp->intr_cnt) {
 869         case 1:
 870                 /*
 871                  * Only one vector, so we must check all OQs for MSI.  For
 872                  * INT-X, there's only one vector anyway, so we can just
 873                  * use the outbound queue bits to keep from having to
 874                  * check each queue for each interrupt.
 875                  */
 876                 if (pwp->int_type == PMCS_INT_FIXED) {
 877                         pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
 878                         pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_GENERAL;
 879                         pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_EVENTS;
 880                 } else {
 881                         pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
 882                         pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_IODONE;
 883                         pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_IODONE;
 884                 }
 885                 break;
 886         case 2:
 887                 /* With 2, we can at least isolate IODONE */
 888                 pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
 889                 pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_GENERAL;
 890                 pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_GENERAL;
 891                 break;
 892         case 4:
 893                 /* With 4 vectors, everybody gets one */
 894                 pwp->oqvec[PMCS_OQ_IODONE] = PMCS_OQ_IODONE;
 895                 pwp->oqvec[PMCS_OQ_GENERAL] = PMCS_OQ_GENERAL;
 896                 pwp->oqvec[PMCS_OQ_EVENTS] = PMCS_OQ_EVENTS;
 897                 break;
 898         }
 899 
 900         /*
 901          * Do the first part of setup
 902          */
 903         if (pmcs_setup(pwp)) {
 904                 goto failure;
 905         }
 906         pmcs_report_fwversion(pwp);
 907 
 908         /*
 909          * Now do some additonal allocations based upon information
 910          * gathered during MPI setup.
 911          */
 912         pwp->root_phys = kmem_zalloc(pwp->nphy * sizeof (pmcs_phy_t), KM_SLEEP);
 913         ASSERT(pwp->nphy < SAS2_PHYNUM_MAX);
 914         phyp = pwp->root_phys;
 915         for (i = 0; i < pwp->nphy; i++) {
 916                 if (i < pwp->nphy-1) {
 917                         phyp->sibling = (phyp + 1);
 918                 }
 919                 mutex_init(&phyp->phy_lock, NULL, MUTEX_DRIVER,
 920                     DDI_INTR_PRI(pwp->intr_pri));
 921                 phyp->phynum = i & SAS2_PHYNUM_MASK;
 922                 pmcs_phy_name(pwp, phyp, phyp->path, sizeof (phyp->path));
 923                 phyp->pwp = pwp;
 924                 phyp->device_id = PMCS_INVALID_DEVICE_ID;
 925                 phyp->portid = PMCS_PHY_INVALID_PORT_ID;
 926                 phyp++;
 927         }
 928 
 929         pwp->work = kmem_zalloc(pwp->max_cmd * sizeof (pmcwork_t), KM_SLEEP);
 930         for (i = 0; i < pwp->max_cmd; i++) {
 931                 pmcwork_t *pwrk = &pwp->work[i];
 932                 mutex_init(&pwrk->lock, NULL, MUTEX_DRIVER,
 933                     DDI_INTR_PRI(pwp->intr_pri));
 934                 cv_init(&pwrk->sleep_cv, NULL, CV_DRIVER, NULL);
 935                 STAILQ_INSERT_TAIL(&pwp->wf, pwrk, next);
 936 
 937         }
 938         pwp->targets = (pmcs_xscsi_t **)
 939             kmem_zalloc(pwp->max_dev * sizeof (pmcs_xscsi_t *), KM_SLEEP);
 940 
 941         pwp->iqpt = (pmcs_iqp_trace_t *)
 942             kmem_zalloc(sizeof (pmcs_iqp_trace_t), KM_SLEEP);
 943         pwp->iqpt->head = kmem_zalloc(PMCS_IQP_TRACE_BUFFER_SIZE, KM_SLEEP);
 944         pwp->iqpt->curpos = pwp->iqpt->head;
 945         pwp->iqpt->size_left = PMCS_IQP_TRACE_BUFFER_SIZE;
 946 
 947         /*
 948          * Start MPI communication.
 949          */
 950         if (pmcs_start_mpi(pwp)) {
 951                 if (pmcs_soft_reset(pwp, B_FALSE)) {
 952                         goto failure;
 953                 }
 954                 pwp->last_reset_reason = PMCS_LAST_RST_ATTACH;
 955         }
 956 
 957         /*
 958          * Do some initial acceptance tests.
 959          * This tests interrupts and queues.
 960          */
 961         if (pmcs_echo_test(pwp)) {
 962                 goto failure;
 963         }
 964 
 965         /* Read VPD - if it exists */
 966         if (pmcs_get_nvmd(pwp, PMCS_NVMD_VPD, PMCIN_NVMD_VPD, 0, NULL, 0)) {
 967                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
 968                     "%s: Unable to read VPD: "
 969                     "attempting to fabricate", __func__);
 970                 /*
 971                  * When we release, this must goto failure and the call
 972                  * to pmcs_fabricate_wwid is removed.
 973                  */
 974                 /* goto failure; */
 975                 if (!pmcs_fabricate_wwid(pwp)) {
 976                         goto failure;
 977                 }
 978         }
 979 
 980         /*
 981          * We're now officially running
 982          */
 983         pwp->state = STATE_RUNNING;
 984 
 985         /*
 986          * Check firmware versions and load new firmware
 987          * if needed and reset.
 988          */
 989         if (pmcs_firmware_update(pwp)) {
 990                 pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
 991                     "%s: Firmware update failed", __func__);
 992                 goto failure;
 993         }
 994 
 995         /*
 996          * Create completion threads.
 997          */
 998         for (i = 0; i < pwp->cq_info.cq_threads; i++) {
 999                 pwp->cq_info.cq_thr_info[i].cq_pwp = pwp;
1000                 pwp->cq_info.cq_thr_info[i].cq_thread =
1001                     thread_create(NULL, 0, pmcs_scsa_cq_run,
1002                     &pwp->cq_info.cq_thr_info[i], 0, &p0, TS_RUN, minclsyspri);
1003         }
1004 
1005         /*
1006          * Create one thread to deal with the updating of the interrupt
1007          * coalescing timer.
1008          */
1009         pwp->ict_thread = thread_create(NULL, 0, pmcs_check_intr_coal,
1010             pwp, 0, &p0, TS_RUN, minclsyspri);
1011 
1012         /*
1013          * Kick off the watchdog
1014          */
1015         pwp->wdhandle = timeout(pmcs_watchdog, pwp,
1016             drv_usectohz(PMCS_WATCH_INTERVAL));
1017         /*
1018          * Do the SCSI attachment code (before starting phys)
1019          */
1020         if (pmcs_scsa_init(pwp, &pmcs_dattr)) {
1021                 goto failure;
1022         }
1023         pwp->hba_attached = 1;
1024 
1025         /* Check all acc & dma handles allocated in attach */
1026         if (pmcs_check_acc_dma_handle(pwp)) {
1027                 ddi_fm_service_impact(pwp->dip, DDI_SERVICE_LOST);
1028                 goto failure;
1029         }
1030 
1031         /*
1032          * Create the iportmap for this HBA instance
1033          */
1034         if (scsi_hba_iportmap_create(dip, iportmap_csync_usec,
1035             iportmap_stable_usec, &pwp->hss_iportmap) != DDI_SUCCESS) {
1036                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1037                     "%s: pmcs%d iportmap_create failed", __func__, inst);
1038                 goto failure;
1039         }
1040         ASSERT(pwp->hss_iportmap);
1041 
1042         /*
1043          * Create the phymap for this HBA instance
1044          */
1045         if (sas_phymap_create(dip, phymap_stable_usec, PHYMAP_MODE_SIMPLE, NULL,
1046             pwp, pmcs_phymap_activate, pmcs_phymap_deactivate,
1047             &pwp->hss_phymap) != DDI_SUCCESS) {
1048                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1049                     "%s: pmcs%d phymap_create failed", __func__, inst);
1050                 goto failure;
1051         }
1052         ASSERT(pwp->hss_phymap);
1053 
1054         /*
1055          * Start the PHYs.
1056          */
1057         if (pmcs_start_phys(pwp)) {
1058                 goto failure;
1059         }
1060 
1061         /*
1062          * From this point on, we can't fail.
1063          */
1064         ddi_report_dev(dip);
1065 
1066         /* SM-HBA */
1067         pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_INT32, PMCS_SMHBA_SUPPORTED,
1068             &sm_hba);
1069 
1070         /* SM-HBA */
1071         pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_STRING, PMCS_DRV_VERSION,
1072             pmcs_driver_rev);
1073 
1074         /* SM-HBA */
1075         chiprev = 'A' + pwp->chiprev;
1076         (void) snprintf(hw_rev, 2, "%s", &chiprev);
1077         pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_STRING, PMCS_HWARE_VERSION,
1078             hw_rev);
1079 
1080         /* SM-HBA */
1081         switch (PMCS_FW_TYPE(pwp)) {
1082         case PMCS_FW_TYPE_RELEASED:
1083                 fwsupport = "Released";
1084                 break;
1085         case PMCS_FW_TYPE_DEVELOPMENT:
1086                 fwsupport = "Development";
1087                 break;
1088         case PMCS_FW_TYPE_ALPHA:
1089                 fwsupport = "Alpha";
1090                 break;
1091         case PMCS_FW_TYPE_BETA:
1092                 fwsupport = "Beta";
1093                 break;
1094         default:
1095                 fwsupport = "Special";
1096                 break;
1097         }
1098         (void) snprintf(fw_rev, sizeof (fw_rev), "%x.%x.%x %s",
1099             PMCS_FW_MAJOR(pwp), PMCS_FW_MINOR(pwp), PMCS_FW_MICRO(pwp),
1100             fwsupport);
1101         pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_STRING, PMCS_FWARE_VERSION,
1102             fw_rev);
1103 
1104         /* SM-HBA */
1105         num_phys = pwp->nphy;
1106         pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_INT32, PMCS_NUM_PHYS_HBA,
1107             &num_phys);
1108 
1109         /* SM-HBA */
1110         protocol = SAS_SSP_SUPPORT | SAS_SATA_SUPPORT | SAS_SMP_SUPPORT;
1111         pmcs_smhba_add_hba_prop(pwp, DATA_TYPE_INT32, PMCS_SUPPORTED_PROTOCOL,
1112             &protocol);
1113 
1114         /* Receptacle properties (FMA) */
1115         pwp->recept_labels[0] = PMCS_RECEPT_LABEL_0;
1116         pwp->recept_pm[0] = PMCS_RECEPT_PM_0;
1117         pwp->recept_labels[1] = PMCS_RECEPT_LABEL_1;
1118         pwp->recept_pm[1] = PMCS_RECEPT_PM_1;
1119         if (ddi_prop_update_string_array(DDI_DEV_T_NONE, dip,
1120             SCSI_HBA_PROP_RECEPTACLE_LABEL, &pwp->recept_labels[0],
1121             PMCS_NUM_RECEPTACLES) != DDI_PROP_SUCCESS) {
1122                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1123                     "%s: failed to create %s property", __func__,
1124                     "receptacle-label");
1125         }
1126         if (ddi_prop_update_string_array(DDI_DEV_T_NONE, dip,
1127             SCSI_HBA_PROP_RECEPTACLE_PM, &pwp->recept_pm[0],
1128             PMCS_NUM_RECEPTACLES) != DDI_PROP_SUCCESS) {
1129                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1130                     "%s: failed to create %s property", __func__,
1131                     "receptacle-pm");
1132         }
1133 
1134         return (DDI_SUCCESS);
1135 
1136 failure:
1137         if (pmcs_unattach(pwp)) {
1138                 pwp->stuck = 1;
1139         }
1140         return (DDI_FAILURE);
1141 }
1142 
1143 int
1144 pmcs_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1145 {
1146         int inst = ddi_get_instance(dip);
1147         pmcs_iport_t    *iport = NULL;
1148         pmcs_hw_t       *pwp = NULL;
1149         scsi_hba_tran_t *tran;
1150 
1151         if (scsi_hba_iport_unit_address(dip) != NULL) {
1152                 /* iport node */
1153                 iport = ddi_get_soft_state(pmcs_iport_softstate, inst);
1154                 ASSERT(iport);
1155                 if (iport == NULL) {
1156                         return (DDI_FAILURE);
1157                 }
1158                 pwp = iport->pwp;
1159         } else {
1160                 /* hba node */
1161                 pwp = (pmcs_hw_t *)ddi_get_soft_state(pmcs_softc_state, inst);
1162                 ASSERT(pwp);
1163                 if (pwp == NULL) {
1164                         return (DDI_FAILURE);
1165                 }
1166         }
1167         switch (cmd) {
1168         case DDI_DETACH:
1169                 if (iport) {
1170                         /* iport detach */
1171                         if (pmcs_iport_unattach(iport)) {
1172                                 return (DDI_FAILURE);
1173                         }
1174                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1175                             "iport%d detached", inst);
1176                         return (DDI_SUCCESS);
1177                 } else {
1178                         /* HBA detach */
1179                         if (pmcs_unattach(pwp)) {
1180                                 return (DDI_FAILURE);
1181                         }
1182                         return (DDI_SUCCESS);
1183                 }
1184 
1185         case DDI_SUSPEND:
1186         case DDI_PM_SUSPEND:
1187                 /* No DDI_SUSPEND on iport nodes */
1188                 if (iport) {
1189                         return (DDI_SUCCESS);
1190                 }
1191 
1192                 if (pwp->stuck) {
1193                         return (DDI_FAILURE);
1194                 }
1195                 tran = (scsi_hba_tran_t *)ddi_get_driver_private(dip);
1196                 if (!tran) {
1197                         return (DDI_FAILURE);
1198                 }
1199 
1200                 pwp = TRAN2PMC(tran);
1201                 if (pwp == NULL) {
1202                         return (DDI_FAILURE);
1203                 }
1204                 mutex_enter(&pwp->lock);
1205                 if (pwp->tq) {
1206                         ddi_taskq_suspend(pwp->tq);
1207                 }
1208                 pwp->suspended = 1;
1209                 mutex_exit(&pwp->lock);
1210                 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "PMC8X6G suspending");
1211                 return (DDI_SUCCESS);
1212 
1213         default:
1214                 return (DDI_FAILURE);
1215         }
1216 }
1217 
1218 static int
1219 pmcs_iport_unattach(pmcs_iport_t *iport)
1220 {
1221         pmcs_hw_t       *pwp = iport->pwp;
1222 
1223         /*
1224          * First, check if there are still any configured targets on this
1225          * iport.  If so, we fail detach.
1226          */
1227         if (pmcs_iport_has_targets(pwp, iport)) {
1228                 pmcs_prt(pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
1229                     "iport%d detach failure: iport has targets (luns)",
1230                     ddi_get_instance(iport->dip));
1231                 return (DDI_FAILURE);
1232         }
1233 
1234         /*
1235          * Remove this iport from our list if it is inactive in the phymap.
1236          */
1237         rw_enter(&pwp->iports_lock, RW_WRITER);
1238         mutex_enter(&iport->lock);
1239 
1240         if ((iport->ua_state == UA_ACTIVE) &&
1241             (detach_with_active_port == 0)) {
1242                 mutex_exit(&iport->lock);
1243                 rw_exit(&pwp->iports_lock);
1244                 pmcs_prt(pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
1245                     "iport%d detach failure: "
1246                     "iport unit address active in phymap",
1247                     ddi_get_instance(iport->dip));
1248                 return (DDI_FAILURE);
1249         }
1250 
1251         /* If it's our only iport, clear iports_attached */
1252         ASSERT(pwp->num_iports >= 1);
1253         if (--pwp->num_iports == 0) {
1254                 pwp->iports_attached = 0;
1255         }
1256 
1257         ASSERT(list_link_active(&iport->list_node));
1258         list_remove(&pwp->iports, iport);
1259         rw_exit(&pwp->iports_lock);
1260 
1261         /*
1262          * We have removed the iport handle from the HBA's iports list,
1263          * there will be no new references to it. Two things must be
1264          * guarded against here.  First, we could have PHY up events,
1265          * adding themselves to the iport->phys list and grabbing ref's
1266          * on our iport handle.  Second, we could have existing references
1267          * to this iport handle from a point in time prior to the list
1268          * removal above.
1269          *
1270          * So first, destroy the phys list. Remove any phys that have snuck
1271          * in after the phymap deactivate, dropping the refcnt accordingly.
1272          * If these PHYs are still up if and when the phymap reactivates
1273          * (i.e. when this iport reattaches), we'll populate the list with
1274          * them and bump the refcnt back up.
1275          */
1276         pmcs_remove_phy_from_iport(iport, NULL);
1277         ASSERT(list_is_empty(&iport->phys));
1278         list_destroy(&iport->phys);
1279         mutex_exit(&iport->lock);
1280 
1281         /*
1282          * Second, wait for any other references to this iport to be
1283          * dropped, then continue teardown.
1284          */
1285         mutex_enter(&iport->refcnt_lock);
1286         while (iport->refcnt != 0) {
1287                 cv_wait(&iport->refcnt_cv, &iport->refcnt_lock);
1288         }
1289         mutex_exit(&iport->refcnt_lock);
1290 
1291 
1292         /* Destroy the iport target map */
1293         if (pmcs_iport_tgtmap_destroy(iport) == B_FALSE) {
1294                 return (DDI_FAILURE);
1295         }
1296 
1297         /* Free the tgt soft state */
1298         if (iport->tgt_sstate != NULL) {
1299                 ddi_soft_state_bystr_fini(&iport->tgt_sstate);
1300         }
1301 
1302         /* Free our unit address string */
1303         strfree(iport->ua);
1304 
1305         /* Finish teardown and free the softstate */
1306         mutex_destroy(&iport->refcnt_lock);
1307         mutex_destroy(&iport->smp_lock);
1308         ASSERT(iport->refcnt == 0);
1309         cv_destroy(&iport->refcnt_cv);
1310         cv_destroy(&iport->smp_cv);
1311         mutex_destroy(&iport->lock);
1312         ddi_soft_state_free(pmcs_iport_softstate, ddi_get_instance(iport->dip));
1313 
1314         return (DDI_SUCCESS);
1315 }
1316 
1317 static int
1318 pmcs_unattach(pmcs_hw_t *pwp)
1319 {
1320         int i;
1321         enum pwpstate curstate;
1322         pmcs_cq_thr_info_t *cqti;
1323 
1324         /*
1325          * Tear down the interrupt infrastructure.
1326          */
1327         if (pmcs_teardown_intr(pwp)) {
1328                 pwp->stuck = 1;
1329         }
1330         pwp->intr_cnt = 0;
1331 
1332         /*
1333          * Grab a lock, if initted, to set state.
1334          */
1335         if (pwp->locks_initted) {
1336                 mutex_enter(&pwp->lock);
1337                 if (pwp->state != STATE_DEAD) {
1338                         pwp->state = STATE_UNPROBING;
1339                 }
1340                 curstate = pwp->state;
1341                 mutex_exit(&pwp->lock);
1342 
1343                 /*
1344                  * Stop the I/O completion threads.
1345                  */
1346                 mutex_enter(&pwp->cq_lock);
1347                 pwp->cq_info.cq_stop = B_TRUE;
1348                 for (i = 0; i < pwp->cq_info.cq_threads; i++) {
1349                         if (pwp->cq_info.cq_thr_info[i].cq_thread) {
1350                                 cqti = &pwp->cq_info.cq_thr_info[i];
1351                                 mutex_enter(&cqti->cq_thr_lock);
1352                                 cv_signal(&cqti->cq_cv);
1353                                 mutex_exit(&cqti->cq_thr_lock);
1354                                 mutex_exit(&pwp->cq_lock);
1355                                 thread_join(cqti->cq_thread->t_did);
1356                                 mutex_enter(&pwp->cq_lock);
1357                         }
1358                 }
1359                 mutex_exit(&pwp->cq_lock);
1360                 kmem_free(pwp->cq_info.cq_thr_info,
1361                     sizeof (pmcs_cq_thr_info_t) * pwp->cq_info.cq_threads);
1362 
1363                 /*
1364                  * Stop the interrupt coalescing timer thread
1365                  */
1366                 if (pwp->ict_thread) {
1367                         mutex_enter(&pwp->ict_lock);
1368                         pwp->io_intr_coal.stop_thread = B_TRUE;
1369                         cv_signal(&pwp->ict_cv);
1370                         mutex_exit(&pwp->ict_lock);
1371                         thread_join(pwp->ict_thread->t_did);
1372                 }
1373         } else {
1374                 if (pwp->state != STATE_DEAD) {
1375                         pwp->state = STATE_UNPROBING;
1376                 }
1377                 curstate = pwp->state;
1378         }
1379 
1380         /*
1381          * Make sure that any pending watchdog won't
1382          * be called from this point on out.
1383          */
1384         (void) untimeout(pwp->wdhandle);
1385         /*
1386          * After the above action, the watchdog
1387          * timer that starts up the worker task
1388          * may trigger but will exit immediately
1389          * on triggering.
1390          *
1391          * Now that this is done, we can destroy
1392          * the task queue, which will wait if we're
1393          * running something on it.
1394          */
1395         if (pwp->tq) {
1396                 ddi_taskq_destroy(pwp->tq);
1397                 pwp->tq = NULL;
1398         }
1399 
1400         pmcs_fm_fini(pwp);
1401 
1402         if (pwp->hba_attached) {
1403                 (void) scsi_hba_detach(pwp->dip);
1404                 pwp->hba_attached = 0;
1405         }
1406 
1407         /*
1408          * If the chip hasn't been marked dead, shut it down now
1409          * to bring it back to a known state without attempting
1410          * a soft reset.
1411          */
1412         if (curstate != STATE_DEAD && pwp->locks_initted) {
1413                 /*
1414                  * De-register all registered devices
1415                  */
1416                 pmcs_deregister_devices(pwp, pwp->root_phys);
1417 
1418                 /*
1419                  * Stop all the phys.
1420                  */
1421                 pmcs_stop_phys(pwp);
1422 
1423                 /*
1424                  * Shut Down Message Passing
1425                  */
1426                 (void) pmcs_stop_mpi(pwp);
1427 
1428                 /*
1429                  * Reset chip
1430                  */
1431                 (void) pmcs_soft_reset(pwp, B_FALSE);
1432                 pwp->last_reset_reason = PMCS_LAST_RST_DETACH;
1433         }
1434 
1435         /*
1436          * Turn off interrupts on the chip
1437          */
1438         if (pwp->mpi_acc_handle) {
1439                 pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_MASK, 0xffffffff);
1440         }
1441 
1442         if (pwp->hss_phymap != NULL) {
1443                 /* Destroy the phymap */
1444                 sas_phymap_destroy(pwp->hss_phymap);
1445         }
1446 
1447         if (pwp->hss_iportmap != NULL) {
1448                 /* Destroy the iportmap */
1449                 scsi_hba_iportmap_destroy(pwp->hss_iportmap);
1450         }
1451 
1452         /* Destroy the iports lock and list */
1453         rw_destroy(&pwp->iports_lock);
1454         ASSERT(list_is_empty(&pwp->iports));
1455         list_destroy(&pwp->iports);
1456 
1457         /*
1458          * Free DMA handles and associated consistent memory
1459          */
1460         if (pwp->regdump_hndl) {
1461                 if (ddi_dma_unbind_handle(pwp->regdump_hndl) != DDI_SUCCESS) {
1462                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1463                             "Condition check failed "
1464                             "at %s():%d", __func__, __LINE__);
1465                 }
1466                 ddi_dma_free_handle(&pwp->regdump_hndl);
1467                 ddi_dma_mem_free(&pwp->regdump_acchdl);
1468                 pwp->regdump_hndl = 0;
1469         }
1470         if (pwp->fwlog_hndl) {
1471                 if (ddi_dma_unbind_handle(pwp->fwlog_hndl) != DDI_SUCCESS) {
1472                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1473                             "Condition check failed "
1474                             "at %s():%d", __func__, __LINE__);
1475                 }
1476                 ddi_dma_free_handle(&pwp->fwlog_hndl);
1477                 ddi_dma_mem_free(&pwp->fwlog_acchdl);
1478                 pwp->fwlog_hndl = 0;
1479         }
1480         if (pwp->cip_handles) {
1481                 if (ddi_dma_unbind_handle(pwp->cip_handles) != DDI_SUCCESS) {
1482                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1483                             "Condition check failed "
1484                             "at %s():%d", __func__, __LINE__);
1485                 }
1486                 ddi_dma_free_handle(&pwp->cip_handles);
1487                 ddi_dma_mem_free(&pwp->cip_acchdls);
1488                 pwp->cip_handles = 0;
1489         }
1490         for (i = 0; i < PMCS_NOQ; i++) {
1491                 if (pwp->oqp_handles[i]) {
1492                         if (ddi_dma_unbind_handle(pwp->oqp_handles[i]) !=
1493                             DDI_SUCCESS) {
1494                                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1495                                     "Condition check failed at %s():%d",
1496                                     __func__, __LINE__);
1497                         }
1498                         ddi_dma_free_handle(&pwp->oqp_handles[i]);
1499                         ddi_dma_mem_free(&pwp->oqp_acchdls[i]);
1500                         pwp->oqp_handles[i] = 0;
1501                 }
1502         }
1503         for (i = 0; i < PMCS_NIQ; i++) {
1504                 if (pwp->iqp_handles[i]) {
1505                         if (ddi_dma_unbind_handle(pwp->iqp_handles[i]) !=
1506                             DDI_SUCCESS) {
1507                                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1508                                     "Condition check failed at %s():%d",
1509                                     __func__, __LINE__);
1510                         }
1511                         ddi_dma_free_handle(&pwp->iqp_handles[i]);
1512                         ddi_dma_mem_free(&pwp->iqp_acchdls[i]);
1513                         pwp->iqp_handles[i] = 0;
1514                 }
1515         }
1516 
1517         pmcs_free_dma_chunklist(pwp);
1518 
1519         /*
1520          * Unmap registers and destroy access handles
1521          */
1522         if (pwp->mpi_acc_handle) {
1523                 ddi_regs_map_free(&pwp->mpi_acc_handle);
1524                 pwp->mpi_acc_handle = 0;
1525         }
1526         if (pwp->top_acc_handle) {
1527                 ddi_regs_map_free(&pwp->top_acc_handle);
1528                 pwp->top_acc_handle = 0;
1529         }
1530         if (pwp->gsm_acc_handle) {
1531                 ddi_regs_map_free(&pwp->gsm_acc_handle);
1532                 pwp->gsm_acc_handle = 0;
1533         }
1534         if (pwp->msg_acc_handle) {
1535                 ddi_regs_map_free(&pwp->msg_acc_handle);
1536                 pwp->msg_acc_handle = 0;
1537         }
1538         if (pwp->pci_acc_handle) {
1539                 pci_config_teardown(&pwp->pci_acc_handle);
1540                 pwp->pci_acc_handle = 0;
1541         }
1542 
1543         /*
1544          * Do memory allocation cleanup.
1545          */
1546         while (pwp->dma_freelist) {
1547                 pmcs_dmachunk_t *this = pwp->dma_freelist;
1548                 pwp->dma_freelist = this->nxt;
1549                 kmem_free(this, sizeof (pmcs_dmachunk_t));
1550         }
1551 
1552         /*
1553          * Free pools
1554          */
1555         if (pwp->iocomp_cb_cache) {
1556                 kmem_cache_destroy(pwp->iocomp_cb_cache);
1557         }
1558 
1559         /*
1560          * Free all PHYs (at level > 0), then free the cache
1561          */
1562         pmcs_free_all_phys(pwp, pwp->root_phys);
1563         if (pwp->phy_cache) {
1564                 kmem_cache_destroy(pwp->phy_cache);
1565         }
1566 
1567         /*
1568          * Free root PHYs
1569          */
1570         if (pwp->root_phys) {
1571                 pmcs_phy_t *phyp = pwp->root_phys;
1572                 for (i = 0; i < pwp->nphy; i++) {
1573                         mutex_destroy(&phyp->phy_lock);
1574                         phyp = phyp->sibling;
1575                 }
1576                 kmem_free(pwp->root_phys, pwp->nphy * sizeof (pmcs_phy_t));
1577                 pwp->root_phys = NULL;
1578                 pwp->nphy = 0;
1579         }
1580 
1581         /* Free the targets list */
1582         if (pwp->targets) {
1583                 kmem_free(pwp->targets,
1584                     sizeof (pmcs_xscsi_t *) * pwp->max_dev);
1585         }
1586 
1587         /*
1588          * Free work structures
1589          */
1590 
1591         if (pwp->work && pwp->max_cmd) {
1592                 for (i = 0; i < pwp->max_cmd; i++) {
1593                         pmcwork_t *pwrk = &pwp->work[i];
1594                         mutex_destroy(&pwrk->lock);
1595                         cv_destroy(&pwrk->sleep_cv);
1596                 }
1597                 kmem_free(pwp->work, sizeof (pmcwork_t) * pwp->max_cmd);
1598                 pwp->work = NULL;
1599                 pwp->max_cmd = 0;
1600         }
1601 
1602         /*
1603          * Do last property and SCSA cleanup
1604          */
1605         if (pwp->smp_tran) {
1606                 smp_hba_tran_free(pwp->smp_tran);
1607                 pwp->smp_tran = NULL;
1608         }
1609         if (pwp->tran) {
1610                 scsi_hba_tran_free(pwp->tran);
1611                 pwp->tran = NULL;
1612         }
1613         if (pwp->reset_notify_listf) {
1614                 scsi_hba_reset_notify_tear_down(pwp->reset_notify_listf);
1615                 pwp->reset_notify_listf = NULL;
1616         }
1617         ddi_prop_remove_all(pwp->dip);
1618         if (pwp->stuck) {
1619                 return (-1);
1620         }
1621 
1622         /* Free register dump area if allocated */
1623         if (pwp->regdumpp) {
1624                 kmem_free(pwp->regdumpp, PMCS_REG_DUMP_SIZE);
1625                 pwp->regdumpp = NULL;
1626         }
1627         if (pwp->iqpt && pwp->iqpt->head) {
1628                 kmem_free(pwp->iqpt->head, PMCS_IQP_TRACE_BUFFER_SIZE);
1629                 pwp->iqpt->head = pwp->iqpt->curpos = NULL;
1630         }
1631         if (pwp->iqpt) {
1632                 kmem_free(pwp->iqpt, sizeof (pmcs_iqp_trace_t));
1633                 pwp->iqpt = NULL;
1634         }
1635 
1636         /* Destroy pwp's lock */
1637         if (pwp->locks_initted) {
1638                 mutex_destroy(&pwp->lock);
1639                 mutex_destroy(&pwp->dma_lock);
1640                 mutex_destroy(&pwp->axil_lock);
1641                 mutex_destroy(&pwp->cq_lock);
1642                 mutex_destroy(&pwp->config_lock);
1643                 mutex_destroy(&pwp->ict_lock);
1644                 mutex_destroy(&pwp->wfree_lock);
1645                 mutex_destroy(&pwp->pfree_lock);
1646                 mutex_destroy(&pwp->dead_phylist_lock);
1647 #ifdef  DEBUG
1648                 mutex_destroy(&pwp->dbglock);
1649 #endif
1650                 cv_destroy(&pwp->config_cv);
1651                 cv_destroy(&pwp->ict_cv);
1652                 cv_destroy(&pwp->drain_cv);
1653                 pwp->locks_initted = 0;
1654         }
1655 
1656         ddi_soft_state_free(pmcs_softc_state, ddi_get_instance(pwp->dip));
1657         return (0);
1658 }
1659 
1660 /*
1661  * quiesce (9E) entry point
1662  *
1663  * This function is called when the system is single-threaded at high PIL
1664  * with preemption disabled. Therefore, the function must not block/wait/sleep.
1665  *
1666  * Returns DDI_SUCCESS or DDI_FAILURE.
1667  *
1668  */
1669 static int
1670 pmcs_quiesce(dev_info_t *dip)
1671 {
1672         pmcs_hw_t       *pwp;
1673         scsi_hba_tran_t *tran;
1674 
1675         if ((tran = ddi_get_driver_private(dip)) == NULL)
1676                 return (DDI_SUCCESS);
1677 
1678         /* No quiesce necessary on a per-iport basis */
1679         if (scsi_hba_iport_unit_address(dip) != NULL) {
1680                 return (DDI_SUCCESS);
1681         }
1682 
1683         if ((pwp = TRAN2PMC(tran)) == NULL)
1684                 return (DDI_SUCCESS);
1685 
1686         /* Stop MPI & Reset chip (no need to re-initialize) */
1687         (void) pmcs_stop_mpi(pwp);
1688         (void) pmcs_soft_reset(pwp, B_TRUE);
1689         pwp->last_reset_reason = PMCS_LAST_RST_QUIESCE;
1690 
1691         return (DDI_SUCCESS);
1692 }
1693 
1694 /*
1695  * Called with xp->statlock and PHY lock and scratch acquired.
1696  */
1697 static int
1698 pmcs_add_sata_device(pmcs_hw_t *pwp, pmcs_xscsi_t *xp)
1699 {
1700         ata_identify_t *ati;
1701         int result, i;
1702         pmcs_phy_t *pptr;
1703         uint16_t *a;
1704         union {
1705                 uint8_t nsa[8];
1706                 uint16_t nsb[4];
1707         } u;
1708 
1709         /*
1710          * Safe defaults - use only if this target is brand new (i.e. doesn't
1711          * already have these settings configured)
1712          */
1713         if (xp->capacity == 0) {
1714                 xp->capacity = (uint64_t)-1;
1715                 xp->ca = 1;
1716                 xp->qdepth = 1;
1717                 xp->pio = 1;
1718         }
1719 
1720         pptr = xp->phy;
1721 
1722         /*
1723          * We only try and issue an IDENTIFY for first level
1724          * (direct attached) devices. We don't try and
1725          * set other quirks here (this will happen later,
1726          * if the device is fully configured)
1727          */
1728         if (pptr->level) {
1729                 return (0);
1730         }
1731 
1732         mutex_exit(&xp->statlock);
1733         result = pmcs_sata_identify(pwp, pptr);
1734         mutex_enter(&xp->statlock);
1735 
1736         if (result) {
1737                 return (result);
1738         }
1739         ati = pwp->scratch;
1740         a = &ati->word108;
1741         for (i = 0; i < 4; i++) {
1742                 u.nsb[i] = ddi_swap16(*a++);
1743         }
1744 
1745         /*
1746          * Check the returned data for being a valid (NAA=5) WWN.
1747          * If so, use that and override the SAS address we were
1748          * given at Link Up time.
1749          */
1750         if ((u.nsa[0] >> 4) == 5) {
1751                 (void) memcpy(pptr->sas_address, u.nsa, 8);
1752         }
1753         pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
1754             "%s: %s has SAS ADDRESS " SAS_ADDR_FMT,
1755             __func__, pptr->path, SAS_ADDR_PRT(pptr->sas_address));
1756         return (0);
1757 }
1758 
1759 /*
1760  * Called with PHY lock and target statlock held and scratch acquired
1761  */
1762 static boolean_t
1763 pmcs_add_new_device(pmcs_hw_t *pwp, pmcs_xscsi_t *target)
1764 {
1765         ASSERT(target != NULL);
1766         pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, target, "%s: target = 0x%p",
1767             __func__, (void *) target);
1768 
1769         switch (target->phy->dtype) {
1770         case SATA:
1771                 if (pmcs_add_sata_device(pwp, target) != 0) {
1772                         pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, target->phy,
1773                             target, "%s: add_sata_device failed for tgt 0x%p",
1774                             __func__, (void *) target);
1775                         return (B_FALSE);
1776                 }
1777                 break;
1778         case SAS:
1779                 target->qdepth = maxqdepth;
1780                 break;
1781         case EXPANDER:
1782                 target->qdepth = 1;
1783                 break;
1784         }
1785 
1786         target->new = 0;
1787         target->assigned = 1;
1788         target->dev_state = PMCS_DEVICE_STATE_OPERATIONAL;
1789         target->dtype = target->phy->dtype;
1790 
1791         /*
1792          * Set the PHY's config stop time to 0.  This is one of the final
1793          * stops along the config path, so we're indicating that we
1794          * successfully configured the PHY.
1795          */
1796         target->phy->config_stop = 0;
1797 
1798         return (B_TRUE);
1799 }
1800 
1801 void
1802 pmcs_worker(void *arg)
1803 {
1804         pmcs_hw_t *pwp = arg;
1805         ulong_t work_flags;
1806 
1807         DTRACE_PROBE2(pmcs__worker, ulong_t, pwp->work_flags, boolean_t,
1808             pwp->config_changed);
1809 
1810         if (pwp->state != STATE_RUNNING) {
1811                 return;
1812         }
1813 
1814         work_flags = atomic_swap_ulong(&pwp->work_flags, 0);
1815 
1816         if (work_flags & PMCS_WORK_FLAG_DUMP_REGS) {
1817                 mutex_enter(&pwp->lock);
1818                 pmcs_register_dump_int(pwp);
1819                 mutex_exit(&pwp->lock);
1820         }
1821 
1822         if (work_flags & PMCS_WORK_FLAG_SAS_HW_ACK) {
1823                 pmcs_ack_events(pwp);
1824         }
1825 
1826         if (work_flags & PMCS_WORK_FLAG_SPINUP_RELEASE) {
1827                 mutex_enter(&pwp->lock);
1828                 pmcs_spinup_release(pwp, NULL);
1829                 mutex_exit(&pwp->lock);
1830         }
1831 
1832         if (work_flags & PMCS_WORK_FLAG_SSP_EVT_RECOVERY) {
1833                 pmcs_ssp_event_recovery(pwp);
1834         }
1835 
1836         if (work_flags & PMCS_WORK_FLAG_DS_ERR_RECOVERY) {
1837                 pmcs_dev_state_recovery(pwp, NULL);
1838         }
1839 
1840         if (work_flags & PMCS_WORK_FLAG_DEREGISTER_DEV) {
1841                 pmcs_deregister_device_work(pwp, NULL);
1842         }
1843 
1844         if (work_flags & PMCS_WORK_FLAG_DISCOVER) {
1845                 pmcs_discover(pwp);
1846         }
1847 
1848         if (work_flags & PMCS_WORK_FLAG_ABORT_HANDLE) {
1849                 if (pmcs_abort_handler(pwp)) {
1850                         SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
1851                 }
1852         }
1853 
1854         if (work_flags & PMCS_WORK_FLAG_SATA_RUN) {
1855                 pmcs_sata_work(pwp);
1856         }
1857 
1858         if (work_flags & PMCS_WORK_FLAG_RUN_QUEUES) {
1859                 pmcs_scsa_wq_run(pwp);
1860                 mutex_enter(&pwp->lock);
1861                 PMCS_CQ_RUN(pwp);
1862                 mutex_exit(&pwp->lock);
1863         }
1864 
1865         if (work_flags & PMCS_WORK_FLAG_ADD_DMA_CHUNKS) {
1866                 if (pmcs_add_more_chunks(pwp,
1867                     ptob(1) * PMCS_ADDTL_CHUNK_PAGES)) {
1868                         SCHEDULE_WORK(pwp, PMCS_WORK_ADD_DMA_CHUNKS);
1869                 } else {
1870                         SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1871                 }
1872         }
1873 }
1874 
1875 static int
1876 pmcs_add_more_chunks(pmcs_hw_t *pwp, unsigned long nsize)
1877 {
1878         pmcs_dmachunk_t *dc;
1879         unsigned long dl;
1880         pmcs_chunk_t    *pchunk = NULL;
1881 
1882         pwp->cip_dma_attr.dma_attr_align = sizeof (uint32_t);
1883 
1884         pchunk = kmem_zalloc(sizeof (pmcs_chunk_t), KM_SLEEP);
1885 
1886         if (pmcs_dma_setup(pwp, &pwp->cip_dma_attr, &pchunk->acc_handle,
1887             &pchunk->dma_handle, nsize, (caddr_t *)&pchunk->addrp,
1888             &pchunk->dma_addr) == B_FALSE) {
1889                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1890                     "Failed to setup DMA for chunks");
1891                 kmem_free(pchunk, sizeof (pmcs_chunk_t));
1892                 return (-1);
1893         }
1894 
1895         if ((pmcs_check_acc_handle(pchunk->acc_handle) != DDI_SUCCESS) ||
1896             (pmcs_check_dma_handle(pchunk->dma_handle) != DDI_SUCCESS)) {
1897                 ddi_fm_service_impact(pwp->dip, DDI_SERVICE_UNAFFECTED);
1898                 return (-1);
1899         }
1900 
1901         bzero(pchunk->addrp, nsize);
1902         dc = NULL;
1903         for (dl = 0; dl < (nsize / PMCS_SGL_CHUNKSZ); dl++) {
1904                 pmcs_dmachunk_t *tmp;
1905                 tmp = kmem_alloc(sizeof (pmcs_dmachunk_t), KM_SLEEP);
1906                 tmp->nxt = dc;
1907                 dc = tmp;
1908         }
1909         mutex_enter(&pwp->dma_lock);
1910         pmcs_idma_chunks(pwp, dc, pchunk, nsize);
1911         pwp->nchunks++;
1912         mutex_exit(&pwp->dma_lock);
1913         return (0);
1914 }
1915 
1916 static void
1917 pmcs_check_forward_progress(pmcs_hw_t *pwp)
1918 {
1919         pmcwork_t       *wrkp;
1920         uint32_t        *iqp;
1921         uint32_t        cur_iqci;
1922         uint32_t        cur_work_idx;
1923         uint32_t        cur_msgu_tick;
1924         uint32_t        cur_iop_tick;
1925         int             i;
1926 
1927         mutex_enter(&pwp->lock);
1928 
1929         if (pwp->state == STATE_IN_RESET) {
1930                 mutex_exit(&pwp->lock);
1931                 return;
1932         }
1933 
1934         /*
1935          * Ensure that inbound work is getting picked up.  First, check to
1936          * see if new work has been posted.  If it has, ensure that the
1937          * work is moving forward by checking the consumer index and the
1938          * last_htag for the work being processed against what we saw last
1939          * time.  Note: we use the work structure's 'last_htag' because at
1940          * any given moment it could be freed back, thus clearing 'htag'
1941          * and setting 'last_htag' (see pmcs_pwork).
1942          */
1943         for (i = 0; i < PMCS_NIQ; i++) {
1944                 cur_iqci = pmcs_rd_iqci(pwp, i);
1945                 iqp = &pwp->iqp[i][cur_iqci * (PMCS_QENTRY_SIZE >> 2)];
1946                 cur_work_idx = PMCS_TAG_INDEX(LE_32(*(iqp+1)));
1947                 wrkp = &pwp->work[cur_work_idx];
1948                 if (cur_iqci == pwp->shadow_iqpi[i]) {
1949                         pwp->last_iqci[i] = cur_iqci;
1950                         pwp->last_htag[i] = wrkp->last_htag;
1951                         continue;
1952                 }
1953                 if ((cur_iqci == pwp->last_iqci[i]) &&
1954                     (wrkp->last_htag == pwp->last_htag[i])) {
1955                         pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1956                             "Inbound Queue stall detected, issuing reset");
1957                         goto hot_reset;
1958                 }
1959                 pwp->last_iqci[i] = cur_iqci;
1960                 pwp->last_htag[i] = wrkp->last_htag;
1961         }
1962 
1963         /*
1964          * Check heartbeat on both the MSGU and IOP.  It is unlikely that
1965          * we'd ever fail here, as the inbound queue monitoring code above
1966          * would detect a stall due to either of these elements being
1967          * stalled, but we might as well keep an eye on them.
1968          */
1969         cur_msgu_tick = pmcs_rd_gst_tbl(pwp, PMCS_GST_MSGU_TICK);
1970         if (cur_msgu_tick == pwp->last_msgu_tick) {
1971                 pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1972                     "Stall detected on MSGU, issuing reset");
1973                 goto hot_reset;
1974         }
1975         pwp->last_msgu_tick = cur_msgu_tick;
1976 
1977         cur_iop_tick  = pmcs_rd_gst_tbl(pwp, PMCS_GST_IOP_TICK);
1978         if (cur_iop_tick == pwp->last_iop_tick) {
1979                 pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1980                     "Stall detected on IOP, issuing reset");
1981                 goto hot_reset;
1982         }
1983         pwp->last_iop_tick = cur_iop_tick;
1984 
1985         mutex_exit(&pwp->lock);
1986         return;
1987 
1988 hot_reset:
1989         pwp->state = STATE_DEAD;
1990         /*
1991          * We've detected a stall. Attempt to recover service via hot
1992          * reset. In case of failure, pmcs_hot_reset() will handle the
1993          * failure and issue any required FM notifications.
1994          * See pmcs_subr.c for more details.
1995          */
1996         if (pmcs_hot_reset(pwp)) {
1997                 pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
1998                     "%s: hot reset failure", __func__);
1999         } else {
2000                 pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2001                     "%s: hot reset complete", __func__);
2002                 pwp->last_reset_reason = PMCS_LAST_RST_STALL;
2003         }
2004         mutex_exit(&pwp->lock);
2005 }
2006 
2007 static void
2008 pmcs_check_commands(pmcs_hw_t *pwp)
2009 {
2010         pmcs_cmd_t *sp;
2011         size_t amt;
2012         char path[32];
2013         pmcwork_t *pwrk;
2014         pmcs_xscsi_t *target;
2015         pmcs_phy_t *phyp;
2016         int rval;
2017 
2018         for (pwrk = pwp->work; pwrk < &pwp->work[pwp->max_cmd]; pwrk++) {
2019                 mutex_enter(&pwrk->lock);
2020 
2021                 /*
2022                  * If the command isn't active, we can't be timing it still.
2023                  * Active means the tag is not free and the state is "on chip".
2024                  */
2025                 if (!PMCS_COMMAND_ACTIVE(pwrk)) {
2026                         mutex_exit(&pwrk->lock);
2027                         continue;
2028                 }
2029 
2030                 /*
2031                  * No timer active for this command.
2032                  */
2033                 if (pwrk->timer == 0) {
2034                         mutex_exit(&pwrk->lock);
2035                         continue;
2036                 }
2037 
2038                 /*
2039                  * Knock off bits for the time interval.
2040                  */
2041                 if (pwrk->timer >= US2WT(PMCS_WATCH_INTERVAL)) {
2042                         pwrk->timer -= US2WT(PMCS_WATCH_INTERVAL);
2043                 } else {
2044                         pwrk->timer = 0;
2045                 }
2046                 if (pwrk->timer > 0) {
2047                         mutex_exit(&pwrk->lock);
2048                         continue;
2049                 }
2050 
2051                 /*
2052                  * The command has now officially timed out.
2053                  * Get the path for it. If it doesn't have
2054                  * a phy pointer any more, it's really dead
2055                  * and can just be put back on the free list.
2056                  * There should *not* be any commands associated
2057                  * with it any more.
2058                  */
2059                 if (pwrk->phy == NULL) {
2060                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2061                             "dead command with gone phy being recycled");
2062                         ASSERT(pwrk->xp == NULL);
2063                         pmcs_pwork(pwp, pwrk);
2064                         continue;
2065                 }
2066                 amt = sizeof (path);
2067                 amt = min(sizeof (pwrk->phy->path), amt);
2068                 (void) memcpy(path, pwrk->phy->path, amt);
2069 
2070                 /*
2071                  * If this is a non-SCSA command, stop here. Eventually
2072                  * we might do something with non-SCSA commands here-
2073                  * but so far their timeout mechanisms are handled in
2074                  * the WAIT_FOR macro.
2075                  */
2076                 if (pwrk->xp == NULL) {
2077                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2078                             "%s: non-SCSA cmd tag 0x%x timed out",
2079                             path, pwrk->htag);
2080                         mutex_exit(&pwrk->lock);
2081                         continue;
2082                 }
2083 
2084                 sp = pwrk->arg;
2085                 ASSERT(sp != NULL);
2086 
2087                 /*
2088                  * Mark it as timed out.
2089                  */
2090                 CMD2PKT(sp)->pkt_reason = CMD_TIMEOUT;
2091                 CMD2PKT(sp)->pkt_statistics |= STAT_TIMEOUT;
2092 #ifdef  DEBUG
2093                 pmcs_prt(pwp, PMCS_PRT_DEBUG, pwrk->phy, pwrk->xp,
2094                     "%s: SCSA cmd tag 0x%x timed out (state %x) onwire=%d",
2095                     path, pwrk->htag, pwrk->state, pwrk->onwire);
2096 #else
2097                 pmcs_prt(pwp, PMCS_PRT_DEBUG, pwrk->phy, pwrk->xp,
2098                     "%s: SCSA cmd tag 0x%x timed out (state %x)",
2099                     path, pwrk->htag, pwrk->state);
2100 #endif
2101                 /*
2102                  * Mark the work structure as timed out.
2103                  */
2104                 pwrk->state = PMCS_WORK_STATE_TIMED_OUT;
2105                 phyp = pwrk->phy;
2106                 target = pwrk->xp;
2107                 ASSERT(target != NULL);
2108                 mutex_exit(&pwrk->lock);
2109 
2110                 pmcs_lock_phy(phyp);
2111                 mutex_enter(&target->statlock);
2112 
2113                 /*
2114                  * No point attempting recovery if the device is gone
2115                  */
2116                 if (target->dev_gone) {
2117                         mutex_exit(&target->statlock);
2118                         pmcs_unlock_phy(phyp);
2119                         pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2120                             "%s: tgt(0x%p) is gone. Returning CMD_DEV_GONE "
2121                             "for htag 0x%08x", __func__,
2122                             (void *)target, pwrk->htag);
2123                         mutex_enter(&pwrk->lock);
2124                         if (!PMCS_COMMAND_DONE(pwrk)) {
2125                                 /* Complete this command here */
2126                                 pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2127                                     "%s: Completing cmd (htag 0x%08x) "
2128                                     "anyway", __func__, pwrk->htag);
2129                                 pwrk->dead = 1;
2130                                 CMD2PKT(sp)->pkt_reason = CMD_DEV_GONE;
2131                                 CMD2PKT(sp)->pkt_state = STATE_GOT_BUS;
2132                                 pmcs_complete_work_impl(pwp, pwrk, NULL, 0);
2133                         } else {
2134                                 mutex_exit(&pwrk->lock);
2135                         }
2136                         continue;
2137                 }
2138 
2139                 mutex_exit(&target->statlock);
2140                 rval = pmcs_abort(pwp, phyp, pwrk->htag, 0, 1);
2141                 if (rval) {
2142                         pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2143                             "%s: Bad status (%d) on abort of HTAG 0x%08x",
2144                             __func__, rval, pwrk->htag);
2145                         pmcs_unlock_phy(phyp);
2146                         mutex_enter(&pwrk->lock);
2147                         if (!PMCS_COMMAND_DONE(pwrk)) {
2148                                 /* Complete this command here */
2149                                 pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, target,
2150                                     "%s: Completing cmd (htag 0x%08x) "
2151                                     "anyway", __func__, pwrk->htag);
2152                                 if (target->dev_gone) {
2153                                         pwrk->dead = 1;
2154                                         CMD2PKT(sp)->pkt_reason = CMD_DEV_GONE;
2155                                         CMD2PKT(sp)->pkt_state = STATE_GOT_BUS;
2156                                 }
2157                                 pmcs_complete_work_impl(pwp, pwrk, NULL, 0);
2158                         } else {
2159                                 mutex_exit(&pwrk->lock);
2160                         }
2161                         pmcs_lock_phy(phyp);
2162                         /*
2163                          * No need to reschedule ABORT if we get any other
2164                          * status
2165                          */
2166                         if (rval == ENOMEM) {
2167                                 phyp->abort_sent = 0;
2168                                 phyp->abort_pending = 1;
2169                                 SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
2170                         }
2171                 }
2172                 pmcs_unlock_phy(phyp);
2173         }
2174         /*
2175          * Run any completions that may have been queued up.
2176          */
2177         PMCS_CQ_RUN(pwp);
2178 }
2179 
2180 static void
2181 pmcs_watchdog(void *arg)
2182 {
2183         pmcs_hw_t *pwp = arg;
2184 
2185         DTRACE_PROBE2(pmcs__watchdog, ulong_t, pwp->work_flags, boolean_t,
2186             pwp->config_changed);
2187 
2188         /*
2189          * Check forward progress on the chip
2190          */
2191         if (++pwp->watchdog_count == PMCS_FWD_PROG_TRIGGER) {
2192                 pwp->watchdog_count = 0;
2193                 pmcs_check_forward_progress(pwp);
2194         }
2195 
2196         /*
2197          * Check to see if we need to kick discovery off again
2198          */
2199         mutex_enter(&pwp->config_lock);
2200         if (pwp->config_restart &&
2201             (ddi_get_lbolt() >= pwp->config_restart_time)) {
2202                 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2203                     "%s: Timer expired for re-enumeration: Start discovery",
2204                     __func__);
2205                 pwp->config_restart = B_FALSE;
2206                 SCHEDULE_WORK(pwp, PMCS_WORK_DISCOVER);
2207         }
2208         mutex_exit(&pwp->config_lock);
2209 
2210         mutex_enter(&pwp->lock);
2211         if (pwp->state != STATE_RUNNING) {
2212                 mutex_exit(&pwp->lock);
2213                 return;
2214         }
2215 
2216         if (atomic_cas_ulong(&pwp->work_flags, 0, 0) != 0) {
2217                 if (ddi_taskq_dispatch(pwp->tq, pmcs_worker, pwp,
2218                     DDI_NOSLEEP) != DDI_SUCCESS) {
2219                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2220                             "Could not dispatch to worker thread");
2221                 }
2222         }
2223         pwp->wdhandle = timeout(pmcs_watchdog, pwp,
2224             drv_usectohz(PMCS_WATCH_INTERVAL));
2225 
2226         mutex_exit(&pwp->lock);
2227 
2228         pmcs_check_commands(pwp);
2229         pmcs_handle_dead_phys(pwp);
2230 }
2231 
2232 static int
2233 pmcs_remove_ihandlers(pmcs_hw_t *pwp, int icnt)
2234 {
2235         int i, r, rslt = 0;
2236         for (i = 0; i < icnt; i++) {
2237                 r = ddi_intr_remove_handler(pwp->ih_table[i]);
2238                 if (r == DDI_SUCCESS) {
2239                         continue;
2240                 }
2241                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2242                     "%s: unable to remove interrupt handler %d", __func__, i);
2243                 rslt = -1;
2244                 break;
2245         }
2246         return (rslt);
2247 }
2248 
2249 static int
2250 pmcs_disable_intrs(pmcs_hw_t *pwp, int icnt)
2251 {
2252         if (pwp->intr_cap & DDI_INTR_FLAG_BLOCK) {
2253                 int r = ddi_intr_block_disable(&pwp->ih_table[0],
2254                     pwp->intr_cnt);
2255                 if (r != DDI_SUCCESS) {
2256                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2257                             "unable to disable interrupt block");
2258                         return (-1);
2259                 }
2260         } else {
2261                 int i;
2262                 for (i = 0; i < icnt; i++) {
2263                         if (ddi_intr_disable(pwp->ih_table[i]) == DDI_SUCCESS) {
2264                                 continue;
2265                         }
2266                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2267                             "unable to disable interrupt %d", i);
2268                         return (-1);
2269                 }
2270         }
2271         return (0);
2272 }
2273 
2274 static int
2275 pmcs_free_intrs(pmcs_hw_t *pwp, int icnt)
2276 {
2277         int i;
2278         for (i = 0; i < icnt; i++) {
2279                 if (ddi_intr_free(pwp->ih_table[i]) == DDI_SUCCESS) {
2280                         continue;
2281                 }
2282                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2283                     "unable to free interrupt %d", i);
2284                 return (-1);
2285         }
2286         kmem_free(pwp->ih_table, pwp->ih_table_size);
2287         pwp->ih_table_size = 0;
2288         return (0);
2289 }
2290 
2291 /*
2292  * Try to set up interrupts of type "type" with a minimum number of interrupts
2293  * of "min".
2294  */
2295 static void
2296 pmcs_setup_intr_impl(pmcs_hw_t *pwp, int type, int min)
2297 {
2298         int rval, avail, count, actual, max;
2299 
2300         rval = ddi_intr_get_nintrs(pwp->dip, type, &count);
2301         if ((rval != DDI_SUCCESS) || (count < min)) {
2302                 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2303                     "%s: get_nintrs failed; type: %d rc: %d count: %d min: %d",
2304                     __func__, type, rval, count, min);
2305                 return;
2306         }
2307 
2308         pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2309             "%s: nintrs = %d for type: %d", __func__, count, type);
2310 
2311         rval = ddi_intr_get_navail(pwp->dip, type, &avail);
2312         if ((rval != DDI_SUCCESS) || (avail < min)) {
2313                 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2314                     "%s: get_navail failed; type: %d rc: %d avail: %d min: %d",
2315                     __func__, type, rval, avail, min);
2316                 return;
2317         }
2318 
2319         pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2320             "%s: navail = %d for type: %d", __func__, avail, type);
2321 
2322         pwp->ih_table_size = avail * sizeof (ddi_intr_handle_t);
2323         pwp->ih_table = kmem_alloc(pwp->ih_table_size, KM_SLEEP);
2324 
2325         switch (type) {
2326         case DDI_INTR_TYPE_MSIX:
2327                 pwp->int_type = PMCS_INT_MSIX;
2328                 max = PMCS_MAX_MSIX;
2329                 break;
2330         case DDI_INTR_TYPE_MSI:
2331                 pwp->int_type = PMCS_INT_MSI;
2332                 max = PMCS_MAX_MSI;
2333                 break;
2334         case DDI_INTR_TYPE_FIXED:
2335         default:
2336                 pwp->int_type = PMCS_INT_FIXED;
2337                 max = PMCS_MAX_FIXED;
2338                 break;
2339         }
2340 
2341         rval = ddi_intr_alloc(pwp->dip, pwp->ih_table, type, 0, max, &actual,
2342             DDI_INTR_ALLOC_NORMAL);
2343         if (rval != DDI_SUCCESS) {
2344                 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2345                     "%s: ddi_intr_alloc failed; type: %d rc: %d",
2346                     __func__, type, rval);
2347                 kmem_free(pwp->ih_table, pwp->ih_table_size);
2348                 pwp->ih_table = NULL;
2349                 pwp->ih_table_size = 0;
2350                 pwp->intr_cnt = 0;
2351                 pwp->int_type = PMCS_INT_NONE;
2352                 return;
2353         }
2354 
2355         pwp->intr_cnt = actual;
2356 }
2357 
2358 /*
2359  * Set up interrupts.
2360  * We return one of three values:
2361  *
2362  * 0 - success
2363  * EAGAIN - failure to set up interrupts
2364  * EIO - "" + we're now stuck partly enabled
2365  *
2366  * If EIO is returned, we can't unload the driver.
2367  */
2368 static int
2369 pmcs_setup_intr(pmcs_hw_t *pwp)
2370 {
2371         int i, r, itypes, oqv_count;
2372         ddi_intr_handler_t **iv_table;
2373         size_t iv_table_size;
2374         uint_t pri;
2375 
2376         if (ddi_intr_get_supported_types(pwp->dip, &itypes) != DDI_SUCCESS) {
2377                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2378                     "cannot get interrupt types");
2379                 return (EAGAIN);
2380         }
2381 
2382         if (disable_msix) {
2383                 itypes &= ~DDI_INTR_TYPE_MSIX;
2384         }
2385         if (disable_msi) {
2386                 itypes &= ~DDI_INTR_TYPE_MSI;
2387         }
2388 
2389         /*
2390          * We won't know what firmware we're running until we call pmcs_setup,
2391          * and we can't call pmcs_setup until we establish interrupts.
2392          */
2393 
2394         pwp->int_type = PMCS_INT_NONE;
2395 
2396         /*
2397          * We want PMCS_MAX_MSIX vectors for MSI-X.  Anything less would be
2398          * uncivilized.
2399          */
2400         if (itypes & DDI_INTR_TYPE_MSIX) {
2401                 pmcs_setup_intr_impl(pwp, DDI_INTR_TYPE_MSIX, PMCS_MAX_MSIX);
2402                 if (pwp->int_type == PMCS_INT_MSIX) {
2403                         itypes = 0;
2404                 }
2405         }
2406 
2407         if (itypes & DDI_INTR_TYPE_MSI) {
2408                 pmcs_setup_intr_impl(pwp, DDI_INTR_TYPE_MSI, 1);
2409                 if (pwp->int_type == PMCS_INT_MSI) {
2410                         itypes = 0;
2411                 }
2412         }
2413 
2414         if (itypes & DDI_INTR_TYPE_FIXED) {
2415                 pmcs_setup_intr_impl(pwp, DDI_INTR_TYPE_FIXED, 1);
2416                 if (pwp->int_type == PMCS_INT_FIXED) {
2417                         itypes = 0;
2418                 }
2419         }
2420 
2421         if (pwp->intr_cnt == 0) {
2422                 pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2423                     "No interrupts available");
2424                 return (EAGAIN);
2425         }
2426 
2427         iv_table_size = sizeof (ddi_intr_handler_t *) * pwp->intr_cnt;
2428         iv_table = kmem_alloc(iv_table_size, KM_SLEEP);
2429 
2430         /*
2431          * Get iblock cookie and add handlers.
2432          */
2433         switch (pwp->intr_cnt) {
2434         case 1:
2435                 iv_table[0] = pmcs_all_intr;
2436                 break;
2437         case 2:
2438                 iv_table[0] = pmcs_iodone_ix;
2439                 iv_table[1] = pmcs_nonio_ix;
2440                 break;
2441         case 4:
2442                 iv_table[PMCS_MSIX_GENERAL] = pmcs_general_ix;
2443                 iv_table[PMCS_MSIX_IODONE] = pmcs_iodone_ix;
2444                 iv_table[PMCS_MSIX_EVENTS] = pmcs_event_ix;
2445                 iv_table[PMCS_MSIX_FATAL] = pmcs_fatal_ix;
2446                 break;
2447         default:
2448                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2449                     "%s: intr_cnt = %d - unexpected", __func__, pwp->intr_cnt);
2450                 kmem_free(iv_table, iv_table_size);
2451                 return (EAGAIN);
2452         }
2453 
2454         for (i = 0; i < pwp->intr_cnt; i++) {
2455                 r = ddi_intr_add_handler(pwp->ih_table[i], iv_table[i],
2456                     (caddr_t)pwp, NULL);
2457                 if (r != DDI_SUCCESS) {
2458                         kmem_free(iv_table, iv_table_size);
2459                         if (pmcs_remove_ihandlers(pwp, i)) {
2460                                 return (EIO);
2461                         }
2462                         if (pmcs_free_intrs(pwp, i)) {
2463                                 return (EIO);
2464                         }
2465                         pwp->intr_cnt = 0;
2466                         return (EAGAIN);
2467                 }
2468         }
2469 
2470         kmem_free(iv_table, iv_table_size);
2471 
2472         if (ddi_intr_get_cap(pwp->ih_table[0], &pwp->intr_cap) != DDI_SUCCESS) {
2473                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2474                     "unable to get int capabilities");
2475                 if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2476                         return (EIO);
2477                 }
2478                 if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2479                         return (EIO);
2480                 }
2481                 pwp->intr_cnt = 0;
2482                 return (EAGAIN);
2483         }
2484 
2485         if (pwp->intr_cap & DDI_INTR_FLAG_BLOCK) {
2486                 r = ddi_intr_block_enable(&pwp->ih_table[0], pwp->intr_cnt);
2487                 if (r != DDI_SUCCESS) {
2488                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2489                             "intr blk enable failed");
2490                         if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2491                                 return (EIO);
2492                         }
2493                         if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2494                                 return (EIO);
2495                         }
2496                         pwp->intr_cnt = 0;
2497                         return (EFAULT);
2498                 }
2499         } else {
2500                 for (i = 0; i < pwp->intr_cnt; i++) {
2501                         r = ddi_intr_enable(pwp->ih_table[i]);
2502                         if (r == DDI_SUCCESS) {
2503                                 continue;
2504                         }
2505                         pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2506                             "unable to enable interrupt %d", i);
2507                         if (pmcs_disable_intrs(pwp, i)) {
2508                                 return (EIO);
2509                         }
2510                         if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2511                                 return (EIO);
2512                         }
2513                         if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2514                                 return (EIO);
2515                         }
2516                         pwp->intr_cnt = 0;
2517                         return (EAGAIN);
2518                 }
2519         }
2520 
2521         /*
2522          * Set up locks.
2523          */
2524         if (ddi_intr_get_pri(pwp->ih_table[0], &pri) != DDI_SUCCESS) {
2525                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2526                     "unable to get interrupt priority");
2527                 if (pmcs_disable_intrs(pwp, pwp->intr_cnt)) {
2528                         return (EIO);
2529                 }
2530                 if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2531                         return (EIO);
2532                 }
2533                 if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2534                         return (EIO);
2535                 }
2536                 pwp->intr_cnt = 0;
2537                 return (EAGAIN);
2538         }
2539 
2540         pwp->locks_initted = 1;
2541         pwp->intr_pri = pri;
2542         mutex_init(&pwp->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2543         mutex_init(&pwp->dma_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2544         mutex_init(&pwp->axil_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2545         mutex_init(&pwp->cq_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2546         mutex_init(&pwp->ict_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2547         mutex_init(&pwp->config_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2548         mutex_init(&pwp->wfree_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2549         mutex_init(&pwp->pfree_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2550         mutex_init(&pwp->dead_phylist_lock, NULL, MUTEX_DRIVER,
2551             DDI_INTR_PRI(pri));
2552 #ifdef  DEBUG
2553         mutex_init(&pwp->dbglock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(pri));
2554 #endif
2555         cv_init(&pwp->ict_cv, NULL, CV_DRIVER, NULL);
2556         cv_init(&pwp->drain_cv, NULL, CV_DRIVER, NULL);
2557         cv_init(&pwp->config_cv, NULL, CV_DRIVER, NULL);
2558         for (i = 0; i < PMCS_NIQ; i++) {
2559                 mutex_init(&pwp->iqp_lock[i], NULL,
2560                     MUTEX_DRIVER, DDI_INTR_PRI(pwp->intr_pri));
2561         }
2562         for (i = 0; i < pwp->cq_info.cq_threads; i++) {
2563                 mutex_init(&pwp->cq_info.cq_thr_info[i].cq_thr_lock, NULL,
2564                     MUTEX_DRIVER, DDI_INTR_PRI(pwp->intr_pri));
2565                 cv_init(&pwp->cq_info.cq_thr_info[i].cq_cv, NULL,
2566                     CV_DRIVER, NULL);
2567         }
2568 
2569         pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "%d %s interrup%s configured",
2570             pwp->intr_cnt, (pwp->int_type == PMCS_INT_MSIX)? "MSI-X" :
2571             ((pwp->int_type == PMCS_INT_MSI)? "MSI" : "INT-X"),
2572             pwp->intr_cnt == 1? "t" : "ts");
2573 
2574 
2575         /*
2576          * Enable Interrupts
2577          */
2578         if (pwp->intr_cnt > PMCS_NOQ) {
2579                 oqv_count = pwp->intr_cnt;
2580         } else {
2581                 oqv_count = PMCS_NOQ;
2582         }
2583         for (pri = 0xffffffff, i = 0; i < oqv_count; i++) {
2584                 pri ^= (1 << i);
2585         }
2586 
2587         mutex_enter(&pwp->lock);
2588         pwp->intr_mask = pri;
2589         pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_MASK, pwp->intr_mask);
2590         pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR, 0xffffffff);
2591         mutex_exit(&pwp->lock);
2592 
2593         return (0);
2594 }
2595 
2596 static int
2597 pmcs_teardown_intr(pmcs_hw_t *pwp)
2598 {
2599         if (pwp->intr_cnt) {
2600                 if (pmcs_disable_intrs(pwp, pwp->intr_cnt)) {
2601                         return (EIO);
2602                 }
2603                 if (pmcs_remove_ihandlers(pwp, pwp->intr_cnt)) {
2604                         return (EIO);
2605                 }
2606                 if (pmcs_free_intrs(pwp, pwp->intr_cnt)) {
2607                         return (EIO);
2608                 }
2609                 pwp->intr_cnt = 0;
2610         }
2611         return (0);
2612 }
2613 
2614 static uint_t
2615 pmcs_general_ix(caddr_t arg1, caddr_t arg2)
2616 {
2617         pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2618         _NOTE(ARGUNUSED(arg2));
2619         pmcs_general_intr(pwp);
2620         return (DDI_INTR_CLAIMED);
2621 }
2622 
2623 static uint_t
2624 pmcs_event_ix(caddr_t arg1, caddr_t arg2)
2625 {
2626         pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2627         _NOTE(ARGUNUSED(arg2));
2628         pmcs_event_intr(pwp);
2629         return (DDI_INTR_CLAIMED);
2630 }
2631 
2632 static uint_t
2633 pmcs_iodone_ix(caddr_t arg1, caddr_t arg2)
2634 {
2635         _NOTE(ARGUNUSED(arg2));
2636         pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2637 
2638         /*
2639          * It's possible that if we just turned interrupt coalescing off
2640          * (and thus, re-enabled auto clear for interrupts on the I/O outbound
2641          * queue) that there was an interrupt already pending.  We use
2642          * io_intr_coal.int_cleared to ensure that we still drop in here and
2643          * clear the appropriate interrupt bit one last time.
2644          */
2645         mutex_enter(&pwp->ict_lock);
2646         if (pwp->io_intr_coal.timer_on ||
2647             (pwp->io_intr_coal.int_cleared == B_FALSE)) {
2648                 pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2649                     (1 << PMCS_OQ_IODONE));
2650                 pwp->io_intr_coal.int_cleared = B_TRUE;
2651         }
2652         mutex_exit(&pwp->ict_lock);
2653 
2654         pmcs_iodone_intr(pwp);
2655 
2656         return (DDI_INTR_CLAIMED);
2657 }
2658 
2659 static uint_t
2660 pmcs_fatal_ix(caddr_t arg1, caddr_t arg2)
2661 {
2662         pmcs_hw_t *pwp = (pmcs_hw_t *)((void *)arg1);
2663         _NOTE(ARGUNUSED(arg2));
2664         pmcs_fatal_handler(pwp);
2665         return (DDI_INTR_CLAIMED);
2666 }
2667 
2668 static uint_t
2669 pmcs_nonio_ix(caddr_t arg1, caddr_t arg2)
2670 {
2671         _NOTE(ARGUNUSED(arg2));
2672         pmcs_hw_t *pwp = (void *)arg1;
2673         uint32_t obdb = pmcs_rd_msgunit(pwp, PMCS_MSGU_OBDB);
2674 
2675         /*
2676          * Check for Fatal Interrupts
2677          */
2678         if (obdb & (1 << PMCS_FATAL_INTERRUPT)) {
2679                 pmcs_fatal_handler(pwp);
2680                 return (DDI_INTR_CLAIMED);
2681         }
2682 
2683         if (obdb & (1 << PMCS_OQ_GENERAL)) {
2684                 pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2685                     (1 << PMCS_OQ_GENERAL));
2686                 pmcs_general_intr(pwp);
2687                 pmcs_event_intr(pwp);
2688         }
2689 
2690         return (DDI_INTR_CLAIMED);
2691 }
2692 
2693 static uint_t
2694 pmcs_all_intr(caddr_t arg1, caddr_t arg2)
2695 {
2696         _NOTE(ARGUNUSED(arg2));
2697         pmcs_hw_t *pwp = (void *) arg1;
2698         uint32_t obdb;
2699         int handled = 0;
2700 
2701         obdb = pmcs_rd_msgunit(pwp, PMCS_MSGU_OBDB);
2702 
2703         /*
2704          * Check for Fatal Interrupts
2705          */
2706         if (obdb & (1 << PMCS_FATAL_INTERRUPT)) {
2707                 pmcs_fatal_handler(pwp);
2708                 return (DDI_INTR_CLAIMED);
2709         }
2710 
2711         /*
2712          * Check for Outbound Queue service needed
2713          */
2714         if (obdb & (1 << PMCS_OQ_IODONE)) {
2715                 pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2716                     (1 << PMCS_OQ_IODONE));
2717                 obdb ^= (1 << PMCS_OQ_IODONE);
2718                 handled++;
2719                 pmcs_iodone_intr(pwp);
2720         }
2721         if (obdb & (1 << PMCS_OQ_GENERAL)) {
2722                 pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2723                     (1 << PMCS_OQ_GENERAL));
2724                 obdb ^= (1 << PMCS_OQ_GENERAL);
2725                 handled++;
2726                 pmcs_general_intr(pwp);
2727         }
2728         if (obdb & (1 << PMCS_OQ_EVENTS)) {
2729                 pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR,
2730                     (1 << PMCS_OQ_EVENTS));
2731                 obdb ^= (1 << PMCS_OQ_EVENTS);
2732                 handled++;
2733                 pmcs_event_intr(pwp);
2734         }
2735         if (obdb) {
2736                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
2737                     "interrupt bits not handled (0x%x)", obdb);
2738                 pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR, obdb);
2739                 handled++;
2740         }
2741         if (pwp->int_type == PMCS_INT_MSI) {
2742                 handled++;
2743         }
2744         return (handled? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
2745 }
2746 
2747 void
2748 pmcs_fatal_handler(pmcs_hw_t *pwp)
2749 {
2750         pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL, "Fatal Interrupt caught");
2751 
2752         mutex_enter(&pwp->lock);
2753         pwp->state = STATE_DEAD;
2754 
2755         /*
2756          * Attempt a hot reset. In case of failure, pmcs_hot_reset() will
2757          * handle the failure and issue any required FM notifications.
2758          * See pmcs_subr.c for more details.
2759          */
2760         if (pmcs_hot_reset(pwp)) {
2761                 pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2762                     "%s: hot reset failure", __func__);
2763         } else {
2764                 pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
2765                     "%s: hot reset complete", __func__);
2766                 pwp->last_reset_reason = PMCS_LAST_RST_FATAL_ERROR;
2767         }
2768         mutex_exit(&pwp->lock);
2769 }
2770 
2771 /*
2772  * Called with PHY lock and target statlock held and scratch acquired.
2773  */
2774 boolean_t
2775 pmcs_assign_device(pmcs_hw_t *pwp, pmcs_xscsi_t *tgt)
2776 {
2777         pmcs_phy_t *pptr = tgt->phy;
2778 
2779         switch (pptr->dtype) {
2780         case SAS:
2781         case EXPANDER:
2782                 break;
2783         case SATA:
2784                 tgt->ca = 1;
2785                 break;
2786         default:
2787                 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, tgt,
2788                     "%s: Target %p has PHY %p with invalid dtype",
2789                     __func__, (void *)tgt, (void *)pptr);
2790                 return (B_FALSE);
2791         }
2792 
2793         tgt->new = 1;
2794         tgt->dev_gone = 0;
2795         tgt->recover_wait = 0;
2796 
2797         pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, tgt,
2798             "%s: config %s vtgt %u for " SAS_ADDR_FMT, __func__,
2799             pptr->path, tgt->target_num, SAS_ADDR_PRT(pptr->sas_address));
2800 
2801         if (pmcs_add_new_device(pwp, tgt) != B_TRUE) {
2802                 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, tgt,
2803                     "%s: Failed for vtgt %u / WWN " SAS_ADDR_FMT, __func__,
2804                     tgt->target_num, SAS_ADDR_PRT(pptr->sas_address));
2805                 mutex_destroy(&tgt->statlock);
2806                 mutex_destroy(&tgt->wqlock);
2807                 mutex_destroy(&tgt->aqlock);
2808                 return (B_FALSE);
2809         }
2810 
2811         return (B_TRUE);
2812 }
2813 
2814 /*
2815  * Called with softstate lock held
2816  */
2817 void
2818 pmcs_remove_device(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
2819 {
2820         pmcs_xscsi_t *xp;
2821         unsigned int vtgt;
2822 
2823         ASSERT(mutex_owned(&pwp->lock));
2824 
2825         for (vtgt = 0; vtgt < pwp->max_dev; vtgt++) {
2826                 xp = pwp->targets[vtgt];
2827                 if (xp == NULL) {
2828                         continue;
2829                 }
2830 
2831                 mutex_enter(&xp->statlock);
2832                 if (xp->phy == pptr) {
2833                         if (xp->new) {
2834                                 xp->new = 0;
2835                                 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, xp,
2836                                     "cancel config of vtgt %u", vtgt);
2837                         } else {
2838                                 pmcs_clear_xp(pwp, xp);
2839                                 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, xp,
2840                                     "Removed tgt 0x%p vtgt %u",
2841                                     (void *)xp, vtgt);
2842                         }
2843                         mutex_exit(&xp->statlock);
2844                         break;
2845                 }
2846                 mutex_exit(&xp->statlock);
2847         }
2848 }
2849 
2850 void
2851 pmcs_prt_impl(pmcs_hw_t *pwp, pmcs_prt_level_t level,
2852     pmcs_phy_t *phyp, pmcs_xscsi_t *target, const char *fmt, ...)
2853 {
2854         va_list ap;
2855         int written = 0;
2856         char *ptr;
2857         uint32_t elem_size = PMCS_TBUF_ELEM_SIZE - 1;
2858         boolean_t system_log;
2859         int system_log_level;
2860         hrtime_t hrtimestamp;
2861 
2862         switch (level) {
2863         case PMCS_PRT_DEBUG_DEVEL:
2864         case PMCS_PRT_DEBUG_DEV_STATE:
2865         case PMCS_PRT_DEBUG_PHY_LOCKING:
2866         case PMCS_PRT_DEBUG_SCSI_STATUS:
2867         case PMCS_PRT_DEBUG_UNDERFLOW:
2868         case PMCS_PRT_DEBUG_CONFIG:
2869         case PMCS_PRT_DEBUG_IPORT:
2870         case PMCS_PRT_DEBUG_MAP:
2871         case PMCS_PRT_DEBUG3:
2872         case PMCS_PRT_DEBUG2:
2873         case PMCS_PRT_DEBUG1:
2874         case PMCS_PRT_DEBUG:
2875                 system_log = B_FALSE;
2876                 break;
2877         case PMCS_PRT_INFO:
2878                 system_log = B_TRUE;
2879                 system_log_level = CE_CONT;
2880                 break;
2881         case PMCS_PRT_WARN:
2882                 system_log = B_TRUE;
2883                 system_log_level = CE_NOTE;
2884                 break;
2885         case PMCS_PRT_ERR:
2886                 system_log = B_TRUE;
2887                 system_log_level = CE_WARN;
2888                 break;
2889         default:
2890                 return;
2891         }
2892 
2893         mutex_enter(&pmcs_trace_lock);
2894         hrtimestamp = gethrtime();
2895         gethrestime(&pmcs_tbuf_ptr->timestamp);
2896 
2897         if (pwp->fw_timestamp != 0) {
2898                 /* Calculate the approximate firmware time stamp... */
2899                 pmcs_tbuf_ptr->fw_timestamp = pwp->fw_timestamp +
2900                     ((hrtimestamp - pwp->hrtimestamp) / PMCS_FWLOG_TIMER_DIV);
2901         } else {
2902                 pmcs_tbuf_ptr->fw_timestamp = 0;
2903         }
2904 
2905         ptr = pmcs_tbuf_ptr->buf;
2906 
2907         /*
2908          * Store the pertinent PHY and target information if there is any
2909          */
2910         if (target == NULL) {
2911                 pmcs_tbuf_ptr->target_num = PMCS_INVALID_TARGET_NUM;
2912                 pmcs_tbuf_ptr->target_ua[0] = '\0';
2913         } else {
2914                 pmcs_tbuf_ptr->target_num = target->target_num;
2915                 (void) strncpy(pmcs_tbuf_ptr->target_ua, target->ua,
2916                     PMCS_TBUF_UA_MAX_SIZE);
2917         }
2918 
2919         if (phyp == NULL) {
2920                 (void) memset(pmcs_tbuf_ptr->phy_sas_address, 0, 8);
2921                 pmcs_tbuf_ptr->phy_path[0] = '\0';
2922                 pmcs_tbuf_ptr->phy_dtype = NOTHING;
2923         } else {
2924                 (void) memcpy(pmcs_tbuf_ptr->phy_sas_address,
2925                     phyp->sas_address, 8);
2926                 (void) strncpy(pmcs_tbuf_ptr->phy_path, phyp->path, 32);
2927                 pmcs_tbuf_ptr->phy_dtype = phyp->dtype;
2928         }
2929 
2930         written += snprintf(ptr, elem_size, "pmcs%d:%d: ",
2931             ddi_get_instance(pwp->dip), level);
2932         ptr += strlen(ptr);
2933         va_start(ap, fmt);
2934         written += vsnprintf(ptr, elem_size - written, fmt, ap);
2935         va_end(ap);
2936         if (written > elem_size - 1) {
2937                 /* Indicate truncation */
2938                 pmcs_tbuf_ptr->buf[elem_size - 1] = '+';
2939         }
2940         if (++pmcs_tbuf_idx == pmcs_tbuf_num_elems) {
2941                 pmcs_tbuf_ptr = pmcs_tbuf;
2942                 pmcs_tbuf_wrap = B_TRUE;
2943                 pmcs_tbuf_idx = 0;
2944         } else {
2945                 ++pmcs_tbuf_ptr;
2946         }
2947         mutex_exit(&pmcs_trace_lock);
2948 
2949         /*
2950          * When pmcs_force_syslog in non-zero, everything goes also
2951          * to syslog, at CE_CONT level.
2952          */
2953         if (pmcs_force_syslog) {
2954                 system_log = B_TRUE;
2955                 system_log_level = CE_CONT;
2956         }
2957 
2958         /*
2959          * Anything that comes in with PMCS_PRT_INFO, WARN, or ERR also
2960          * goes to syslog.
2961          */
2962         if (system_log) {
2963                 char local[196];
2964 
2965                 switch (system_log_level) {
2966                 case CE_CONT:
2967                         (void) snprintf(local, sizeof (local), "%sINFO: ",
2968                             pmcs_console ? "" : "?");
2969                         break;
2970                 case CE_NOTE:
2971                 case CE_WARN:
2972                         local[0] = 0;
2973                         break;
2974                 default:
2975                         return;
2976                 }
2977 
2978                 ptr = local;
2979                 ptr += strlen(local);
2980                 (void) snprintf(ptr, (sizeof (local)) -
2981                     ((size_t)ptr - (size_t)local), "pmcs%d: ",
2982                     ddi_get_instance(pwp->dip));
2983                 ptr += strlen(ptr);
2984                 va_start(ap, fmt);
2985                 (void) vsnprintf(ptr,
2986                     (sizeof (local)) - ((size_t)ptr - (size_t)local), fmt, ap);
2987                 va_end(ap);
2988                 if (level == CE_CONT) {
2989                         (void) strlcat(local, "\n", sizeof (local));
2990                 }
2991                 cmn_err(system_log_level, local);
2992         }
2993 
2994 }
2995 
2996 /*
2997  * pmcs_acquire_scratch
2998  *
2999  * If "wait" is true, the caller will wait until it can acquire the scratch.
3000  * This implies the caller needs to be in a context where spinning for an
3001  * indeterminate amount of time is acceptable.
3002  */
3003 int
3004 pmcs_acquire_scratch(pmcs_hw_t *pwp, boolean_t wait)
3005 {
3006         int rval;
3007 
3008         if (!wait) {
3009                 return (atomic_swap_8(&pwp->scratch_locked, 1));
3010         }
3011 
3012         /*
3013          * Caller will wait for scratch.
3014          */
3015         while ((rval = atomic_swap_8(&pwp->scratch_locked, 1)) != 0) {
3016                 drv_usecwait(100);
3017         }
3018 
3019         return (rval);
3020 }
3021 
3022 void
3023 pmcs_release_scratch(pmcs_hw_t *pwp)
3024 {
3025         pwp->scratch_locked = 0;
3026 }
3027 
3028 /* Called with iport_lock and phy lock held */
3029 void
3030 pmcs_create_one_phy_stats(pmcs_iport_t *iport, pmcs_phy_t *phyp)
3031 {
3032         sas_phy_stats_t         *ps;
3033         pmcs_hw_t               *pwp;
3034         int                     ndata;
3035         char                    ks_name[KSTAT_STRLEN];
3036 
3037         ASSERT(mutex_owned(&iport->lock));
3038         pwp = iport->pwp;
3039         ASSERT(pwp != NULL);
3040         ASSERT(mutex_owned(&phyp->phy_lock));
3041 
3042         if (phyp->phy_stats != NULL) {
3043                 /*
3044                  * Delete existing kstats with name containing
3045                  * old iport instance# and allow creation of
3046                  * new kstats with new iport instance# in the name.
3047                  */
3048                 kstat_delete(phyp->phy_stats);
3049         }
3050 
3051         ndata = (sizeof (sas_phy_stats_t)/sizeof (kstat_named_t));
3052 
3053         (void) snprintf(ks_name, sizeof (ks_name),
3054             "%s.%llx.%d.%d", ddi_driver_name(iport->dip),
3055             (longlong_t)pwp->sas_wwns[0],
3056             ddi_get_instance(iport->dip), phyp->phynum);
3057 
3058         phyp->phy_stats = kstat_create("pmcs",
3059             ddi_get_instance(iport->dip), ks_name, KSTAT_SAS_PHY_CLASS,
3060             KSTAT_TYPE_NAMED, ndata, 0);
3061 
3062         if (phyp->phy_stats == NULL) {
3063                 pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL,
3064                     "%s: Failed to create %s kstats for PHY(0x%p) at %s",
3065                     __func__, ks_name, (void *)phyp, phyp->path);
3066                 return;
3067         }
3068 
3069         ps = (sas_phy_stats_t *)phyp->phy_stats->ks_data;
3070 
3071         kstat_named_init(&ps->seconds_since_last_reset,
3072             "SecondsSinceLastReset", KSTAT_DATA_ULONGLONG);
3073         kstat_named_init(&ps->tx_frames,
3074             "TxFrames", KSTAT_DATA_ULONGLONG);
3075         kstat_named_init(&ps->rx_frames,
3076             "RxFrames", KSTAT_DATA_ULONGLONG);
3077         kstat_named_init(&ps->tx_words,
3078             "TxWords", KSTAT_DATA_ULONGLONG);
3079         kstat_named_init(&ps->rx_words,
3080             "RxWords", KSTAT_DATA_ULONGLONG);
3081         kstat_named_init(&ps->invalid_dword_count,
3082             "InvalidDwordCount", KSTAT_DATA_ULONGLONG);
3083         kstat_named_init(&ps->running_disparity_error_count,
3084             "RunningDisparityErrorCount", KSTAT_DATA_ULONGLONG);
3085         kstat_named_init(&ps->loss_of_dword_sync_count,
3086             "LossofDwordSyncCount", KSTAT_DATA_ULONGLONG);
3087         kstat_named_init(&ps->phy_reset_problem_count,
3088             "PhyResetProblemCount", KSTAT_DATA_ULONGLONG);
3089 
3090         phyp->phy_stats->ks_private = phyp;
3091         phyp->phy_stats->ks_update = pmcs_update_phy_stats;
3092         kstat_install(phyp->phy_stats);
3093 }
3094 
3095 static void
3096 pmcs_create_all_phy_stats(pmcs_iport_t *iport)
3097 {
3098         pmcs_hw_t               *pwp;
3099         pmcs_phy_t              *phyp;
3100 
3101         ASSERT(iport != NULL);
3102         pwp = iport->pwp;
3103         ASSERT(pwp != NULL);
3104 
3105         mutex_enter(&iport->lock);
3106 
3107         for (phyp = list_head(&iport->phys);
3108             phyp != NULL;
3109             phyp = list_next(&iport->phys, phyp)) {
3110 
3111                 mutex_enter(&phyp->phy_lock);
3112                 pmcs_create_one_phy_stats(iport, phyp);
3113                 mutex_exit(&phyp->phy_lock);
3114         }
3115 
3116         mutex_exit(&iport->lock);
3117 }
3118 
3119 int
3120 pmcs_update_phy_stats(kstat_t *ks, int rw)
3121 {
3122         int             val, ret = DDI_FAILURE;
3123         pmcs_phy_t      *pptr = (pmcs_phy_t *)ks->ks_private;
3124         pmcs_hw_t       *pwp = pptr->pwp;
3125         sas_phy_stats_t *ps = ks->ks_data;
3126 
3127         _NOTE(ARGUNUSED(rw));
3128         ASSERT((pptr != NULL) && (pwp != NULL));
3129 
3130         /*
3131          * We just want to lock against other invocations of kstat;
3132          * we don't need to pmcs_lock_phy() for this.
3133          */
3134         mutex_enter(&pptr->phy_lock);
3135 
3136         /* Get Stats from Chip */
3137         val = pmcs_get_diag_report(pwp, PMCS_INVALID_DWORD_CNT, pptr->phynum);
3138         if (val == DDI_FAILURE)
3139                 goto fail;
3140         ps->invalid_dword_count.value.ull = (unsigned long long)val;
3141 
3142         val = pmcs_get_diag_report(pwp, PMCS_DISPARITY_ERR_CNT, pptr->phynum);
3143         if (val == DDI_FAILURE)
3144                 goto fail;
3145         ps->running_disparity_error_count.value.ull = (unsigned long long)val;
3146 
3147         val = pmcs_get_diag_report(pwp, PMCS_LOST_DWORD_SYNC_CNT, pptr->phynum);
3148         if (val == DDI_FAILURE)
3149                 goto fail;
3150         ps->loss_of_dword_sync_count.value.ull = (unsigned long long)val;
3151 
3152         val = pmcs_get_diag_report(pwp, PMCS_RESET_FAILED_CNT, pptr->phynum);
3153         if (val == DDI_FAILURE)
3154                 goto fail;
3155         ps->phy_reset_problem_count.value.ull = (unsigned long long)val;
3156 
3157         ret = DDI_SUCCESS;
3158 fail:
3159         mutex_exit(&pptr->phy_lock);
3160         return (ret);
3161 }
3162 
3163 /*ARGSUSED*/
3164 static int
3165 pmcs_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3166 {
3167         /*
3168          * as the driver can always deal with an error in any dma or
3169          * access handle, we can just return the fme_status value.
3170          */
3171         pci_ereport_post(dip, err, NULL);
3172         return (err->fme_status);
3173 }
3174 
3175 static void
3176 pmcs_fm_init(pmcs_hw_t *pwp)
3177 {
3178         ddi_iblock_cookie_t     fm_ibc;
3179 
3180         /* Only register with IO Fault Services if we have some capability */
3181         if (pwp->fm_capabilities) {
3182                 /* Adjust access and dma attributes for FMA */
3183                 pwp->reg_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
3184                 pwp->iqp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3185                 pwp->oqp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3186                 pwp->cip_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3187                 pwp->fwlog_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3188 
3189                 /*
3190                  * Register capabilities with IO Fault Services.
3191                  */
3192                 ddi_fm_init(pwp->dip, &pwp->fm_capabilities, &fm_ibc);
3193 
3194                 /*
3195                  * Initialize pci ereport capabilities if ereport
3196                  * capable (should always be.)
3197                  */
3198                 if (DDI_FM_EREPORT_CAP(pwp->fm_capabilities) ||
3199                     DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3200                         pci_ereport_setup(pwp->dip);
3201                 }
3202 
3203                 /*
3204                  * Register error callback if error callback capable.
3205                  */
3206                 if (DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3207                         ddi_fm_handler_register(pwp->dip,
3208                             pmcs_fm_error_cb, (void *) pwp);
3209                 }
3210         }
3211 }
3212 
3213 static void
3214 pmcs_fm_fini(pmcs_hw_t *pwp)
3215 {
3216         /* Only unregister FMA capabilities if registered */
3217         if (pwp->fm_capabilities) {
3218                 /*
3219                  * Un-register error callback if error callback capable.
3220                  */
3221                 if (DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3222                         ddi_fm_handler_unregister(pwp->dip);
3223                 }
3224 
3225                 /*
3226                  * Release any resources allocated by pci_ereport_setup()
3227                  */
3228                 if (DDI_FM_EREPORT_CAP(pwp->fm_capabilities) ||
3229                     DDI_FM_ERRCB_CAP(pwp->fm_capabilities)) {
3230                         pci_ereport_teardown(pwp->dip);
3231                 }
3232 
3233                 /* Unregister from IO Fault Services */
3234                 ddi_fm_fini(pwp->dip);
3235 
3236                 /* Adjust access and dma attributes for FMA */
3237                 pwp->reg_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
3238                 pwp->iqp_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3239                 pwp->oqp_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3240                 pwp->cip_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3241                 pwp->fwlog_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3242         }
3243 }
3244 
3245 static boolean_t
3246 pmcs_fabricate_wwid(pmcs_hw_t *pwp)
3247 {
3248         char *cp, c;
3249         uint64_t adr;
3250         int i;
3251 
3252         cp = &c;
3253         (void) ddi_strtoul(hw_serial, &cp, 10, (unsigned long *)&adr);
3254 
3255         if (adr == 0) {
3256                 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
3257                     "%s: No serial number available to fabricate WWN",
3258                     __func__);
3259 
3260                 adr = (uint64_t)gethrtime();
3261         }
3262 
3263         adr <<= 8;
3264         adr |= ((uint64_t)ddi_get_instance(pwp->dip) << 52);
3265         adr |= (5ULL << 60);
3266 
3267         for (i = 0; i < PMCS_MAX_PORTS; i++) {
3268                 pwp->sas_wwns[i] = adr + i;
3269         }
3270 
3271         return (B_TRUE);
3272 }