1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved.
  24  * Copyright (c) 2012, Alexey Zaytsev <alexey.zaytsev@gmail.com>
  25  */
  26 
  27 
  28 #include <sys/modctl.h>
  29 #include <sys/blkdev.h>
  30 #include <sys/types.h>
  31 #include <sys/errno.h>
  32 #include <sys/param.h>
  33 #include <sys/stropts.h>
  34 #include <sys/stream.h>
  35 #include <sys/strsubr.h>
  36 #include <sys/kmem.h>
  37 #include <sys/conf.h>
  38 #include <sys/devops.h>
  39 #include <sys/ksynch.h>
  40 #include <sys/stat.h>
  41 #include <sys/modctl.h>
  42 #include <sys/debug.h>
  43 #include <sys/pci.h>
  44 #include <sys/sysmacros.h>
  45 #include "virtiovar.h"
  46 #include "virtioreg.h"
  47 
  48 /* Feature bits */
  49 #define VIRTIO_BLK_F_BARRIER    (1<<0)
  50 #define VIRTIO_BLK_F_SIZE_MAX   (1<<1)
  51 #define VIRTIO_BLK_F_SEG_MAX    (1<<2)
  52 #define VIRTIO_BLK_F_GEOMETRY   (1<<4)
  53 #define VIRTIO_BLK_F_RO         (1<<5)
  54 #define VIRTIO_BLK_F_BLK_SIZE   (1<<6)
  55 #define VIRTIO_BLK_F_SCSI       (1<<7)
  56 #define VIRTIO_BLK_F_FLUSH      (1<<9)
  57 #define VIRTIO_BLK_F_TOPOLOGY   (1<<10)
  58 
  59 /* Configuration registers */
  60 #define VIRTIO_BLK_CONFIG_CAPACITY      0 /* 64bit */
  61 #define VIRTIO_BLK_CONFIG_SIZE_MAX      8 /* 32bit */
  62 #define VIRTIO_BLK_CONFIG_SEG_MAX       12 /* 32bit */
  63 #define VIRTIO_BLK_CONFIG_GEOMETRY_C    16 /* 16bit */
  64 #define VIRTIO_BLK_CONFIG_GEOMETRY_H    18 /* 8bit */
  65 #define VIRTIO_BLK_CONFIG_GEOMETRY_S    19 /* 8bit */
  66 #define VIRTIO_BLK_CONFIG_BLK_SIZE      20 /* 32bit */
  67 #define VIRTIO_BLK_CONFIG_TOPO_PBEXP    24 /* 8bit */
  68 #define VIRTIO_BLK_CONFIG_TOPO_ALIGN    25 /* 8bit */
  69 #define VIRTIO_BLK_CONFIG_TOPO_MIN_SZ   26 /* 16bit */
  70 #define VIRTIO_BLK_CONFIG_TOPO_OPT_SZ   28 /* 32bit */
  71 
  72 /* Command */
  73 #define VIRTIO_BLK_T_IN                 0
  74 #define VIRTIO_BLK_T_OUT                1
  75 #define VIRTIO_BLK_T_SCSI_CMD           2
  76 #define VIRTIO_BLK_T_SCSI_CMD_OUT       3
  77 #define VIRTIO_BLK_T_FLUSH              4
  78 #define VIRTIO_BLK_T_FLUSH_OUT          5
  79 #define VIRTIO_BLK_T_GET_ID             8
  80 #define VIRTIO_BLK_T_BARRIER            0x80000000
  81 
  82 #define VIRTIO_BLK_ID_BYTES     20 /* devid */
  83 
  84 /* Statuses */
  85 #define VIRTIO_BLK_S_OK         0
  86 #define VIRTIO_BLK_S_IOERR      1
  87 #define VIRTIO_BLK_S_UNSUPP     2
  88 
  89 #define DEF_MAXINDIRECT         (128)
  90 #define DEF_MAXSECTOR           (4096)
  91 
  92 #define VIOBLK_POISON           0xdead0001dead0001
  93 
  94 /*
  95  * Static Variables.
  96  */
  97 static char vioblk_ident[] = "VirtIO block driver";
  98 
  99 /* Request header structure */
 100 struct vioblk_req_hdr {
 101         uint32_t                type;   /* VIRTIO_BLK_T_* */
 102         uint32_t                ioprio;
 103         uint64_t                sector;
 104 };
 105 
 106 struct vioblk_req {
 107         struct vioblk_req_hdr   hdr;
 108         uint8_t                 status;
 109         uint8_t                 unused[3];
 110         unsigned int            ndmac;
 111         ddi_dma_handle_t        dmah;
 112         ddi_dma_handle_t        bd_dmah;
 113         ddi_dma_cookie_t        dmac;
 114         bd_xfer_t               *xfer;
 115 };
 116 
 117 struct vioblk_stats {
 118         struct kstat_named      sts_rw_outofmemory;
 119         struct kstat_named      sts_rw_badoffset;
 120         struct kstat_named      sts_rw_queuemax;
 121         struct kstat_named      sts_rw_cookiesmax;
 122         struct kstat_named      sts_rw_cacheflush;
 123         struct kstat_named      sts_intr_queuemax;
 124         struct kstat_named      sts_intr_total;
 125         struct kstat_named      sts_io_errors;
 126         struct kstat_named      sts_unsupp_errors;
 127         struct kstat_named      sts_nxio_errors;
 128 };
 129 
 130 struct vioblk_lstats {
 131         uint64_t                rw_cacheflush;
 132         uint64_t                intr_total;
 133         unsigned int            rw_cookiesmax;
 134         unsigned int            intr_queuemax;
 135         unsigned int            io_errors;
 136         unsigned int            unsupp_errors;
 137         unsigned int            nxio_errors;
 138 };
 139 
 140 struct vioblk_softc {
 141         dev_info_t              *sc_dev; /* mirrors virtio_softc->sc_dev */
 142         struct virtio_softc     sc_virtio;
 143         struct virtqueue        *sc_vq;
 144         bd_handle_t             bd_h;
 145         struct vioblk_req       *sc_reqs;
 146         struct vioblk_stats     *ks_data;
 147         kstat_t                 *sc_intrstat;
 148         uint64_t                sc_capacity;
 149         uint64_t                sc_nblks;
 150         struct vioblk_lstats    sc_stats;
 151         short                   sc_blkflags;
 152         boolean_t               sc_in_poll_mode;
 153         boolean_t               sc_readonly;
 154         int                     sc_blk_size;
 155         int                     sc_pblk_size;
 156         int                     sc_seg_max;
 157         int                     sc_seg_size_max;
 158         kmutex_t                lock_devid;
 159         kcondvar_t              cv_devid;
 160         char                    devid[VIRTIO_BLK_ID_BYTES + 1];
 161 };
 162 
 163 static int vioblk_read(void *arg, bd_xfer_t *xfer);
 164 static int vioblk_write(void *arg, bd_xfer_t *xfer);
 165 static int vioblk_flush(void *arg, bd_xfer_t *xfer);
 166 static void vioblk_driveinfo(void *arg, bd_drive_t *drive);
 167 static int vioblk_mediainfo(void *arg, bd_media_t *media);
 168 static int vioblk_devid_init(void *, dev_info_t *, ddi_devid_t *);
 169 uint_t vioblk_int_handler(caddr_t arg1, caddr_t arg2);
 170 
 171 static bd_ops_t vioblk_ops = {
 172         BD_OPS_VERSION_0,
 173         vioblk_driveinfo,
 174         vioblk_mediainfo,
 175         vioblk_devid_init,
 176         vioblk_flush,
 177         vioblk_read,
 178         vioblk_write,
 179 };
 180 
 181 static int vioblk_quiesce(dev_info_t *);
 182 static int vioblk_attach(dev_info_t *, ddi_attach_cmd_t);
 183 static int vioblk_detach(dev_info_t *, ddi_detach_cmd_t);
 184 
 185 static struct dev_ops vioblk_dev_ops = {
 186         DEVO_REV,
 187         0,
 188         ddi_no_info,
 189         nulldev,        /* identify */
 190         nulldev,        /* probe */
 191         vioblk_attach,  /* attach */
 192         vioblk_detach,  /* detach */
 193         nodev,          /* reset */
 194         NULL,           /* cb_ops */
 195         NULL,           /* bus_ops */
 196         NULL,           /* power */
 197         vioblk_quiesce  /* quiesce */
 198 };
 199 
 200 
 201 
 202 /* Standard Module linkage initialization for a Streams driver */
 203 extern struct mod_ops mod_driverops;
 204 
 205 static struct modldrv modldrv = {
 206         &mod_driverops,             /* Type of module.  This one is a driver */
 207         vioblk_ident,    /* short description */
 208         &vioblk_dev_ops     /* driver specific ops */
 209 };
 210 
 211 static struct modlinkage modlinkage = {
 212         MODREV_1,
 213         {
 214                 (void *)&modldrv,
 215                 NULL,
 216         },
 217 };
 218 
 219 ddi_device_acc_attr_t vioblk_attr = {
 220         DDI_DEVICE_ATTR_V0,
 221         DDI_NEVERSWAP_ACC,      /* virtio is always native byte order */
 222         DDI_STORECACHING_OK_ACC,
 223         DDI_DEFAULT_ACC
 224 };
 225 
 226 /* DMA attr for the header/status blocks. */
 227 static ddi_dma_attr_t vioblk_req_dma_attr = {
 228         DMA_ATTR_V0,                    /* dma_attr version     */
 229         0,                              /* dma_attr_addr_lo     */
 230         0xFFFFFFFFFFFFFFFFull,          /* dma_attr_addr_hi     */
 231         0x00000000FFFFFFFFull,          /* dma_attr_count_max   */
 232         1,                              /* dma_attr_align       */
 233         1,                              /* dma_attr_burstsizes  */
 234         1,                              /* dma_attr_minxfer     */
 235         0xFFFFFFFFull,                  /* dma_attr_maxxfer     */
 236         0xFFFFFFFFFFFFFFFFull,          /* dma_attr_seg         */
 237         1,                              /* dma_attr_sgllen      */
 238         1,                              /* dma_attr_granular    */
 239         0,                              /* dma_attr_flags       */
 240 };
 241 
 242 /* DMA attr for the data blocks. */
 243 static ddi_dma_attr_t vioblk_bd_dma_attr = {
 244         DMA_ATTR_V0,                    /* dma_attr version     */
 245         0,                              /* dma_attr_addr_lo     */
 246         0xFFFFFFFFFFFFFFFFull,          /* dma_attr_addr_hi     */
 247         0x00000000FFFFFFFFull,          /* dma_attr_count_max   */
 248         1,                              /* dma_attr_align       */
 249         1,                              /* dma_attr_burstsizes  */
 250         1,                              /* dma_attr_minxfer     */
 251         0,                              /* dma_attr_maxxfer, set in attach */
 252         0xFFFFFFFFFFFFFFFFull,          /* dma_attr_seg         */
 253         0,                              /* dma_attr_sgllen, set in attach */
 254         1,                              /* dma_attr_granular    */
 255         0,                              /* dma_attr_flags       */
 256 };
 257 
 258 static int
 259 vioblk_rw(struct vioblk_softc *sc, bd_xfer_t *xfer, int type,
 260     uint32_t len)
 261 {
 262         struct vioblk_req *req;
 263         struct vq_entry *ve_hdr;
 264         int total_cookies, write;
 265 
 266         write = (type == VIRTIO_BLK_T_OUT ||
 267             type == VIRTIO_BLK_T_FLUSH_OUT) ? 1 : 0;
 268         total_cookies = 2;
 269 
 270         if ((xfer->x_blkno + xfer->x_nblks) > sc->sc_nblks) {
 271                 sc->ks_data->sts_rw_badoffset.value.ui64++;
 272                 return (EINVAL);
 273         }
 274 
 275         /* allocate top entry */
 276         ve_hdr = vq_alloc_entry(sc->sc_vq);
 277         if (!ve_hdr) {
 278                 sc->ks_data->sts_rw_outofmemory.value.ui64++;
 279                 return (ENOMEM);
 280         }
 281 
 282         /* getting request */
 283         req = &sc->sc_reqs[ve_hdr->qe_index];
 284         req->hdr.type = type;
 285         req->hdr.ioprio = 0;
 286         req->hdr.sector = xfer->x_blkno;
 287         req->xfer = xfer;
 288 
 289         /* Header */
 290         virtio_ve_add_indirect_buf(ve_hdr, req->dmac.dmac_laddress,
 291             sizeof (struct vioblk_req_hdr), B_TRUE);
 292 
 293         /* Payload */
 294         if (len > 0) {
 295                 virtio_ve_add_cookie(ve_hdr, xfer->x_dmah, xfer->x_dmac,
 296                     xfer->x_ndmac, write ? B_TRUE : B_FALSE);
 297                 total_cookies += xfer->x_ndmac;
 298         }
 299 
 300         /* Status */
 301         virtio_ve_add_indirect_buf(ve_hdr,
 302             req->dmac.dmac_laddress + sizeof (struct vioblk_req_hdr),
 303             sizeof (uint8_t), B_FALSE);
 304 
 305         /* sending the whole chain to the device */
 306         virtio_push_chain(ve_hdr, B_TRUE);
 307 
 308         if (sc->sc_stats.rw_cookiesmax < total_cookies)
 309                 sc->sc_stats.rw_cookiesmax = total_cookies;
 310 
 311         return (DDI_SUCCESS);
 312 }
 313 
 314 /*
 315  * Now in polling mode. Interrupts are off, so we
 316  * 1) poll for the already queued requests to complete.
 317  * 2) push our request.
 318  * 3) wait for our request to complete.
 319  */
 320 static int
 321 vioblk_rw_poll(struct vioblk_softc *sc, bd_xfer_t *xfer,
 322     int type, uint32_t len)
 323 {
 324         clock_t tmout;
 325         int ret;
 326 
 327         ASSERT(xfer->x_flags & BD_XFER_POLL);
 328 
 329         /* Prevent a hard hang. */
 330         tmout = drv_usectohz(30000000);
 331 
 332         /* Poll for an empty queue */
 333         while (vq_num_used(sc->sc_vq)) {
 334                 /* Check if any pending requests completed. */
 335                 ret = vioblk_int_handler((caddr_t)&sc->sc_virtio, NULL);
 336                 if (ret != DDI_INTR_CLAIMED) {
 337                         drv_usecwait(10);
 338                         tmout -= 10;
 339                         return (ETIMEDOUT);
 340                 }
 341         }
 342 
 343         ret = vioblk_rw(sc, xfer, type, len);
 344         if (ret)
 345                 return (ret);
 346 
 347         tmout = drv_usectohz(30000000);
 348         /* Poll for an empty queue again. */
 349         while (vq_num_used(sc->sc_vq)) {
 350                 /* Check if any pending requests completed. */
 351                 ret = vioblk_int_handler((caddr_t)&sc->sc_virtio, NULL);
 352                 if (ret != DDI_INTR_CLAIMED) {
 353                         drv_usecwait(10);
 354                         tmout -= 10;
 355                         return (ETIMEDOUT);
 356                 }
 357         }
 358 
 359         return (DDI_SUCCESS);
 360 }
 361 
 362 static int
 363 vioblk_read(void *arg, bd_xfer_t *xfer)
 364 {
 365         int ret;
 366         struct vioblk_softc *sc = (void *)arg;
 367 
 368         if (xfer->x_flags & BD_XFER_POLL) {
 369                 if (!sc->sc_in_poll_mode) {
 370                         virtio_stop_vq_intr(sc->sc_vq);
 371                         sc->sc_in_poll_mode = 1;
 372                 }
 373 
 374                 ret = vioblk_rw_poll(sc, xfer, VIRTIO_BLK_T_IN,
 375                     xfer->x_nblks * DEV_BSIZE);
 376         } else {
 377                 if (sc->sc_in_poll_mode) {
 378                         virtio_start_vq_intr(sc->sc_vq);
 379                         sc->sc_in_poll_mode = 0;
 380                 }
 381 
 382                 ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_IN,
 383                     xfer->x_nblks * DEV_BSIZE);
 384         }
 385 
 386         return (ret);
 387 }
 388 
 389 static int
 390 vioblk_write(void *arg, bd_xfer_t *xfer)
 391 {
 392         int ret;
 393         struct vioblk_softc *sc = (void *)arg;
 394 
 395         if (xfer->x_flags & BD_XFER_POLL) {
 396                 if (!sc->sc_in_poll_mode) {
 397                         virtio_stop_vq_intr(sc->sc_vq);
 398                         sc->sc_in_poll_mode = 1;
 399                 }
 400 
 401                 ret = vioblk_rw_poll(sc, xfer, VIRTIO_BLK_T_OUT,
 402                     xfer->x_nblks * DEV_BSIZE);
 403         } else {
 404                 if (sc->sc_in_poll_mode) {
 405                         virtio_start_vq_intr(sc->sc_vq);
 406                         sc->sc_in_poll_mode = 0;
 407                 }
 408 
 409                 ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_OUT,
 410                     xfer->x_nblks * DEV_BSIZE);
 411         }
 412         return (ret);
 413 }
 414 
 415 static int
 416 vioblk_flush(void *arg, bd_xfer_t *xfer)
 417 {
 418         int ret;
 419         struct vioblk_softc *sc = (void *)arg;
 420 
 421         ASSERT((xfer->x_flags & BD_XFER_POLL) == 0);
 422 
 423         ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_FLUSH_OUT,
 424             xfer->x_nblks * DEV_BSIZE);
 425 
 426         if (!ret)
 427                 sc->sc_stats.rw_cacheflush++;
 428 
 429         return (ret);
 430 }
 431 
 432 
 433 static void
 434 vioblk_driveinfo(void *arg, bd_drive_t *drive)
 435 {
 436         struct vioblk_softc *sc = (void *)arg;
 437 
 438         drive->d_qsize = sc->sc_vq->vq_num;
 439         drive->d_removable = B_FALSE;
 440         drive->d_hotpluggable = B_TRUE;
 441         drive->d_target = 0;
 442         drive->d_lun = 0;
 443 }
 444 
 445 static int
 446 vioblk_mediainfo(void *arg, bd_media_t *media)
 447 {
 448         struct vioblk_softc *sc = (void *)arg;
 449 
 450         media->m_nblks = sc->sc_nblks;
 451         media->m_blksize = sc->sc_blk_size;
 452         media->m_readonly = sc->sc_readonly;
 453         media->m_pblksize = sc->sc_pblk_size;
 454         return (0);
 455 }
 456 
 457 static int
 458 vioblk_devid_init(void *arg, dev_info_t *devinfo, ddi_devid_t *devid)
 459 {
 460         struct vioblk_softc *sc = (void *)arg;
 461         clock_t deadline;
 462         int ret;
 463         bd_xfer_t xfer;
 464 
 465         deadline = ddi_get_lbolt() + (clock_t)drv_usectohz(3 * 1000000);
 466         (void) memset(&xfer, 0, sizeof (bd_xfer_t));
 467         xfer.x_nblks = 1;
 468 
 469         ret = ddi_dma_alloc_handle(sc->sc_dev, &vioblk_bd_dma_attr,
 470             DDI_DMA_SLEEP, NULL, &xfer.x_dmah);
 471         if (ret != DDI_SUCCESS)
 472                 goto out_alloc;
 473 
 474         ret = ddi_dma_addr_bind_handle(xfer.x_dmah, NULL, (caddr_t)&sc->devid,
 475             VIRTIO_BLK_ID_BYTES, DDI_DMA_READ | DDI_DMA_CONSISTENT,
 476             DDI_DMA_SLEEP, NULL, &xfer.x_dmac, &xfer.x_ndmac);
 477         if (ret != DDI_DMA_MAPPED) {
 478                 ret = DDI_FAILURE;
 479                 goto out_map;
 480         }
 481 
 482         mutex_enter(&sc->lock_devid);
 483 
 484         ret = vioblk_rw(sc, &xfer, VIRTIO_BLK_T_GET_ID,
 485             VIRTIO_BLK_ID_BYTES);
 486         if (ret) {
 487                 mutex_exit(&sc->lock_devid);
 488                 goto out_rw;
 489         }
 490 
 491         /* wait for reply */
 492         ret = cv_timedwait(&sc->cv_devid, &sc->lock_devid, deadline);
 493         mutex_exit(&sc->lock_devid);
 494 
 495         (void) ddi_dma_unbind_handle(xfer.x_dmah);
 496         ddi_dma_free_handle(&xfer.x_dmah);
 497 
 498         /* timeout */
 499         if (ret < 0) {
 500                 dev_err(devinfo, CE_WARN, "Cannot get devid from the device");
 501                 return (DDI_FAILURE);
 502         }
 503 
 504         ret = ddi_devid_init(devinfo, DEVID_ATA_SERIAL,
 505             VIRTIO_BLK_ID_BYTES, sc->devid, devid);
 506         if (ret != DDI_SUCCESS) {
 507                 dev_err(devinfo, CE_WARN, "Cannot build devid from the device");
 508                 return (ret);
 509         }
 510 
 511         dev_debug(sc->sc_dev, CE_NOTE,
 512             "devid %x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x",
 513             sc->devid[0], sc->devid[1], sc->devid[2], sc->devid[3],
 514             sc->devid[4], sc->devid[5], sc->devid[6], sc->devid[7],
 515             sc->devid[8], sc->devid[9], sc->devid[10], sc->devid[11],
 516             sc->devid[12], sc->devid[13], sc->devid[14], sc->devid[15],
 517             sc->devid[16], sc->devid[17], sc->devid[18], sc->devid[19]);
 518 
 519         return (0);
 520 
 521 out_rw:
 522         (void) ddi_dma_unbind_handle(xfer.x_dmah);
 523 out_map:
 524         ddi_dma_free_handle(&xfer.x_dmah);
 525 out_alloc:
 526         return (ret);
 527 }
 528 
 529 static void
 530 vioblk_show_features(struct vioblk_softc *sc, const char *prefix,
 531     uint32_t features)
 532 {
 533         char buf[512];
 534         char *bufp = buf;
 535         char *bufend = buf + sizeof (buf);
 536 
 537         /* LINTED E_PTRDIFF_OVERFLOW */
 538         bufp += snprintf(bufp, bufend - bufp, prefix);
 539 
 540         /* LINTED E_PTRDIFF_OVERFLOW */
 541         bufp += virtio_show_features(features, bufp, bufend - bufp);
 542 
 543 
 544         /* LINTED E_PTRDIFF_OVERFLOW */
 545         bufp += snprintf(bufp, bufend - bufp, "Vioblk ( ");
 546 
 547         if (features & VIRTIO_BLK_F_BARRIER)
 548                 /* LINTED E_PTRDIFF_OVERFLOW */
 549                 bufp += snprintf(bufp, bufend - bufp, "BARRIER ");
 550         if (features & VIRTIO_BLK_F_SIZE_MAX)
 551                 /* LINTED E_PTRDIFF_OVERFLOW */
 552                 bufp += snprintf(bufp, bufend - bufp, "SIZE_MAX ");
 553         if (features & VIRTIO_BLK_F_SEG_MAX)
 554                 /* LINTED E_PTRDIFF_OVERFLOW */
 555                 bufp += snprintf(bufp, bufend - bufp, "SEG_MAX ");
 556         if (features & VIRTIO_BLK_F_GEOMETRY)
 557                 /* LINTED E_PTRDIFF_OVERFLOW */
 558                 bufp += snprintf(bufp, bufend - bufp, "GEOMETRY ");
 559         if (features & VIRTIO_BLK_F_RO)
 560                 /* LINTED E_PTRDIFF_OVERFLOW */
 561                 bufp += snprintf(bufp, bufend - bufp, "RO ");
 562         if (features & VIRTIO_BLK_F_BLK_SIZE)
 563                 /* LINTED E_PTRDIFF_OVERFLOW */
 564                 bufp += snprintf(bufp, bufend - bufp, "BLK_SIZE ");
 565         if (features & VIRTIO_BLK_F_SCSI)
 566                 /* LINTED E_PTRDIFF_OVERFLOW */
 567                 bufp += snprintf(bufp, bufend - bufp, "SCSI ");
 568         if (features & VIRTIO_BLK_F_FLUSH)
 569                 /* LINTED E_PTRDIFF_OVERFLOW */
 570                 bufp += snprintf(bufp, bufend - bufp, "FLUSH ");
 571         if (features & VIRTIO_BLK_F_TOPOLOGY)
 572                 /* LINTED E_PTRDIFF_OVERFLOW */
 573                 bufp += snprintf(bufp, bufend - bufp, "TOPOLOGY ");
 574 
 575         /* LINTED E_PTRDIFF_OVERFLOW */
 576         bufp += snprintf(bufp, bufend - bufp, ")");
 577         *bufp = '\0';
 578 
 579         dev_debug(sc->sc_dev, CE_NOTE, "%s", buf);
 580 }
 581 
 582 static int
 583 vioblk_dev_features(struct vioblk_softc *sc)
 584 {
 585         uint32_t host_features;
 586 
 587         host_features = virtio_negotiate_features(&sc->sc_virtio,
 588             VIRTIO_BLK_F_RO |
 589             VIRTIO_BLK_F_GEOMETRY |
 590             VIRTIO_BLK_F_BLK_SIZE |
 591             VIRTIO_BLK_F_FLUSH |
 592             VIRTIO_BLK_F_TOPOLOGY |
 593             VIRTIO_BLK_F_SEG_MAX |
 594             VIRTIO_BLK_F_SIZE_MAX |
 595             VIRTIO_F_RING_INDIRECT_DESC);
 596 
 597         vioblk_show_features(sc, "Host features: ", host_features);
 598         vioblk_show_features(sc, "Negotiated features: ",
 599             sc->sc_virtio.sc_features);
 600 
 601         if (!(sc->sc_virtio.sc_features & VIRTIO_F_RING_INDIRECT_DESC)) {
 602                 dev_err(sc->sc_dev, CE_NOTE,
 603                     "Host does not support RING_INDIRECT_DESC, bye.");
 604                 return (DDI_FAILURE);
 605         }
 606 
 607         return (DDI_SUCCESS);
 608 }
 609 
 610 /* ARGSUSED */
 611 uint_t
 612 vioblk_int_handler(caddr_t arg1, caddr_t arg2)
 613 {
 614         struct virtio_softc *vsc = (void *)arg1;
 615         struct vioblk_softc *sc = container_of(vsc,
 616             struct vioblk_softc, sc_virtio);
 617         struct vq_entry *ve;
 618         uint32_t len;
 619         int i = 0, error;
 620 
 621         while ((ve = virtio_pull_chain(sc->sc_vq, &len))) {
 622                 struct vioblk_req *req = &sc->sc_reqs[ve->qe_index];
 623                 bd_xfer_t *xfer = req->xfer;
 624                 uint8_t status = req->status;
 625                 uint32_t type = req->hdr.type;
 626 
 627                 if (req->xfer == (void *)VIOBLK_POISON) {
 628                         dev_err(sc->sc_dev, CE_WARN, "Poisoned descriptor!");
 629                         virtio_free_chain(ve);
 630                         return (DDI_INTR_CLAIMED);
 631                 }
 632 
 633                 req->xfer = (void *) VIOBLK_POISON;
 634 
 635                 /* Note: blkdev tears down the payload mapping for us. */
 636                 virtio_free_chain(ve);
 637 
 638                 /* returning payload back to blkdev */
 639                 switch (status) {
 640                         case VIRTIO_BLK_S_OK:
 641                                 error = 0;
 642                                 break;
 643                         case VIRTIO_BLK_S_IOERR:
 644                                 error = EIO;
 645                                 sc->sc_stats.io_errors++;
 646                                 break;
 647                         case VIRTIO_BLK_S_UNSUPP:
 648                                 sc->sc_stats.unsupp_errors++;
 649                                 error = ENOTTY;
 650                                 break;
 651                         default:
 652                                 sc->sc_stats.nxio_errors++;
 653                                 error = ENXIO;
 654                                 break;
 655                 }
 656 
 657                 if (type == VIRTIO_BLK_T_GET_ID) {
 658                         /* notify devid_init */
 659                         mutex_enter(&sc->lock_devid);
 660                         cv_broadcast(&sc->cv_devid);
 661                         mutex_exit(&sc->lock_devid);
 662                 } else
 663                         bd_xfer_done(xfer, error);
 664 
 665                 i++;
 666         }
 667 
 668         /* update stats */
 669         if (sc->sc_stats.intr_queuemax < i)
 670                 sc->sc_stats.intr_queuemax = i;
 671         sc->sc_stats.intr_total++;
 672 
 673         return (DDI_INTR_CLAIMED);
 674 }
 675 
 676 /* ARGSUSED */
 677 uint_t
 678 vioblk_config_handler(caddr_t arg1, caddr_t arg2)
 679 {
 680         return (DDI_INTR_CLAIMED);
 681 }
 682 
 683 static int
 684 vioblk_register_ints(struct vioblk_softc *sc)
 685 {
 686         int ret;
 687 
 688         struct virtio_int_handler vioblk_conf_h = {
 689                 vioblk_config_handler
 690         };
 691 
 692         struct virtio_int_handler vioblk_vq_h[] = {
 693                 { vioblk_int_handler },
 694                 { NULL },
 695         };
 696 
 697         ret = virtio_register_ints(&sc->sc_virtio,
 698             &vioblk_conf_h, vioblk_vq_h);
 699 
 700         return (ret);
 701 }
 702 
 703 static void
 704 vioblk_free_reqs(struct vioblk_softc *sc)
 705 {
 706         int i, qsize;
 707 
 708         qsize = sc->sc_vq->vq_num;
 709 
 710         for (i = 0; i < qsize; i++) {
 711                 struct vioblk_req *req = &sc->sc_reqs[i];
 712 
 713                 if (req->ndmac)
 714                         (void) ddi_dma_unbind_handle(req->dmah);
 715 
 716                 if (req->dmah)
 717                         ddi_dma_free_handle(&req->dmah);
 718         }
 719 
 720         kmem_free(sc->sc_reqs, sizeof (struct vioblk_req) * qsize);
 721 }
 722 
 723 static int
 724 vioblk_alloc_reqs(struct vioblk_softc *sc)
 725 {
 726         int i, qsize;
 727         int ret;
 728 
 729         qsize = sc->sc_vq->vq_num;
 730 
 731         sc->sc_reqs = kmem_zalloc(sizeof (struct vioblk_req) * qsize, KM_SLEEP);
 732 
 733         for (i = 0; i < qsize; i++) {
 734                 struct vioblk_req *req = &sc->sc_reqs[i];
 735 
 736                 ret = ddi_dma_alloc_handle(sc->sc_dev, &vioblk_req_dma_attr,
 737                     DDI_DMA_SLEEP, NULL, &req->dmah);
 738                 if (ret != DDI_SUCCESS) {
 739 
 740                         dev_err(sc->sc_dev, CE_WARN,
 741                             "Can't allocate dma handle for req "
 742                             "buffer %d", i);
 743                         goto exit;
 744                 }
 745 
 746                 ret = ddi_dma_addr_bind_handle(req->dmah, NULL,
 747                     (caddr_t)&req->hdr,
 748                     sizeof (struct vioblk_req_hdr) + sizeof (uint8_t),
 749                     DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP,
 750                     NULL, &req->dmac, &req->ndmac);
 751                 if (ret != DDI_DMA_MAPPED) {
 752                         dev_err(sc->sc_dev, CE_WARN,
 753                             "Can't bind req buffer %d", i);
 754                         goto exit;
 755                 }
 756         }
 757 
 758         return (0);
 759 
 760 exit:
 761         vioblk_free_reqs(sc);
 762         return (ENOMEM);
 763 }
 764 
 765 
 766 static int
 767 vioblk_ksupdate(kstat_t *ksp, int rw)
 768 {
 769         struct vioblk_softc *sc = ksp->ks_private;
 770 
 771         if (rw == KSTAT_WRITE)
 772                 return (EACCES);
 773 
 774         sc->ks_data->sts_rw_cookiesmax.value.ui32 = sc->sc_stats.rw_cookiesmax;
 775         sc->ks_data->sts_intr_queuemax.value.ui32 = sc->sc_stats.intr_queuemax;
 776         sc->ks_data->sts_unsupp_errors.value.ui32 = sc->sc_stats.unsupp_errors;
 777         sc->ks_data->sts_nxio_errors.value.ui32 = sc->sc_stats.nxio_errors;
 778         sc->ks_data->sts_io_errors.value.ui32 = sc->sc_stats.io_errors;
 779         sc->ks_data->sts_rw_cacheflush.value.ui64 = sc->sc_stats.rw_cacheflush;
 780         sc->ks_data->sts_intr_total.value.ui64 = sc->sc_stats.intr_total;
 781 
 782 
 783         return (0);
 784 }
 785 
 786 static int
 787 vioblk_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
 788 {
 789         int ret = DDI_SUCCESS;
 790         int instance;
 791         struct vioblk_softc *sc;
 792         struct virtio_softc *vsc;
 793         struct vioblk_stats *ks_data;
 794 
 795         instance = ddi_get_instance(devinfo);
 796 
 797         switch (cmd) {
 798         case DDI_ATTACH:
 799                 break;
 800 
 801         case DDI_RESUME:
 802         case DDI_PM_RESUME:
 803                 dev_err(devinfo, CE_WARN, "resume not supported yet");
 804                 ret = DDI_FAILURE;
 805                 goto exit;
 806 
 807         default:
 808                 dev_err(devinfo, CE_WARN, "cmd 0x%x not recognized", cmd);
 809                 ret = DDI_FAILURE;
 810                 goto exit;
 811         }
 812 
 813         sc = kmem_zalloc(sizeof (struct vioblk_softc), KM_SLEEP);
 814         ddi_set_driver_private(devinfo, sc);
 815 
 816         vsc = &sc->sc_virtio;
 817 
 818         /* Duplicate for faster access / less typing */
 819         sc->sc_dev = devinfo;
 820         vsc->sc_dev = devinfo;
 821 
 822         cv_init(&sc->cv_devid, NULL, CV_DRIVER, NULL);
 823         mutex_init(&sc->lock_devid, NULL, MUTEX_DRIVER, NULL);
 824 
 825         /*
 826          * Initialize interrupt kstat.  This should not normally fail, since
 827          * we don't use a persistent stat.  We do it this way to avoid having
 828          * to test for it at run time on the hot path.
 829          */
 830         sc->sc_intrstat = kstat_create("vioblk", instance,
 831             "intrs", "controller", KSTAT_TYPE_NAMED,
 832             sizeof (struct vioblk_stats) / sizeof (kstat_named_t),
 833             KSTAT_FLAG_PERSISTENT);
 834         if (sc->sc_intrstat == NULL) {
 835                 dev_err(devinfo, CE_WARN, "kstat_create failed");
 836                 goto exit_intrstat;
 837         }
 838         ks_data = (struct vioblk_stats *)sc->sc_intrstat->ks_data;
 839         kstat_named_init(&ks_data->sts_rw_outofmemory,
 840             "total_rw_outofmemory", KSTAT_DATA_UINT64);
 841         kstat_named_init(&ks_data->sts_rw_badoffset,
 842             "total_rw_badoffset", KSTAT_DATA_UINT64);
 843         kstat_named_init(&ks_data->sts_intr_total,
 844             "total_intr", KSTAT_DATA_UINT64);
 845         kstat_named_init(&ks_data->sts_io_errors,
 846             "total_io_errors", KSTAT_DATA_UINT32);
 847         kstat_named_init(&ks_data->sts_unsupp_errors,
 848             "total_unsupp_errors", KSTAT_DATA_UINT32);
 849         kstat_named_init(&ks_data->sts_nxio_errors,
 850             "total_nxio_errors", KSTAT_DATA_UINT32);
 851         kstat_named_init(&ks_data->sts_rw_cacheflush,
 852             "total_rw_cacheflush", KSTAT_DATA_UINT64);
 853         kstat_named_init(&ks_data->sts_rw_cookiesmax,
 854             "max_rw_cookies", KSTAT_DATA_UINT32);
 855         kstat_named_init(&ks_data->sts_intr_queuemax,
 856             "max_intr_queue", KSTAT_DATA_UINT32);
 857         sc->ks_data = ks_data;
 858         sc->sc_intrstat->ks_private = sc;
 859         sc->sc_intrstat->ks_update = vioblk_ksupdate;
 860         kstat_install(sc->sc_intrstat);
 861 
 862         /* map BAR0 */
 863         ret = ddi_regs_map_setup(devinfo, 1,
 864             (caddr_t *)&sc->sc_virtio.sc_io_addr,
 865             0, 0, &vioblk_attr, &sc->sc_virtio.sc_ioh);
 866         if (ret != DDI_SUCCESS) {
 867                 dev_err(devinfo, CE_WARN, "unable to map bar0: [%d]", ret);
 868                 goto exit_map;
 869         }
 870 
 871         virtio_device_reset(&sc->sc_virtio);
 872         virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
 873         virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
 874 
 875         if (vioblk_register_ints(sc)) {
 876                 dev_err(devinfo, CE_WARN, "Unable to add interrupt");
 877                 goto exit_int;
 878         }
 879 
 880         ret = vioblk_dev_features(sc);
 881         if (ret)
 882                 goto exit_features;
 883 
 884         if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_RO)
 885                 sc->sc_readonly = B_TRUE;
 886         else
 887                 sc->sc_readonly = B_FALSE;
 888 
 889         sc->sc_capacity = virtio_read_device_config_8(&sc->sc_virtio,
 890             VIRTIO_BLK_CONFIG_CAPACITY);
 891         sc->sc_nblks = sc->sc_capacity;
 892 
 893         sc->sc_blk_size = DEV_BSIZE;
 894         if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_BLK_SIZE) {
 895                 sc->sc_blk_size = virtio_read_device_config_4(&sc->sc_virtio,
 896                     VIRTIO_BLK_CONFIG_BLK_SIZE);
 897         }
 898 
 899         sc->sc_pblk_size = sc->sc_blk_size;
 900         if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_TOPOLOGY) {
 901                 sc->sc_pblk_size <<= virtio_read_device_config_1(&sc->sc_virtio,
 902                     VIRTIO_BLK_CONFIG_TOPO_PBEXP);
 903         }
 904 
 905         /* Flushing is not supported. */
 906         if (!(sc->sc_virtio.sc_features & VIRTIO_BLK_F_FLUSH)) {
 907                 vioblk_ops.o_sync_cache = NULL;
 908         }
 909 
 910         sc->sc_seg_max = DEF_MAXINDIRECT;
 911         /* The max number of segments (cookies) in a request */
 912         if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_SEG_MAX) {
 913                 sc->sc_seg_max = virtio_read_device_config_4(&sc->sc_virtio,
 914                     VIRTIO_BLK_CONFIG_SEG_MAX);
 915 
 916                 /* That's what Linux does. */
 917                 if (!sc->sc_seg_max)
 918                         sc->sc_seg_max = 1;
 919 
 920                 /*
 921                  * SEG_MAX corresponds to the number of _data_
 922                  * blocks in a request
 923                  */
 924                 sc->sc_seg_max += 2;
 925         }
 926         /* 2 descriptors taken for header/status */
 927         vioblk_bd_dma_attr.dma_attr_sgllen = sc->sc_seg_max - 2;
 928 
 929 
 930         /* The maximum size for a cookie in a request. */
 931         sc->sc_seg_size_max = DEF_MAXSECTOR;
 932         if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_SIZE_MAX) {
 933                 sc->sc_seg_size_max = virtio_read_device_config_4(
 934                     &sc->sc_virtio, VIRTIO_BLK_CONFIG_SIZE_MAX);
 935         }
 936 
 937         /* The maximum request size */
 938         vioblk_bd_dma_attr.dma_attr_maxxfer =
 939             vioblk_bd_dma_attr.dma_attr_sgllen * sc->sc_seg_size_max;
 940 
 941         dev_debug(devinfo, CE_NOTE,
 942             "nblks=%" PRIu64 " blksize=%d (%d) num_seg=%d, "
 943             "seg_size=%d, maxxfer=%" PRIu64,
 944             sc->sc_nblks, sc->sc_blk_size, sc->sc_pblk_size,
 945             vioblk_bd_dma_attr.dma_attr_sgllen,
 946             sc->sc_seg_size_max,
 947             vioblk_bd_dma_attr.dma_attr_maxxfer);
 948 
 949 
 950         sc->sc_vq = virtio_alloc_vq(&sc->sc_virtio, 0, 0,
 951             sc->sc_seg_max, "I/O request");
 952         if (sc->sc_vq == NULL) {
 953                 goto exit_alloc1;
 954         }
 955 
 956         ret = vioblk_alloc_reqs(sc);
 957         if (ret) {
 958                 goto exit_alloc2;
 959         }
 960 
 961         sc->bd_h = bd_alloc_handle(sc, &vioblk_ops, &vioblk_bd_dma_attr,
 962             KM_SLEEP);
 963 
 964 
 965         virtio_set_status(&sc->sc_virtio,
 966             VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
 967         virtio_start_vq_intr(sc->sc_vq);
 968 
 969         ret = virtio_enable_ints(&sc->sc_virtio);
 970         if (ret)
 971                 goto exit_enable_ints;
 972 
 973         ret = bd_attach_handle(devinfo, sc->bd_h);
 974         if (ret != DDI_SUCCESS) {
 975                 dev_err(devinfo, CE_WARN, "Failed to attach blkdev");
 976                 goto exit_attach_bd;
 977         }
 978 
 979         return (DDI_SUCCESS);
 980 
 981 exit_attach_bd:
 982         /*
 983          * There is no virtio_disable_ints(), it's done in virtio_release_ints.
 984          * If they ever get split, don't forget to add a call here.
 985          */
 986 exit_enable_ints:
 987         virtio_stop_vq_intr(sc->sc_vq);
 988         bd_free_handle(sc->bd_h);
 989         vioblk_free_reqs(sc);
 990 exit_alloc2:
 991         virtio_free_vq(sc->sc_vq);
 992 exit_alloc1:
 993 exit_features:
 994         virtio_release_ints(&sc->sc_virtio);
 995 exit_int:
 996         virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
 997         ddi_regs_map_free(&sc->sc_virtio.sc_ioh);
 998 exit_map:
 999         kstat_delete(sc->sc_intrstat);
1000 exit_intrstat:
1001         mutex_destroy(&sc->lock_devid);
1002         cv_destroy(&sc->cv_devid);
1003         kmem_free(sc, sizeof (struct vioblk_softc));
1004 exit:
1005         return (ret);
1006 }
1007 
1008 static int
1009 vioblk_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
1010 {
1011         struct vioblk_softc *sc = ddi_get_driver_private(devinfo);
1012 
1013         switch (cmd) {
1014         case DDI_DETACH:
1015                 break;
1016 
1017         case DDI_PM_SUSPEND:
1018                 cmn_err(CE_WARN, "suspend not supported yet");
1019                 return (DDI_FAILURE);
1020 
1021         default:
1022                 cmn_err(CE_WARN, "cmd 0x%x unrecognized", cmd);
1023                 return (DDI_FAILURE);
1024         }
1025 
1026         (void) bd_detach_handle(sc->bd_h);
1027         virtio_stop_vq_intr(sc->sc_vq);
1028         virtio_release_ints(&sc->sc_virtio);
1029         vioblk_free_reqs(sc);
1030         virtio_free_vq(sc->sc_vq);
1031         virtio_device_reset(&sc->sc_virtio);
1032         ddi_regs_map_free(&sc->sc_virtio.sc_ioh);
1033         kstat_delete(sc->sc_intrstat);
1034         kmem_free(sc, sizeof (struct vioblk_softc));
1035 
1036         return (DDI_SUCCESS);
1037 }
1038 
1039 static int
1040 vioblk_quiesce(dev_info_t *devinfo)
1041 {
1042         struct vioblk_softc *sc = ddi_get_driver_private(devinfo);
1043 
1044         virtio_stop_vq_intr(sc->sc_vq);
1045         virtio_device_reset(&sc->sc_virtio);
1046 
1047         return (DDI_SUCCESS);
1048 }
1049 
1050 int
1051 _init(void)
1052 {
1053         int rv;
1054 
1055         bd_mod_init(&vioblk_dev_ops);
1056 
1057         if ((rv = mod_install(&modlinkage)) != 0) {
1058                 bd_mod_fini(&vioblk_dev_ops);
1059         }
1060 
1061         return (rv);
1062 }
1063 
1064 int
1065 _fini(void)
1066 {
1067         int rv;
1068 
1069         if ((rv = mod_remove(&modlinkage)) == 0) {
1070                 bd_mod_fini(&vioblk_dev_ops);
1071         }
1072 
1073         return (rv);
1074 }
1075 
1076 int
1077 _info(struct modinfo *modinfop)
1078 {
1079         return (mod_info(&modlinkage, modinfop));
1080 }